diff --git a/.changeset/exemplar-mode-metrics.md b/.changeset/exemplar-mode-metrics.md new file mode 100644 index 0000000000..dda84d862c --- /dev/null +++ b/.changeset/exemplar-mode-metrics.md @@ -0,0 +1,22 @@ +--- +"@hyperdx/common-utils": minor +"@hyperdx/api": minor +"@hyperdx/app": minor +--- + +feat: add exemplar overlay for metric and PromQL charts + +Time charts on metric and PromQL sources can now overlay exemplars — +individual data points linked to a trace — via the "Exemplars" toggle in the +chart editor (next to "As Ratio" for metric charts, in the PromQL editor for +PromQL charts). Markers snap onto the series line so the chart stays honest; +hovering a marker shows trace metadata (service, span, duration, status) from a +configurable exemplar trace source, with a button to open the trace directly. + +For structured metric sources, exemplars are read directly from the OTel metric +tables' `Exemplars.*` columns (`renderMetricExemplarsChartConfig`), honoring the +chart's time range, metric name, and filters. For PromQL sources backed by a +real Prometheus endpoint, the new `/v1/prometheus/query_exemplars` route proxies +to Prometheus's native `/api/v1/query_exemplars`. The overlay is opt-in and runs +its query in parallel only when enabled, so charts that don't use it are +unaffected. Trace-source exemplar generation lands in a follow-up. diff --git a/.changeset/span-metrics-connector.md b/.changeset/span-metrics-connector.md new file mode 100644 index 0000000000..002e942870 --- /dev/null +++ b/.changeset/span-metrics-connector.md @@ -0,0 +1,15 @@ +--- +"@hyperdx/api": minor +"@hyperdx/otel-collector": minor +--- + +feat: optional spanmetrics connector for metric exemplars + +Adds the `spanmetricsconnector` to the collector build and wires it into the +OpAMP-generated collector config, gated on the `ENABLE_SPAN_METRICS` env flag +(off by default). When enabled, the collector derives `traces.span.metrics.*` +(calls + duration histogram) from spans with **exemplars enabled**, so the +duration histogram lands in ClickHouse with `Exemplars.*` pointing back at the +spans they were measured from — giving coherent, fully-OTLP metric exemplars +without any direct ClickHouse writes. Enabled in local dev to back the new +`telemetry-generator` service. diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml index 95e882eca0..7ec6efc04a 100644 --- a/docker-compose.dev.yml +++ b/docker-compose.dev.yml @@ -41,6 +41,9 @@ services: OTEL_SUPERVISOR_LOGS: 'true' HYPERDX_OTEL_EXPORTER_TABLES_TTL: '24h' ENABLE_PROMQL: 'true' + # Remote-write the span-derived metrics (with exemplars) to the dev + # Prometheus so the native query_exemplars path is testable with real data. + SPAN_METRICS_PROM_RW_ENDPOINT: 'http://prometheus:9090/api/v1/write' volumes: - ./docker/otel-collector/config.yaml:/etc/otelcol-contrib/config.yaml - ./docker/otel-collector/supervisor_docker.yaml.tmpl:/etc/otel/supervisor.yaml.tmpl @@ -157,9 +160,15 @@ services: hdx.dev.service: prometheus hdx.dev.port: '${HDX_DEV_PROMETHEUS_PORT:-9090}' hdx.dev.url: 'http://localhost:${HDX_DEV_PROMETHEUS_PORT:-9090}' - profiles: - - prometheus image: prom/prometheus:latest + # exemplar-storage: query_exemplars data; remote-write receiver: ingest the + # collector's span-derived metrics (with exemplars) from real generated data. + command: + - '--config.file=/etc/prometheus/prometheus.yml' + - '--storage.tsdb.path=/prometheus' + - '--enable-feature=exemplar-storage' + - '--web.enable-remote-write-receiver' + - '--web.enable-lifecycle' ports: - '${HDX_DEV_PROMETHEUS_PORT:-9090}:9090' volumes: @@ -169,5 +178,25 @@ services: - internal restart: on-failure + # Synthetic traces (via OTLP) + coherent metric exemplars (to ClickHouse) for + # local dev + e2e. See telemetry-generator/README.md. + telemetry-generator: + labels: + <<: *hdx-labels + hdx.dev.service: telemetry-generator + build: + context: ./telemetry-generator + environment: + OTEL_EXPORTER_OTLP_ENDPOINT: 'http://otel-collector:4318' + GEN_OTLP_API_KEY: '${INGESTION_API_KEY:-super-secure-ingestion-api-key}' + GEN_BACKFILL_MINUTES: '30' + GEN_RATE_PER_SEC: '20' + networks: + - internal + restart: on-failure + depends_on: + otel-collector: + condition: service_started + networks: internal: diff --git a/docker-compose.yml b/docker-compose.yml index bdd7339e03..7a2ae855c5 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -74,6 +74,8 @@ services: # Uncomment the next two lines to enable PromQL (Prometheus-compatible metrics) # ENABLE_PROMQL: 'true' # NEXT_PUBLIC_ENABLE_PROMQL: 'true' + # Uncomment to enable exemplar overlays (trace markers on time charts) + # NEXT_PUBLIC_ENABLE_EXEMPLARS: 'true' DEFAULT_CONNECTIONS: '[{"name":"Local ClickHouse","host":"http://ch-server:8123","username":"default","password":""}]' diff --git a/docker/prometheus/prometheus.yml b/docker/prometheus/prometheus.yml index 94877ac65d..ff6643b21f 100644 --- a/docker/prometheus/prometheus.yml +++ b/docker/prometheus/prometheus.yml @@ -18,3 +18,7 @@ scrape_configs: static_configs: - targets: ['ch-server:9363'] metrics_path: '/metrics' + + # Note: the span-derived request metrics (with exemplars) arrive via + # remote-write from the OTel collector, not a scrape — see the collector's + # SPAN_METRICS_PROM_RW_ENDPOINT and the spanmetrics connector. diff --git a/knip.json b/knip.json index 8e9cf032c8..420285cfc7 100644 --- a/knip.json +++ b/knip.json @@ -39,6 +39,7 @@ ".github/scripts/**", ".github/actions/**", "docker/hyperdx/**", + "telemetry-generator/**", ".claude/**" ], "ignoreBinaries": ["make", "migrate"], diff --git a/packages/api/.env.development b/packages/api/.env.development index 8ba0a6c0c8..a8ec2bd9ca 100644 --- a/packages/api/.env.development +++ b/packages/api/.env.development @@ -20,9 +20,11 @@ REDIS_URL=redis://localhost:6379 USAGE_STATS_ENABLED=false NODE_OPTIONS="--max-http-header-size=131072" ENABLE_SWAGGER=true -DEFAULT_CONNECTIONS=[{"name":"Local ClickHouse","host":"http://localhost:${HDX_DEV_CH_HTTP_PORT}","username":"default","password":""}] -DEFAULT_SOURCES=[{"from":{"databaseName":"default","tableName":"otel_logs"},"kind":"log","timestampValueExpression":"Timestamp","name":"Logs","displayedTimestampValueExpression":"Timestamp","implicitColumnExpression":"Body","serviceNameExpression":"ServiceName","bodyExpression":"Body","eventAttributesExpression":"LogAttributes","resourceAttributesExpression":"ResourceAttributes","defaultTableSelectExpression":"Timestamp,ServiceName,SeverityText,Body","severityTextExpression":"SeverityText","traceIdExpression":"TraceId","spanIdExpression":"SpanId","metadataMaterializedViews":{"keyRollupTable":"otel_logs_key_rollup_15m","kvRollupTable":"otel_logs_kv_rollup_15m","granularity":"15 minute"},"connection":"Local ClickHouse","traceSourceId":"Traces","sessionSourceId":"Sessions","metricSourceId":"Metrics"},{"from":{"databaseName":"default","tableName":"otel_traces"},"kind":"trace","timestampValueExpression":"Timestamp","name":"Traces","displayedTimestampValueExpression":"Timestamp","implicitColumnExpression":"SpanName","serviceNameExpression":"ServiceName","eventAttributesExpression":"SpanAttributes","resourceAttributesExpression":"ResourceAttributes","defaultTableSelectExpression":"Timestamp,ServiceName,StatusCode,round(Duration/1e6),SpanName","traceIdExpression":"TraceId","spanIdExpression":"SpanId","durationExpression":"Duration","durationPrecision":9,"parentSpanIdExpression":"ParentSpanId","spanNameExpression":"SpanName","spanKindExpression":"SpanKind","statusCodeExpression":"StatusCode","statusMessageExpression":"StatusMessage","metadataMaterializedViews":{"keyRollupTable":"otel_traces_key_rollup_15m","kvRollupTable":"otel_traces_kv_rollup_15m","granularity":"15 minute"},"connection":"Local ClickHouse","logSourceId":"Logs","sessionSourceId":"Sessions","metricSourceId":"Metrics"},{"from":{"databaseName":"default","tableName":""},"kind":"metric","timestampValueExpression":"TimeUnix","name":"Metrics","resourceAttributesExpression":"ResourceAttributes","metricTables":{"gauge":"otel_metrics_gauge","histogram":"otel_metrics_histogram","sum":"otel_metrics_sum","_id":"682586a8b1f81924e628e808","id":"682586a8b1f81924e628e808"},"connection":"Local ClickHouse","logSourceId":"Logs","traceSourceId":"Traces","sessionSourceId":"Sessions"},{"from":{"databaseName":"default","tableName":"hyperdx_sessions"},"kind":"session","timestampValueExpression":"TimestampTime","name":"Sessions","displayedTimestampValueExpression":"Timestamp","implicitColumnExpression":"Body","serviceNameExpression":"ServiceName","bodyExpression":"Body","eventAttributesExpression":"LogAttributes","resourceAttributesExpression":"ResourceAttributes","defaultTableSelectExpression":"Timestamp,ServiceName,SeverityText,Body","severityTextExpression":"SeverityText","traceIdExpression":"TraceId","spanIdExpression":"SpanId","connection":"Local ClickHouse","logSourceId":"Logs","traceSourceId":"Traces","metricSourceId":"Metrics"},{"from":{"databaseName":"otel_json","tableName":"otel_logs"},"kind":"log","timestampValueExpression":"Timestamp","name":"JSON Logs","displayedTimestampValueExpression":"Timestamp","implicitColumnExpression":"Body","serviceNameExpression":"ServiceName","bodyExpression":"Body","eventAttributesExpression":"LogAttributes","resourceAttributesExpression":"ResourceAttributes","defaultTableSelectExpression":"Timestamp,ServiceName,SeverityText,Body","severityTextExpression":"SeverityText","traceIdExpression":"TraceId","spanIdExpression":"SpanId","connection":"Local ClickHouse","traceSourceId":"JSON Traces","metricSourceId":"JSON Metrics"},{"from":{"databaseName":"otel_json","tableName":"otel_traces"},"kind":"trace","timestampValueExpression":"Timestamp","name":"JSON Traces","displayedTimestampValueExpression":"Timestamp","implicitColumnExpression":"SpanName","serviceNameExpression":"ServiceName","eventAttributesExpression":"SpanAttributes","resourceAttributesExpression":"ResourceAttributes","defaultTableSelectExpression":"Timestamp,ServiceName,StatusCode,round(Duration/1e6),SpanName","traceIdExpression":"TraceId","spanIdExpression":"SpanId","durationExpression":"Duration","durationPrecision":9,"parentSpanIdExpression":"ParentSpanId","spanNameExpression":"SpanName","spanKindExpression":"SpanKind","statusCodeExpression":"StatusCode","statusMessageExpression":"StatusMessage","connection":"Local ClickHouse","logSourceId":"JSON Logs","metricSourceId":"JSON Metrics"},{"from":{"databaseName":"otel_json","tableName":""},"kind":"metric","timestampValueExpression":"TimeUnix","name":"JSON Metrics","resourceAttributesExpression":"ResourceAttributes","metricTables":{"gauge":"otel_metrics_gauge","histogram":"otel_metrics_histogram","sum":"otel_metrics_sum"},"connection":"Local ClickHouse","logSourceId":"JSON Logs","traceSourceId":"JSON Traces"}] +DEFAULT_CONNECTIONS=[{"name":"Local ClickHouse","host":"http://localhost:${HDX_DEV_CH_HTTP_PORT}","username":"default","password":""},{"name":"Local Prometheus","host":"http://localhost:${HDX_DEV_PROMETHEUS_PORT}","username":"","password":"","isPrometheusEndpoint":true}] +DEFAULT_SOURCES=[{"from":{"databaseName":"default","tableName":"otel_logs"},"kind":"log","timestampValueExpression":"Timestamp","name":"Logs","displayedTimestampValueExpression":"Timestamp","implicitColumnExpression":"Body","serviceNameExpression":"ServiceName","bodyExpression":"Body","eventAttributesExpression":"LogAttributes","resourceAttributesExpression":"ResourceAttributes","defaultTableSelectExpression":"Timestamp,ServiceName,SeverityText,Body","severityTextExpression":"SeverityText","traceIdExpression":"TraceId","spanIdExpression":"SpanId","metadataMaterializedViews":{"keyRollupTable":"otel_logs_key_rollup_15m","kvRollupTable":"otel_logs_kv_rollup_15m","granularity":"15 minute"},"connection":"Local ClickHouse","traceSourceId":"Traces","sessionSourceId":"Sessions","metricSourceId":"Metrics"},{"from":{"databaseName":"default","tableName":"otel_traces"},"kind":"trace","timestampValueExpression":"Timestamp","name":"Traces","displayedTimestampValueExpression":"Timestamp","implicitColumnExpression":"SpanName","serviceNameExpression":"ServiceName","eventAttributesExpression":"SpanAttributes","resourceAttributesExpression":"ResourceAttributes","defaultTableSelectExpression":"Timestamp,ServiceName,StatusCode,round(Duration/1e6),SpanName","traceIdExpression":"TraceId","spanIdExpression":"SpanId","durationExpression":"Duration","durationPrecision":9,"parentSpanIdExpression":"ParentSpanId","spanNameExpression":"SpanName","spanKindExpression":"SpanKind","statusCodeExpression":"StatusCode","statusMessageExpression":"StatusMessage","metadataMaterializedViews":{"keyRollupTable":"otel_traces_key_rollup_15m","kvRollupTable":"otel_traces_kv_rollup_15m","granularity":"15 minute"},"connection":"Local ClickHouse","logSourceId":"Logs","sessionSourceId":"Sessions","metricSourceId":"Metrics"},{"from":{"databaseName":"default","tableName":""},"kind":"metric","timestampValueExpression":"TimeUnix","name":"Metrics","resourceAttributesExpression":"ResourceAttributes","metricTables":{"gauge":"otel_metrics_gauge","histogram":"otel_metrics_histogram","sum":"otel_metrics_sum","_id":"682586a8b1f81924e628e808","id":"682586a8b1f81924e628e808"},"connection":"Local ClickHouse","logSourceId":"Logs","traceSourceId":"Traces","sessionSourceId":"Sessions"},{"from":{"databaseName":"default","tableName":"hyperdx_sessions"},"kind":"session","timestampValueExpression":"TimestampTime","name":"Sessions","displayedTimestampValueExpression":"Timestamp","implicitColumnExpression":"Body","serviceNameExpression":"ServiceName","bodyExpression":"Body","eventAttributesExpression":"LogAttributes","resourceAttributesExpression":"ResourceAttributes","defaultTableSelectExpression":"Timestamp,ServiceName,SeverityText,Body","severityTextExpression":"SeverityText","traceIdExpression":"TraceId","spanIdExpression":"SpanId","connection":"Local ClickHouse","logSourceId":"Logs","traceSourceId":"Traces","metricSourceId":"Metrics"},{"from":{"databaseName":"otel_json","tableName":"otel_logs"},"kind":"log","timestampValueExpression":"Timestamp","name":"JSON Logs","displayedTimestampValueExpression":"Timestamp","implicitColumnExpression":"Body","serviceNameExpression":"ServiceName","bodyExpression":"Body","eventAttributesExpression":"LogAttributes","resourceAttributesExpression":"ResourceAttributes","defaultTableSelectExpression":"Timestamp,ServiceName,SeverityText,Body","severityTextExpression":"SeverityText","traceIdExpression":"TraceId","spanIdExpression":"SpanId","connection":"Local ClickHouse","traceSourceId":"JSON Traces","metricSourceId":"JSON Metrics"},{"from":{"databaseName":"otel_json","tableName":"otel_traces"},"kind":"trace","timestampValueExpression":"Timestamp","name":"JSON Traces","displayedTimestampValueExpression":"Timestamp","implicitColumnExpression":"SpanName","serviceNameExpression":"ServiceName","eventAttributesExpression":"SpanAttributes","resourceAttributesExpression":"ResourceAttributes","defaultTableSelectExpression":"Timestamp,ServiceName,StatusCode,round(Duration/1e6),SpanName","traceIdExpression":"TraceId","spanIdExpression":"SpanId","durationExpression":"Duration","durationPrecision":9,"parentSpanIdExpression":"ParentSpanId","spanNameExpression":"SpanName","spanKindExpression":"SpanKind","statusCodeExpression":"StatusCode","statusMessageExpression":"StatusMessage","connection":"Local ClickHouse","logSourceId":"JSON Logs","metricSourceId":"JSON Metrics"},{"from":{"databaseName":"otel_json","tableName":""},"kind":"metric","timestampValueExpression":"TimeUnix","name":"JSON Metrics","resourceAttributesExpression":"ResourceAttributes","metricTables":{"gauge":"otel_metrics_gauge","histogram":"otel_metrics_histogram","sum":"otel_metrics_sum"},"connection":"Local ClickHouse","logSourceId":"JSON Logs","traceSourceId":"JSON Traces"},{"from":{"databaseName":"prometheus","tableName":"prometheus"},"kind":"promql","timestampValueExpression":"timestamp","name":"Prometheus","connection":"Local Prometheus","traceSourceId":"Traces"}] INGESTION_API_KEY="super-secure-ingestion-api-key" HYPERDX_API_KEY=$INGESTION_API_KEY ANTHROPIC_API_KEY="your-anthropic-api-key-here" ENABLE_PROMQL=true +ENABLE_SPAN_METRICS=true +ENABLE_SPAN_METRICS_PROM_RW=true diff --git a/packages/api/src/config.ts b/packages/api/src/config.ts index 7c6a242969..28da4e564b 100644 --- a/packages/api/src/config.ts +++ b/packages/api/src/config.ts @@ -57,6 +57,21 @@ export const DEFAULT_SOURCES = env.DEFAULT_SOURCES; export const IS_PROMQL_ENABLED = env.ENABLE_PROMQL === 'true'; +// Opt-in: have the collector derive request metrics (with trace exemplars) from +// spans via the spanmetrics connector. Off by default; enabled in dev so the +// telemetry-generator's traces produce coherent metric exemplars end-to-end. +export const IS_SPAN_METRICS_ENABLED = env.ENABLE_SPAN_METRICS === 'true'; + +// Opt-in: also remote-write the span-derived metrics (with exemplars) to a +// Prometheus endpoint so the native Prometheus query_exemplars path can be +// tested against real data. The endpoint is resolved here (API side) and +// inlined into the generated collector config, so the collector container does +// not need SPAN_METRICS_PROM_RW_ENDPOINT in its own environment. Requires the +// endpoint to be set; without it the feature stays disabled. +export const SPAN_METRICS_PROM_RW_ENDPOINT = env.SPAN_METRICS_PROM_RW_ENDPOINT; +export const IS_SPAN_METRICS_PROM_RW_ENABLED = + env.ENABLE_SPAN_METRICS_PROM_RW === 'true' && !!SPAN_METRICS_PROM_RW_ENDPOINT; + // FOR CI ONLY export const CLICKHOUSE_HOST = env.CLICKHOUSE_HOST as string; export const CLICKHOUSE_USER = env.CLICKHOUSE_USER as string; diff --git a/packages/api/src/models/team.ts b/packages/api/src/models/team.ts index 4d6b620162..259c695f2c 100644 --- a/packages/api/src/models/team.ts +++ b/packages/api/src/models/team.ts @@ -42,6 +42,7 @@ export default mongoose.model( fieldMetadataDisabled: Boolean, parallelizeWhenPossible: Boolean, filterKeysFetchLimit: Number, + maxExemplars: Number, }, { timestamps: true, diff --git a/packages/api/src/opamp/controllers/opampController.ts b/packages/api/src/opamp/controllers/opampController.ts index 6dfb5f989c..fdd5a21b27 100644 --- a/packages/api/src/opamp/controllers/opampController.ts +++ b/packages/api/src/opamp/controllers/opampController.ts @@ -105,6 +105,13 @@ type CollectorConfig = { pipelines: string[]; }>; }; + span_metrics?: { + histogram: { unit: string; explicit: { buckets: string[] } }; + dimensions: Array<{ name: string }>; + exemplars: { enabled: boolean }; + metrics_flush_interval: string; + namespace?: string; + }; }; exporters?: { nop?: null; @@ -155,6 +162,15 @@ type CollectorConfig = { enabled: boolean; }; }; + 'prometheusremotewrite/spanmetrics'?: { + endpoint: string; + tls: { + insecure: boolean; + }; + resource_to_telemetry_conversion: { + enabled: boolean; + }; + }; }; service: { extensions: string[]; @@ -345,6 +361,69 @@ export const buildOtelCollectorConfig = ( }; } + if ( + config.IS_SPAN_METRICS_ENABLED && + otelCollectorConfig.connectors && + otelCollectorConfig.exporters + ) { + // Derive request metrics (with trace exemplars) from spans. The connector + // consumes the traces pipeline and feeds a dedicated metrics pipeline, so + // the resulting `traces.span.metrics.*` land in ClickHouse with + // `Exemplars.*` pointing back at the spans they were measured from. + otelCollectorConfig.connectors.span_metrics = { + histogram: { + unit: 'ms', + explicit: { + buckets: [ + '2ms', + '5ms', + '10ms', + '25ms', + '50ms', + '100ms', + '250ms', + '500ms', + '1s', + '2.5s', + '5s', + '10s', + ], + }, + }, + dimensions: [ + { name: 'http.route' }, + { name: 'http.method' }, + { name: 'host.region' }, + { name: 'app.tenant_id' }, + { name: 'http.status_code' }, + ], + exemplars: { enabled: true }, + metrics_flush_interval: '15s', + }; + otelCollectorConfig.service.pipelines.traces.exporters.push( + 'span_metrics', + ); + + const spanMetricsExporters = ['clickhouse']; + // Optionally also remote-write the derived metrics (with exemplars) to a + // Prometheus endpoint, so the native Prometheus `query_exemplars` path can + // be exercised against the same real, generated data. + if (config.IS_SPAN_METRICS_PROM_RW_ENABLED) { + otelCollectorConfig.exporters['prometheusremotewrite/spanmetrics'] = { + // Guaranteed set by IS_SPAN_METRICS_PROM_RW_ENABLED above. + endpoint: config.SPAN_METRICS_PROM_RW_ENDPOINT!, + tls: { insecure: true }, + resource_to_telemetry_conversion: { enabled: true }, + }; + spanMetricsExporters.push('prometheusremotewrite/spanmetrics'); + } + otelCollectorConfig.service.pipelines['metrics/spanmetrics'] = { + receivers: ['span_metrics'], + processors: ['memory_limiter', 'batch'], + exporters: spanMetricsExporters, + }; + } + if (collectorAuthenticationEnforced) { if (otelCollectorConfig.receivers['otlp/hyperdx'] == null) { // should never happen diff --git a/packages/api/src/routers/api/__tests__/prometheus.int.test.ts b/packages/api/src/routers/api/__tests__/prometheus.int.test.ts index dc48dd8499..10f8313f0a 100644 --- a/packages/api/src/routers/api/__tests__/prometheus.int.test.ts +++ b/packages/api/src/routers/api/__tests__/prometheus.int.test.ts @@ -270,4 +270,95 @@ describe('prometheus router', () => { expect(calledUrl).toContain('/api/v1/label/__name__/values'); }); }); + + describe('GET /v1/prometheus/query_exemplars', () => { + it('returns 400 when query parameter is missing', async () => { + const { agent } = await getLoggedInAgent(server); + const res = await agent + .get('/v1/prometheus/query_exemplars') + .query({ connectionId: new Types.ObjectId().toString() }) + .expect(400); + expect(res.body).toMatchObject({ + status: 'error', + errorType: 'bad_data', + error: expect.stringContaining('query'), + }); + }); + + it('returns 400 when connectionId is missing', async () => { + const { agent } = await getLoggedInAgent(server); + const res = await agent + .get('/v1/prometheus/query_exemplars') + .query({ query: 'up' }) + .expect(400); + expect(res.body).toMatchObject({ + status: 'error', + errorType: 'bad_data', + error: expect.stringContaining('connectionId'), + }); + expect(mockFetch).not.toHaveBeenCalled(); + }); + + it('returns 404 for a connection owned by another team', async () => { + const { agent } = await getLoggedInAgent(server); + // A real, resolvable connection — just not this team's. The 404 must come + // from the team scoping, not from the id simply not existing. + const otherTeamConn = await seedPrometheusConnection( + new Types.ObjectId(), + ); + + const res = await agent + .get('/v1/prometheus/query_exemplars') + .query({ query: 'up', connectionId: otherTeamConn._id.toString() }) + .expect(404); + expect(res.body).toMatchObject({ + status: 'error', + error: 'Connection not found', + }); + expect(mockFetch).not.toHaveBeenCalled(); + }); + + it('proxies to upstream Prometheus when connection isPrometheusEndpoint', async () => { + const { agent, team } = await getLoggedInAgent(server); + const conn = await seedPrometheusConnection(team._id); + + const promResponse = { status: 'success', data: [] }; + mockFetch.mockResolvedValueOnce( + fakeUpstreamResponse(promResponse) as any, + ); + + const res = await agent + .get('/v1/prometheus/query_exemplars') + .query({ + query: 'up', + start: '1700000000', + end: '1700000060', + connectionId: conn._id.toString(), + }) + .expect(200); + + expect(res.body).toEqual(promResponse); + const calledUrl = mockFetch.mock.calls[0][0] as string; + expect(calledUrl).toContain('/api/v1/query_exemplars'); + expect(calledUrl).toContain('query=up'); + }); + + it('returns an empty result for ClickHouse-backed connections (no native exemplar table function)', async () => { + const { agent, team } = await getLoggedInAgent(server); + const conn = await seedClickHouseConnection(team._id); + + const res = await agent + .get('/v1/prometheus/query_exemplars') + .query({ + query: 'up', + start: '1700000000', + end: '1700000060', + connectionId: conn._id.toString(), + }) + .expect(200); + + expect(res.body).toEqual({ status: 'success', data: [] }); + expect(mockFetch).not.toHaveBeenCalled(); + }); + }); }); diff --git a/packages/api/src/routers/api/prometheus.ts b/packages/api/src/routers/api/prometheus.ts index 07f446816f..ad35357206 100644 --- a/packages/api/src/routers/api/prometheus.ts +++ b/packages/api/src/routers/api/prometheus.ts @@ -478,6 +478,86 @@ const queryHandler: express.RequestHandler = async (req, res) => { router.get('/query', queryHandler); router.post('/query', queryHandler); +// -------------------------- +// GET|POST /query_exemplars +// -------------------------- + +// Native Prometheus exposes exemplars via /api/v1/query_exemplars. We proxy +// straight through for Prometheus-backed connections. ClickHouse-backed metric +// exemplars are read directly from the OTel metric tables' `Exemplars.*` +// columns in the app (via renderMetricExemplarsChartConfig), so there is no +// ClickHouse table function to call here — return an empty result instead. +const queryExemplarsHandler: express.RequestHandler = async (req, res) => { + const startedAt = performance.now(); + let backend: PrometheusBackend = 'unknown'; + try { + const { teamId } = getNonNullUserWithTeam(req); + const params = getParams(req); + + const query = params.query; + if (!query) { + return res.status(400).json({ + status: 'error', + errorType: 'bad_data', + error: 'missing required parameter: query', + }); + } + + const connectionId = params.connectionId; + if (!connectionId) { + return res.status(400).json({ + status: 'error', + errorType: 'bad_data', + error: 'missing required parameter: connectionId', + }); + } + + const connection = await getConnectionById( + teamId.toString(), + connectionId, + true, + ); + if (!connection) { + return res.status(404).json({ + status: 'error', + errorType: 'bad_data', + error: 'Connection not found', + }); + } + + if (connection.isPrometheusEndpoint) { + backend = 'prometheus'; + await proxyToPrometheus( + connection.host, + '/api/v1/query_exemplars', + params, + res, + ); + return; + } + + // ClickHouse-backed PromQL: no native exemplar table function. Exemplars + // for structured metric charts are fetched app-side from the metric table. + backend = 'clickhouse'; + return res.json({ status: 'success', data: [] }); + } catch (e) { + prometheusQueryErrors.add(1, { endpoint: 'query_exemplars', backend }); + logger.error(e, 'Prometheus query_exemplars error'); + return res.status(400).json({ + status: 'error', + errorType: 'bad_data', + error: e instanceof Error ? e.message : String(e), + }); + } finally { + prometheusQueryDuration.record(performance.now() - startedAt, { + endpoint: 'query_exemplars', + backend, + }); + } +}; +router.get('/query_exemplars', queryExemplarsHandler); +router.post('/query_exemplars', queryExemplarsHandler); + // -------------------------- // GET /label/:name/values // -------------------------- diff --git a/packages/app/.env.development b/packages/app/.env.development index 7fd59cdfde..6c2de5a184 100644 --- a/packages/app/.env.development +++ b/packages/app/.env.development @@ -13,3 +13,4 @@ NEXT_PUBLIC_OTEL_EXPORTER_OTLP_ENDPOINT="http://localhost:${HDX_DEV_OTEL_HTTP_PO # NEXT_PUBLIC_OTEL_EXPORTER_OTLP_METRICS_ENDPOINT= # NEXT_PUBLIC_OTEL_EXPORTER_OTLP_LOGS_ENDPOINT= NEXT_PUBLIC_ENABLE_PROMQL=true +NEXT_PUBLIC_ENABLE_EXEMPLARS=true diff --git a/packages/app/public/whats-new.json b/packages/app/public/whats-new.json new file mode 100644 index 0000000000..05ad0d0f11 --- /dev/null +++ b/packages/app/public/whats-new.json @@ -0,0 +1,189 @@ +{ + "releases": [ + { + "version": "2.31.0", + "features": [ + { + "scope": "general", + "text": "Convert current builder config to SQL during editor switch" + }, + { + "scope": "dashboards", + "text": "Add background area sparklines to the Browser RUM dashboard number tiles" + }, + { + "scope": "dashboards", + "text": "Revamp the out-of-the-box Browser RUM dashboard" + }, + { + "scope": "charts", + "text": "Add per-series actions to the chart drill-down menu" + }, + { + "scope": "charts", + "text": "Clicking a time-chart point now locks the tooltip in place instead of opening a separate drill-down menu" + }, + { + "scope": "app", + "text": "Make the selected source clearer in the source picker" + } + ], + "highlight": { + "title": "Interactive charts & a revamped Browser RUM dashboard", + "blurb": "Click a time-chart point to **pin its tooltip** and drill in per series, and explore the out-of-the-box **Browser RUM dashboard** — now reorganised into Core Web Vitals, Load Time, Traffic, and Errors with colour-coded tiles." + } + }, + { + "version": "2.30.1", + "features": [ + { + "scope": "general", + "text": "Allow freeform text in VirtualMultiSelect" + } + ] + }, + { + "version": "2.30.0", + "features": [ + { + "scope": "general", + "text": "Add categorical bar chart display type" + }, + { + "scope": "general", + "text": "Add event patterns as a first-class dashboard tile type" + }, + { + "scope": "dashboards", + "text": "Overlay alert firing/recovery markers on tile charts" + }, + { + "scope": "general", + "text": "Add a custom ORDER BY input for Bar and Pie charts" + }, + { + "scope": "general", + "text": "Allow specifying a limit on pie and bar chart series" + }, + { "scope": "trace", "text": "Add a trace minimap above the waterfall" }, + { + "scope": "general", + "text": "Redesign the trace waterfall — per-service span colors, vertical service color bar, child counts, duration outside the bar with the span body on hover, expand/collapse depth controls" + } + ] + }, + { + "version": "2.29.0", + "features": [ + { "scope": "general", "text": "Add Browser RUM dashboard template" }, + { + "scope": "general", + "text": "Lazy-load dashboard tiles based on viewport visibility" + }, + { + "scope": "dashboards", + "text": "Add a background trend sparkline to number tiles" + }, + { + "scope": "general", + "text": "Add an \"external link\" row-click action for dashboard table tiles" + }, + { "scope": "general", "text": "Trace panel inline split detail" }, + { + "scope": "general", + "text": "Add option to fit time chart y-axis lower bound" + }, + { + "scope": "general", + "text": "Add a \"Connect your AI assistant\" section to Team Settings" + }, + { + "scope": "dashboard", + "text": "Auto-resize font in number tiles to fit container" + }, + { + "scope": "chart-explorer", + "text": "Duplicate a series in the chart builder" + }, + { + "scope": "search", + "text": "Make active filter pills editable in place" + }, + { "scope": "general", "text": "Use optimistic updates for favorites" }, + { + "scope": "general", + "text": "Allow selecting the column or SQL expression used for event pattern grouping (with shareable URL state)" + }, + { + "scope": "charts", + "text": "The time-chart series limit is now configured per chart in the Display Settings drawer instead of as a workspace-wide team setting (the team \"Time Chart Series Limit\" setting is removed)" + }, + { + "scope": "mcp", + "text": "Add denoise option to clickstack_search tool" + }, + { + "scope": "general", + "text": "Auto-fill metric table dropdowns when creating a Metrics source" + }, + { "scope": "general", "text": "Add source field suggestions" }, + { + "scope": "service-map", + "text": "Server-side filtering, latency percentiles, throughput & focus" + }, + { "scope": "source-picker", "text": "Chip + kebab menu UX" }, + { + "scope": "general", + "text": "Add an optional Section field to data sources" + }, + { + "scope": "general", + "text": "Add Known Columns List setting for distributed tables" + }, + { + "scope": "charts", + "text": "Cap group-by time charts to a top-N series limit to prevent browser memory exhaustion on high-cardinality group-bys" + }, + { + "scope": "general", + "text": "Emphasize the series nearest the cursor in multi-series time charts" + } + ] + }, + { + "version": "2.28.0", + "features": [ + { "scope": "general", "text": "Experimental promql support" }, + { + "scope": "general", + "text": "Emit Lucene conditions from sidebar/dashboard filters to enable KV items direct_read optimization on Map columns" + }, + { "scope": "general", "text": "Upgrade to clickhouse-server 26.5" }, + { + "scope": "dashboard", + "text": "Add Table of Contents right rail with bulk collapse/expand" + }, + { + "scope": "general", + "text": "Default the direct_read map column optimization on supported ClickHouse versions" + }, + { + "scope": "general", + "text": "Preserve compatible filters when switching sources" + }, + { + "scope": "general", + "text": "Add source scoping to dashboard filters" + }, + { + "scope": "general", + "text": "Add option for force-enabling/disabling text index support" + }, + { + "scope": "general", + "text": "Filters reflect all values, not search aware; filters use metadata MVs if available" + } + ] + } + ] +} diff --git a/packages/app/src/HDXMultiSeriesTimeChart.tsx b/packages/app/src/HDXMultiSeriesTimeChart.tsx index b71b49a143..d95533e34d 100644 --- a/packages/app/src/HDXMultiSeriesTimeChart.tsx +++ b/packages/app/src/HDXMultiSeriesTimeChart.tsx @@ -19,6 +19,7 @@ import { CartesianGrid, Legend, ReferenceArea, + ReferenceDot, ReferenceLine, ResponsiveContainer, Tooltip, @@ -27,7 +28,7 @@ import { } from 'recharts'; import { AxisDomain } from 'recharts/types/util/types'; import { convertGranularityToSeconds } from '@hyperdx/common-utils/dist/core/utils'; -import { DisplayType } from '@hyperdx/common-utils/dist/types'; +import { DisplayType, Exemplar } from '@hyperdx/common-utils/dist/types'; import { Button, Popover, Tooltip as MantineTooltip } from '@mantine/core'; import { IconZoomReset } from '@tabler/icons-react'; @@ -45,6 +46,12 @@ import { toViewportPoint, useChartTooltipZIndex, } from './components/charts/ChartTooltip'; +import { + clampExemplarY, + computeExemplarPoints, + computeExemplarYBounds, + ExemplarDot, +} from './components/Exemplars'; import { useChartSyncId } from './chartSync'; import { findNearestSeriesKey, @@ -642,6 +649,10 @@ export const MemoChart = memo(function MemoChart({ granularity, dateRangeEndInclusive = true, fitYAxisToData = false, + exemplars, + maxExemplars = 12, + onExemplarHover, + onExemplarHoverEnd, }: { graphResults: any[]; setIsClickActive: (v: ActiveClickPayload | undefined) => void; @@ -675,9 +686,17 @@ export const MemoChart = memo(function MemoChart({ * (with padding) instead of zero. **/ fitYAxisToData?: boolean; + /** Exemplar markers to overlay on the chart (linked to traces). */ + exemplars?: Exemplar[]; + /** Target number of exemplar markers to show (0 = unlimited). */ + maxExemplars?: number; + /** Invoked when the cursor enters an exemplar marker, with its pixel coords. */ + onExemplarHover?: (exemplar: Exemplar, cx: number, cy: number) => void; + /** Invoked when the cursor leaves an exemplar marker. */ + onExemplarHoverEnd?: () => void; }) { - const _id = useId(); - const id = _id.replace(/:/g, ''); + const rawId = useId(); + const id = rawId.replace(/:/g, ''); // recharts sync group, scoped via context (see chartSync). const syncId = useChartSyncId(); @@ -778,6 +797,28 @@ export const MemoChart = memo(function MemoChart({ captureActivePointY, ]); + // Max value across the visible series. Used as the exemplar clamp's upper + // bound when the y-axis domain is 'auto', so a single slow-trace outlier (which + // can be 100x the p99 line) can't stretch the axis and crush the series flat — + // the marker pins to the top of the series range while its hover card still + // shows the true duration. See computeExemplarYBounds. + const visibleSeriesMax = useMemo(() => { + const hasSelection = selectedSeriesNames && selectedSeriesNames.size > 0; + let max = -Infinity; + graphResults.forEach(dataPoint => { + lineData.forEach(ld => { + const seriesName = ld.displayName || ld.dataKey; + if (!hasSelection || selectedSeriesNames.has(seriesName)) { + const value = dataPoint[ld.dataKey]; + if (typeof value === 'number' && !isNaN(value)) { + max = Math.max(max, value); + } + } + }); + }); + return max; + }, [graphResults, lineData, selectedSeriesNames]); + const yAxisDomain: AxisDomain = useMemo(() => { const hasSelection = selectedSeriesNames && selectedSeriesNames.size > 0; @@ -787,9 +828,9 @@ export const MemoChart = memo(function MemoChart({ const shouldFitYAxis = fitYAxisToData && displayType !== DisplayType.StackedBar; - // The data min/max is only needed to either zoom into a selection or to - // fit the lower bound to the data. When neither applies, let Recharts - // auto-calculate the upper bound while pinning the lower bound to zero. + // The domain follows the visible series only — exemplar markers are clamped + // to the series max at render, so they never need to widen the axis. When + // there's no selection or fit, let Recharts auto-scale (lower pinned to 0). if (!hasSelection && !shouldFitYAxis) { return [0, 'auto']; } @@ -837,6 +878,14 @@ export const MemoChart = memo(function MemoChart({ displayType, ]); + // Bounds an exemplar marker is clamped into before rendering, derived from the + // domain the y-axis actually renders — see computeExemplarYBounds for why an + // unclamped marker can silently vanish. + const exemplarYBounds = useMemo( + () => computeExemplarYBounds(yAxisDomain, visibleSeriesMax), + [yAxisDomain, visibleSeriesMax], + ); + const [containerWidth, setContainerWidth] = useState(0); // The chart's outer positioned container. Used to convert a pointer's @@ -932,6 +981,31 @@ export const MemoChart = memo(function MemoChart({ const [highlightEnd, setHighlightEnd] = useState(); const mouseDownPosRef = useRef(null); + // While the cursor is over an exemplar marker, the exemplar hover card owns + // the tooltip real estate — suppress the series hover tooltip so the two don't + // overlap. Wraps the parent's exemplar-hover callbacks to also track it here. + // Track the hovered marker by key (not just a boolean) so we can detect when a + // refetch/re-thinning unmounts it — React fires no mouseleave in that case, so + // the boolean would otherwise stick `true` and permanently suppress the series + // tooltip. The reset effect lives after `exemplarPoints` is computed. + const [hoveredExemplarKey, setHoveredExemplarKey] = useState( + null, + ); + const isExemplarHovered = hoveredExemplarKey != null; + const handleExemplarHoverStart = useCallback( + (exemplar: Exemplar, cx: number, cy: number) => { + setHoveredExemplarKey( + `exemplar-${exemplar.traceId}-${exemplar.timestamp}`, + ); + onExemplarHover?.(exemplar, cx, cy); + }, + [onExemplarHover], + ); + const handleExemplarHoverEnd = useCallback(() => { + setHoveredExemplarKey(null); + onExemplarHoverEnd?.(); + }, [onExemplarHoverEnd]); + // Tracks the time range that was displayed before the user brushed to zoom // in, so a "Reset zoom" control can restore it (mirrors Highcharts). It holds // the earliest pre-zoom range across consecutive zoom-ins so resetting jumps @@ -991,6 +1065,37 @@ export const MemoChart = memo(function MemoChart({ return map; }, [lineData]); + // Place each exemplar at its own value (the trace/span's actual measurement), + // never remapped onto the series line — the marker's height must match what + // the linked trace reports. Thinned to keep ~maxExemplars markers across the + // visible range: the highest-value (most notable, e.g. slowest) trace per + // window. The window is coarser than the chart granularity so the count stays + // readable even when every fine-grained bucket has an exemplar. + // maxExemplars <= 0 means "unlimited" — show every exemplar (deduped). + const exemplarPoints = useMemo( + () => + computeExemplarPoints(exemplars, { + maxExemplars, + granularity, + dateRange, + }), + [exemplars, maxExemplars, granularity, dateRange], + ); + + // If a refetch/re-thinning drops the hovered marker from the rendered set, its + // unmounts without a mouseleave. Reset the hover here (against the actual + // rendered points) so the series tooltip un-suppresses and the parent's hover + // card closes via onExemplarHoverEnd. + useEffect(() => { + if ( + hoveredExemplarKey != null && + !exemplarPoints.some(p => p.key === hoveredExemplarKey) + ) { + setHoveredExemplarKey(null); + onExemplarHoverEnd?.(); + } + }, [exemplarPoints, hoveredExemplarKey, onExemplarHoverEnd]); + const xAxisDomain: AxisDomain = useMemo(() => { let startTime = toStartOfInterval(dateRange[0], granularity); let endTime = toStartOfInterval(dateRange[1], granularity); @@ -1271,7 +1376,7 @@ export const MemoChart = memo(function MemoChart({ Hidden once a point is clicked, where the pinned tooltip takes over. Portaled to body so HDXLineChartTooltip can self-position (see its docblock) and escape the chart's bounds near an edge. */} - {isClickActive == null && ( + {isClickActive == null && !isExemplarHovered && ( ( + + } + /> + ))} {highlightStart && highlightEnd ? ( ({ }, })); +// Exemplar hooks need a QueryClientProvider (via useMetadataWithSettings) and +// are irrelevant to queryKey consistency, so stub them out. +jest.mock('@/hooks/useExemplars', () => ({ + useExemplars: () => ({ exemplars: [], isLoading: false, isError: false }), + useExemplarTraceMeta: () => ({ data: null, isLoading: false }), +})); + jest.mock('@/hooks/useMVOptimizationExplanation', () => ({ useMVOptimizationExplanation: jest.fn().mockReturnValue({ data: undefined, diff --git a/packages/app/src/api.ts b/packages/app/src/api.ts index 21facd9c05..b8fafed44a 100644 --- a/packages/app/src/api.ts +++ b/packages/app/src/api.ts @@ -538,6 +538,18 @@ type PrometheusLabelValuesResponse = { data?: string[]; error?: string; }; +// Native Prometheus /query_exemplars shape: one entry per series, each with its +// own exemplar list. `labels` carries the trace/span id (label naming varies by +// exporter, e.g. trace_id vs traceID — normalized downstream). +export type PrometheusExemplarsResult = { + seriesLabels: PrometheusMetric; + exemplars: { labels: PrometheusMetric; value: string; timestamp: number }[]; +}; +type PrometheusQueryExemplarsResponse = { + status: 'success' | 'error'; + data?: PrometheusExemplarsResult[]; + error?: string; +}; async function prometheusFetch( path: string, @@ -583,6 +595,23 @@ export const prometheusApi = { ...(params.table ? { table: params.table } : {}), }), + queryExemplars: (params: { + query: string; + start: number; + end: number; + connectionId: string; + database?: string; + table?: string; + }): Promise => + prometheusFetch('v1/prometheus/query_exemplars', { + query: params.query, + start: String(params.start), + end: String(params.end), + connectionId: params.connectionId, + ...(params.database ? { database: params.database } : {}), + ...(params.table ? { table: params.table } : {}), + }), + labelValues: (params: { label: string; connectionId: string; diff --git a/packages/app/src/components/ChartEditor/PromqlChartEditor.tsx b/packages/app/src/components/ChartEditor/PromqlChartEditor.tsx index 1b6a7b346d..e35dcd5d8e 100644 --- a/packages/app/src/components/ChartEditor/PromqlChartEditor.tsx +++ b/packages/app/src/components/ChartEditor/PromqlChartEditor.tsx @@ -1,9 +1,10 @@ import { Control, useController, useWatch } from 'react-hook-form'; import { SourceKind } from '@hyperdx/common-utils/dist/types'; -import { Box, Button, Flex, Stack, Text } from '@mantine/core'; +import { Box, Button, Flex, Stack, Switch, Text } from '@mantine/core'; import PromQLEditor from '@/components/PromQLEditor/PromQLEditor'; import { SourceSelectControlled } from '@/components/SourceSelect'; +import { IS_EXEMPLARS_ENABLED } from '@/config'; import { usePromqlMetricNames } from '@/hooks/usePromqlMetadata'; import { useSource } from '@/source'; @@ -22,6 +23,10 @@ export default function PromqlChartEditor({ control, name: 'promqlExpression', }); + const { field: exemplarsField } = useController({ + control, + name: 'enableExemplars', + }); const sourceId = useWatch({ control, name: 'source' }); const { data: source } = useSource({ id: sourceId }); @@ -57,7 +62,37 @@ export default function PromqlChartEditor({ metricNames={metricNames} /> - + + + {IS_EXEMPLARS_ENABLED && ( + <> + { + exemplarsField.onChange(exemplarsField.value !== true); + onSubmit(); + }} + /> + {exemplarsField.value === true && ( + + + Trace source + + + + )} + + )} + + + + + ); +} diff --git a/packages/app/src/components/Exemplars/__tests__/exemplarPoints.test.ts b/packages/app/src/components/Exemplars/__tests__/exemplarPoints.test.ts new file mode 100644 index 0000000000..8465d53876 --- /dev/null +++ b/packages/app/src/components/Exemplars/__tests__/exemplarPoints.test.ts @@ -0,0 +1,148 @@ +import { Exemplar } from '@hyperdx/common-utils/dist/types'; + +import { + clampExemplarY, + computeExemplarPoints, + computeExemplarYBounds, +} from '@/components/Exemplars/exemplarPoints'; + +const ex = ( + over: Partial & { timestamp: number; value: number }, +): Exemplar => ({ + traceId: `t-${over.timestamp}-${over.value}`, + ...over, +}); + +const RANGE: [Date, Date] = [ + new Date('2025-01-01T00:00:00Z'), + new Date('2025-01-01T01:00:00Z'), +]; + +describe('computeExemplarPoints', () => { + const opts = { maxExemplars: 12, granularity: '1 minute', dateRange: RANGE }; + + it('returns [] for empty/undefined', () => { + expect(computeExemplarPoints(undefined, opts)).toEqual([]); + expect(computeExemplarPoints([], opts)).toEqual([]); + }); + + it('maps timestamp (ms) to seconds on the x-axis and value to y', () => { + const [p] = computeExemplarPoints( + [ex({ timestamp: 1_700_000_000_000, value: 42 })], + opts, + ); + expect(p.x).toBe(1_700_000_000); // ms -> s + expect(p.y).toBe(42); + }); + + it('unlimited (maxExemplars <= 0): keeps all, deduped by trace id + timestamp', () => { + const points = computeExemplarPoints( + [ + ex({ traceId: 'a', timestamp: 1000, value: 1 }), + ex({ traceId: 'a', timestamp: 1000, value: 1 }), // dup + ex({ traceId: 'b', timestamp: 2000, value: 2 }), + ], + { ...opts, maxExemplars: 0 }, + ); + expect(points).toHaveLength(2); + }); + + it('skips exemplars with a non-finite value', () => { + const points = computeExemplarPoints( + [ex({ timestamp: 1000, value: NaN }), ex({ timestamp: 2000, value: 3 })], + { ...opts, maxExemplars: 0 }, + ); + expect(points).toHaveLength(1); + expect(points[0].y).toBe(3); + }); + + it('keeps only the highest-value exemplar per time bucket', () => { + // Two exemplars in the same coarse bucket (same ms) -> the max wins. + const points = computeExemplarPoints( + [ + ex({ traceId: 'low', timestamp: 1000, value: 1 }), + ex({ traceId: 'high', timestamp: 1000, value: 9 }), + ], + opts, + ); + expect(points).toHaveLength(1); + expect(points[0].y).toBe(9); + }); + + it('separates buckets by series (groupKey) so distinct series both survive', () => { + // Same time bucket, different groupKey -> both kept (one per series). + const points = computeExemplarPoints( + [ + ex({ traceId: 'a', timestamp: 1000, value: 5, groupKey: 'svc=a' }), + ex({ traceId: 'b', timestamp: 1000, value: 5, groupKey: 'svc=b' }), + ], + opts, + ); + expect(points).toHaveLength(2); + }); + + it('does not divide by zero when the range is empty (start == end)', () => { + const points = computeExemplarPoints([ex({ timestamp: 1000, value: 1 })], { + ...opts, + dateRange: [RANGE[0], RANGE[0]], + }); + expect(points).toHaveLength(1); + }); +}); + +describe('computeExemplarYBounds', () => { + it('uses both numeric domain bounds when the axis is fitted to data', () => { + expect(computeExemplarYBounds([120, 480], 400)).toEqual({ + min: 120, + max: 480, + }); + }); + + it("falls back to the visible series max for an 'auto' upper bound", () => { + expect(computeExemplarYBounds([0, 'auto'], 400)).toEqual({ + min: 0, + max: 400, + }); + }); + + it("floors at 0 for an 'auto' lower bound", () => { + expect(computeExemplarYBounds(['auto', 'auto'], 400)).toEqual({ + min: 0, + max: 400, + }); + }); + + it('tolerates a non-array domain (recharts allows a function)', () => { + expect(computeExemplarYBounds(() => [0, 1], 400)).toEqual({ + min: 0, + max: 400, + }); + }); +}); + +describe('clampExemplarY', () => { + it('pins an outlier to the top of the domain instead of overflowing it', () => { + // Without this, recharts' default ifOverflow="discard" drops the marker and + // the overlay silently loses the slowest trace. + expect(clampExemplarY(9000, { min: 0, max: 400 })).toBe(400); + }); + + it('lifts a marker below a fitted axis floor up to the floor', () => { + // The regression this guards: fitYAxisToData puts the floor above an + // individual duration when the series is a high quantile. + expect(clampExemplarY(12, { min: 120, max: 480 })).toBe(120); + }); + + it('leaves an in-domain value untouched', () => { + expect(clampExemplarY(200, { min: 120, max: 480 })).toBe(200); + }); + + it('leaves the value alone when there is no numeric series data yet', () => { + // visibleSeriesMax is -Infinity before any data arrives. + expect(clampExemplarY(200, { min: 0, max: -Infinity })).toBe(200); + }); + + it('leaves the value alone for inverted bounds', () => { + expect(clampExemplarY(200, { min: 480, max: 120 })).toBe(200); + }); +}); diff --git a/packages/app/src/components/Exemplars/exemplarPoints.ts b/packages/app/src/components/Exemplars/exemplarPoints.ts new file mode 100644 index 0000000000..8e43bb4763 --- /dev/null +++ b/packages/app/src/components/Exemplars/exemplarPoints.ts @@ -0,0 +1,118 @@ +import { convertGranularityToSeconds } from '@hyperdx/common-utils/dist/core/utils'; +import { Exemplar } from '@hyperdx/common-utils/dist/types'; + +/** A single exemplar plotted on the chart: x in chart time units, y = value. */ +type ExemplarPoint = { + x: number; + y: number; + exemplar: Exemplar; + key: string; +}; + +function finiteOrNull(v: unknown): number | null { + return typeof v === 'number' && !isNaN(v) ? v : null; +} + +/** + * Turn raw exemplars into plotted points, thinned to keep the chart legible. + * + * - `maxExemplars <= 0`: no thinning — every exemplar is a point (deduped by + * trace id + timestamp). + * - `maxExemplars > 0`: keep the single highest-value exemplar per time bucket + * per series (`groupKey`), where the bucket width is the larger of the chart + * granularity and `range / maxExemplars`. This caps marker count while + * surfacing the most notable trace in each window. + * + * Pure and side-effect free so the thinning behaviour can be unit-tested without + * a recharts render. + */ +export function computeExemplarPoints( + exemplars: Exemplar[] | undefined, + opts: { + maxExemplars: number; + granularity: string; + dateRange: [Date, Date] | Readonly<[Date, Date]>; + }, +): ExemplarPoint[] { + if (!exemplars?.length) return []; + const { maxExemplars, granularity, dateRange } = opts; + + const toPoint = (exemplar: Exemplar, value: number): ExemplarPoint => ({ + x: exemplar.timestamp / 1000, // ms -> seconds (chart x unit) + y: value, + exemplar, + key: `exemplar-${exemplar.traceId}-${exemplar.timestamp}`, + }); + + if (maxExemplars <= 0) { + const all = new Map(); + for (const exemplar of exemplars) { + const value = finiteOrNull(exemplar.value); + if (value == null) continue; + const p = toPoint(exemplar, value); + all.set(p.key, p); // dedupe identical trace+time + } + return Array.from(all.values()); + } + + const granMs = convertGranularityToSeconds(granularity) * 1000; + const rangeMs = dateRange[1].getTime() - dateRange[0].getTime(); + const bucketMs = Math.max( + granMs || 1, + rangeMs > 0 ? Math.floor(rangeMs / maxExemplars) : granMs || 1, + ); + + const bestPerBucket = new Map(); + for (const exemplar of exemplars) { + const value = finiteOrNull(exemplar.value); + if (value == null) continue; + const bucket = Math.floor(exemplar.timestamp / bucketMs); + const key = `${exemplar.groupKey ?? ''}@${bucket}`; + const existing = bestPerBucket.get(key); + if (!existing || value > existing.y) { + bestPerBucket.set(key, toPoint(exemplar, value)); + } + } + return Array.from(bestPerBucket.values()); +} + +/** The y range an exemplar marker may be drawn in. */ +export type ExemplarYBounds = { min: number; max: number }; + +/** + * Derive the marker clamp range from the y-axis domain the chart actually + * renders. A recharts `ReferenceDot` defaults to `ifOverflow="discard"`, so a + * marker outside the domain simply vanishes — which cuts both ways: an outlier + * above the series max would stretch the axis (crushing the line flat) or be + * dropped, and a `fitYAxisToData` floor can sit *above* a marker's value + * (routine when the series is a high quantile and exemplars are individual + * durations), silently emptying the overlay. + * + * `'auto'` bounds are resolved by recharts against the series data, so the + * series max is in-domain by construction and 0 is a safe floor for the + * non-negative durations exemplars are scoped to today. + */ +export function computeExemplarYBounds( + yAxisDomain: unknown, + visibleSeriesMax: number, +): ExemplarYBounds { + const [lower, upper] = Array.isArray(yAxisDomain) + ? yAxisDomain + : [undefined, undefined]; + return { + min: typeof lower === 'number' ? lower : 0, + max: typeof upper === 'number' ? upper : visibleSeriesMax, + }; +} + +/** + * Pin an exemplar's value inside `bounds` so recharts keeps drawing it. The + * hover card still reports the exemplar's true value. Degenerate bounds (no + * numeric data yet, or inverted) leave the value untouched — better an + * occasionally-discarded marker than one pinned to a meaningless height. + */ +export function clampExemplarY(y: number, bounds: ExemplarYBounds): number { + const { min, max } = bounds; + if (!Number.isFinite(max) || !Number.isFinite(min) || min > max) return y; + return Math.min(Math.max(y, min), max); +} diff --git a/packages/app/src/components/Exemplars/index.ts b/packages/app/src/components/Exemplars/index.ts new file mode 100644 index 0000000000..bc81d9a31b --- /dev/null +++ b/packages/app/src/components/Exemplars/index.ts @@ -0,0 +1,7 @@ +export { ExemplarDot } from './ExemplarDot'; +export { ExemplarHoverCard } from './ExemplarHoverCard'; +export { + clampExemplarY, + computeExemplarPoints, + computeExemplarYBounds, +} from './exemplarPoints'; diff --git a/packages/app/src/components/TeamSettings/TeamQueryConfigSection.tsx b/packages/app/src/components/TeamSettings/TeamQueryConfigSection.tsx index e685112003..cfba1f5564 100644 --- a/packages/app/src/components/TeamSettings/TeamQueryConfigSection.tsx +++ b/packages/app/src/components/TeamSettings/TeamQueryConfigSection.tsx @@ -19,8 +19,10 @@ import { IconHelpCircle, IconPencil } from '@tabler/icons-react'; import api from '@/api'; import SelectControlled from '@/components/SelectControlled'; +import { IS_EXEMPLARS_ENABLED } from '@/config'; import { DEFAULT_FILTER_KEYS_FETCH_LIMIT, + DEFAULT_MAX_EXEMPLARS, DEFAULT_QUERY_TIMEOUT, DEFAULT_SEARCH_ROW_LIMIT, } from '@/defaults'; @@ -344,6 +346,30 @@ export default function TeamQueryConfigSection() { /> + + {IS_EXEMPLARS_ENABLED && ( + <> + + Chart Settings + + + + + + + + + )} ); } diff --git a/packages/app/src/components/__tests__/DBTimeChart.test.tsx b/packages/app/src/components/__tests__/DBTimeChart.test.tsx index 1e86e1e60d..763966cd03 100644 --- a/packages/app/src/components/__tests__/DBTimeChart.test.tsx +++ b/packages/app/src/components/__tests__/DBTimeChart.test.tsx @@ -39,6 +39,15 @@ jest.mock('@/source', () => ({ .mockReturnValue({ formatByColumn: new Map(), chartFormat: undefined }), })); +jest.mock('@/hooks/useExemplars', () => ({ + useExemplars: jest + .fn() + .mockReturnValue({ exemplars: [], isLoading: false, isError: false }), + useExemplarTraceMeta: jest + .fn() + .mockReturnValue({ data: null, isLoading: false }), +})); + jest.mock('../MaterializedViews/MVOptimizationIndicator', () => jest.fn(() => null), ); diff --git a/packages/app/src/config.ts b/packages/app/src/config.ts index be36368c43..92a92a501d 100644 --- a/packages/app/src/config.ts +++ b/packages/app/src/config.ts @@ -66,3 +66,7 @@ export const IS_METRICS_ENABLED = true; export const IS_MTVIEWS_ENABLED = false; export const IS_SESSIONS_ENABLED = true; export const IS_PROMQL_ENABLED = env('NEXT_PUBLIC_ENABLE_PROMQL') === 'true'; +// Exemplar overlay (trace markers on time charts). Off by default while the +// feature is being tested; set NEXT_PUBLIC_ENABLE_EXEMPLARS=true to enable. +export const IS_EXEMPLARS_ENABLED = + env('NEXT_PUBLIC_ENABLE_EXEMPLARS') === 'true'; diff --git a/packages/app/src/defaults.ts b/packages/app/src/defaults.ts index 95d151307f..d38ed01118 100644 --- a/packages/app/src/defaults.ts +++ b/packages/app/src/defaults.ts @@ -5,6 +5,8 @@ export const DEFAULT_SEARCH_ROW_LIMIT = 200; export const DEFAULT_QUERY_TIMEOUT = 60; // max_execution_time, seconds export const DEFAULT_FILTER_KEYS_FETCH_LIMIT = 100; export const DEFAULT_SERIES_LIMIT = 100; +// Target number of exemplar markers shown per chart (0 = unlimited). +export const DEFAULT_MAX_EXEMPLARS = 12; export function searchChartConfigDefaults( team: any | undefined | null, diff --git a/packages/app/src/hooks/__tests__/useExemplars.test.tsx b/packages/app/src/hooks/__tests__/useExemplars.test.tsx new file mode 100644 index 0000000000..36033cf6e1 --- /dev/null +++ b/packages/app/src/hooks/__tests__/useExemplars.test.tsx @@ -0,0 +1,356 @@ +/* eslint-disable @typescript-eslint/no-unsafe-type-assertion */ +import React from 'react'; +import { EXEMPLAR_QUERY_LIMIT } from '@hyperdx/common-utils/dist/core/renderChartConfig'; +import { + ChartConfigWithOptDateRange, + DisplayType, + MetricsDataType, + SourceKind, + TSource, +} from '@hyperdx/common-utils/dist/types'; +import { QueryClient, QueryClientProvider } from '@tanstack/react-query'; +import { renderHook, waitFor } from '@testing-library/react'; + +import { prometheusApi } from '@/api'; +import { useClickhouseClient } from '@/clickhouse'; +import { + normalizePrometheusExemplars, + useExemplars, +} from '@/hooks/useExemplars'; + +// Flipped per-test to exercise the deployment feature gate. A getter (rather +// than a literal) so the hook reads the current value on each render. +let isExemplarsEnabled = true; +jest.mock('@/config', () => ({ + get IS_EXEMPLARS_ENABLED() { + return isExemplarsEnabled; + }, +})); + +jest.mock('@/api', () => ({ + __esModule: true, + prometheusApi: { queryExemplars: jest.fn() }, +})); + +jest.mock('@/clickhouse', () => ({ + __esModule: true, + useClickhouseClient: jest.fn(), +})); + +jest.mock('@/hooks/useMetadata', () => ({ + __esModule: true, + useMetadataWithSettings: jest.fn().mockReturnValue({ + getColumns: jest.fn().mockResolvedValue([]), + getMaterializedColumnsLookupTable: jest.fn().mockResolvedValue(null), + getColumn: jest.fn().mockResolvedValue(undefined), + getTableMetadata: jest.fn().mockResolvedValue({ primary_key: 'TimeUnix' }), + getSkipIndices: jest.fn().mockResolvedValue([]), + getSetting: jest.fn().mockResolvedValue(undefined), + isClickHouseCloud: jest.fn().mockResolvedValue(false), + }), +})); + +jest.mock('@/source', () => ({ + __esModule: true, + getDurationMsExpression: jest.fn().mockReturnValue('Duration / 1e6'), +})); + +describe('normalizePrometheusExemplars', () => { + it('returns [] for undefined/empty input', () => { + expect(normalizePrometheusExemplars(undefined)).toEqual([]); + expect(normalizePrometheusExemplars([])).toEqual([]); + }); + + it('maps trace/span ids, value, and seconds→ms timestamp', () => { + const result = normalizePrometheusExemplars([ + { + seriesLabels: { __name__: 'http_latency', service: 'api' }, + exemplars: [ + { + labels: { trace_id: 'abc', span_id: 'def' }, + value: '1.5', + timestamp: 1700000000, + }, + ], + }, + ]); + expect(result).toEqual([ + { + timestamp: 1700000000 * 1000, + value: 1.5, + traceId: 'abc', + spanId: 'def', + groupKey: 'service="api"', + }, + ]); + }); + + it('accepts alternate label spellings (traceID/spanID)', () => { + const [ex] = normalizePrometheusExemplars([ + { + seriesLabels: {}, + exemplars: [ + { + labels: { traceID: 'xyz', spanID: 's1' }, + value: '2', + timestamp: 1, + }, + ], + }, + ]); + expect(ex.traceId).toBe('xyz'); + expect(ex.spanId).toBe('s1'); + expect(ex.groupKey).toBeUndefined(); + }); + + it('drops the overlay entirely when the query returns multiple series', () => { + // Exemplars are single-series only; multi-series markers can't be attributed + // or scaled meaningfully, so the whole set is dropped rather than rendered. + const multiSeries = [ + { + seriesLabels: { service: 'api' }, + exemplars: [ + { labels: { trace_id: 'a' }, value: '1', timestamp: 1700000000 }, + ], + }, + { + seriesLabels: { service: 'web' }, + exemplars: [ + { labels: { trace_id: 'b' }, value: '2', timestamp: 1700000000 }, + ], + }, + ]; + expect(normalizePrometheusExemplars(multiSeries)).toEqual([]); + }); + + it('skips exemplars without a trace id', () => { + expect( + normalizePrometheusExemplars([ + { + seriesLabels: {}, + exemplars: [{ labels: { foo: 'bar' }, value: '1', timestamp: 1 }], + }, + ]), + ).toEqual([]); + }); +}); + +describe('useExemplars', () => { + const mockQuery = jest.fn(); + const mockQueryExemplars = prometheusApi.queryExemplars as jest.Mock; + + const wrapper = ({ children }: { children: React.ReactNode }) => ( + + {children} + + ); + + const metricSource = { kind: SourceKind.Metric } as TSource; + const promqlSource = { kind: SourceKind.Promql } as TSource; + + const histogramConfig = { + displayType: DisplayType.Line, + connection: 'test-connection', + metricTables: { + gauge: 'otel_metrics_gauge', + histogram: 'otel_metrics_histogram', + sum: 'otel_metrics_sum', + summary: 'otel_metrics_summary', + 'exponential histogram': 'otel_metrics_exponential_histogram', + }, + from: { databaseName: 'default', tableName: '' }, + select: [ + { + aggFn: 'quantile', + aggCondition: '', + aggConditionLanguage: 'lucene', + valueExpression: 'Value', + level: 0.95, + metricName: 'http.server.duration', + metricType: MetricsDataType.Histogram, + }, + ], + where: '', + whereLanguage: 'lucene', + timestampValueExpression: 'TimeUnix', + dateRange: [new Date('2025-02-12'), new Date('2025-02-14')], + granularity: '1 minute', + enableExemplars: true, + } as ChartConfigWithOptDateRange; + + const promqlConfig = { + configType: 'promql', + displayType: DisplayType.Line, + connection: 'test-connection', + promqlExpression: 'histogram_quantile(0.95, http_latency)', + from: { databaseName: 'default', tableName: 'metrics' }, + select: '', + where: '', + dateRange: [new Date('2025-02-12'), new Date('2025-02-14')], + granularity: '1 minute', + enableExemplars: true, + } as unknown as ChartConfigWithOptDateRange; + + beforeAll(() => { + // The stubbed metadata can't resolve column types, which the SQL renderer + // warns about — expected here, and noisy. + jest.spyOn(console, 'warn').mockImplementation(jest.fn()); + jest.spyOn(console, 'error').mockImplementation(jest.fn()); + }); + afterAll(() => { + jest.restoreAllMocks(); + }); + + beforeEach(() => { + isExemplarsEnabled = true; + jest.clearAllMocks(); + (useClickhouseClient as jest.Mock).mockReturnValue({ query: mockQuery }); + }); + + describe('gating', () => { + it('does not fetch when the chart has not opted in', async () => { + const { result } = renderHook( + () => + useExemplars( + { ...histogramConfig, enableExemplars: undefined }, + metricSource, + ), + { wrapper }, + ); + expect(result.current.exemplars).toEqual([]); + expect(mockQuery).not.toHaveBeenCalled(); + }); + + it('does not fetch while the deployment feature flag is off', async () => { + isExemplarsEnabled = false; + const { result } = renderHook( + () => useExemplars(histogramConfig, metricSource), + { wrapper }, + ); + expect(result.current.exemplars).toEqual([]); + expect(mockQuery).not.toHaveBeenCalled(); + }); + + it('does not fetch for source kinds that cannot produce exemplars', async () => { + const { result } = renderHook( + () => + useExemplars(histogramConfig, { kind: SourceKind.Log } as TSource), + { wrapper }, + ); + expect(result.current.exemplars).toEqual([]); + expect(mockQuery).not.toHaveBeenCalled(); + }); + + it('does not fetch without a source', async () => { + const { result } = renderHook( + () => useExemplars(histogramConfig, undefined), + { wrapper }, + ); + expect(result.current.exemplars).toEqual([]); + expect(mockQuery).not.toHaveBeenCalled(); + }); + }); + + describe('metric source (ClickHouse)', () => { + it('maps exemplar rows and drops rows without a trace id', async () => { + mockQuery.mockResolvedValue({ + json: async () => ({ + data: [ + { + timestamp: '1700000000000', + value: '1.5', + traceId: 'a', + spanId: 's1', + }, + // No trace id → nothing to link to, must be dropped. + { timestamp: '1700000001000', value: '2.5', traceId: '' }, + { timestamp: '1700000002000', value: '3.5', traceId: 'b' }, + ], + }), + }); + + const { result } = renderHook( + () => useExemplars(histogramConfig, metricSource), + { wrapper }, + ); + + await waitFor(() => expect(result.current.exemplars).toHaveLength(2)); + expect(result.current.exemplars).toEqual([ + { timestamp: 1700000000000, value: 1.5, traceId: 'a', spanId: 's1' }, + { + timestamp: 1700000002000, + value: 3.5, + traceId: 'b', + spanId: undefined, + }, + ]); + }); + + it('returns [] without querying when the config is not exemplar-eligible', async () => { + // A Group By makes markers unattributable, so the renderer returns no SQL. + const { result } = renderHook( + () => + useExemplars( + { ...histogramConfig, groupBy: 'ServiceName' }, + metricSource, + ), + { wrapper }, + ); + await waitFor(() => expect(result.current.isLoading).toBe(false)); + expect(result.current.exemplars).toEqual([]); + expect(mockQuery).not.toHaveBeenCalled(); + }); + }); + + describe('promql source', () => { + it('caps the result at EXEMPLAR_QUERY_LIMIT, keeping the highest values', async () => { + const exemplars = Array.from( + { length: EXEMPLAR_QUERY_LIMIT + 50 }, + (_, i) => ({ + labels: { trace_id: `t${i}` }, + value: String(i), + timestamp: 1700000000 + i, + }), + ); + mockQueryExemplars.mockResolvedValue({ + status: 'success', + data: [{ seriesLabels: {}, exemplars }], + }); + + const { result } = renderHook( + () => useExemplars(promqlConfig, promqlSource), + { wrapper }, + ); + + await waitFor(() => + expect(result.current.exemplars).toHaveLength(EXEMPLAR_QUERY_LIMIT), + ); + // Highest value first, and the low-value tail is what got dropped. + expect(result.current.exemplars[0].value).toBe(EXEMPLAR_QUERY_LIMIT + 49); + expect(result.current.exemplars.every(ex => ex.value >= 50)).toBe(true); + expect(mockQuery).not.toHaveBeenCalled(); + }); + + it('surfaces an error when the proxy reports a non-success status', async () => { + mockQueryExemplars.mockResolvedValue({ + status: 'error', + error: 'upstream exploded', + }); + + const { result } = renderHook( + () => useExemplars(promqlConfig, promqlSource), + { wrapper }, + ); + + // The hook sets retry: 1, so allow for the retry's backoff delay. + await waitFor(() => expect(result.current.isError).toBe(true), { + timeout: 5000, + }); + expect(result.current.exemplars).toEqual([]); + }); + }); +}); diff --git a/packages/app/src/hooks/useExemplars.tsx b/packages/app/src/hooks/useExemplars.tsx new file mode 100644 index 0000000000..c4f89bcc72 --- /dev/null +++ b/packages/app/src/hooks/useExemplars.tsx @@ -0,0 +1,222 @@ +import { + EXEMPLAR_QUERY_LIMIT, + renderMetricExemplarsChartConfig, +} from '@hyperdx/common-utils/dist/core/renderChartConfig'; +import { isPromqlChartConfig } from '@hyperdx/common-utils/dist/guards'; +import { + ChartConfigWithOptDateRange, + Exemplar, + SourceKind, + TSource, +} from '@hyperdx/common-utils/dist/types'; +import { useQuery } from '@tanstack/react-query'; + +import { prometheusApi, type PrometheusExemplarsResult } from '@/api'; +import { useClickhouseClient } from '@/clickhouse'; +import { IS_EXEMPLARS_ENABLED } from '@/config'; +import { useMetadataWithSettings } from '@/hooks/useMetadata'; +import { getDurationMsExpression } from '@/source'; + +// Source kinds that can produce exemplars today: native metric/promql sources. +// Trace-generated exemplars are added in a follow-up. +const EXEMPLAR_SUPPORTED_KINDS: SourceKind[] = [ + SourceKind.Metric, + SourceKind.Promql, +]; + +// Native Prometheus exporters disagree on the trace/span id label name; accept +// the common spellings. +const TRACE_ID_LABELS = ['trace_id', 'traceID', 'traceId', 'trace.id']; +const SPAN_ID_LABELS = ['span_id', 'spanID', 'spanId', 'span.id']; + +function pick(labels: Record, keys: string[]) { + for (const k of keys) { + if (labels[k]) return labels[k]; + } + return undefined; +} + +/** + * Normalize a native Prometheus /query_exemplars response into the shared + * Exemplar shape. Exported for testing — label naming varies by exporter. + */ +export function normalizePrometheusExemplars( + data: PrometheusExemplarsResult[] | undefined, +): Exemplar[] { + if (!data) return []; + // Exemplars are a single-series feature: their y-position is the trace's own + // value on the chart's shared axis, so markers from multiple series can't be + // attributed or scaled meaningfully. If the PromQL expression fans out to more + // than one series, drop the overlay rather than render ambiguous markers. + if (data.length > 1) return []; + const out: Exemplar[] = []; + for (const series of data) { + const seriesLabels = series.seriesLabels ?? {}; + const groupKey = + Object.entries(seriesLabels) + .filter(([k]) => k !== '__name__') + .map(([k, v]) => `${k}="${v}"`) + .join(', ') || undefined; + for (const ex of series.exemplars ?? []) { + const traceId = pick(ex.labels ?? {}, TRACE_ID_LABELS); + if (!traceId) continue; + out.push({ + timestamp: ex.timestamp * 1000, // prometheus exemplar ts is unix seconds + value: Number(ex.value), + traceId, + spanId: pick(ex.labels ?? {}, SPAN_ID_LABELS), + groupKey, + }); + } + } + return out; +} + +/** Map raw ClickHouse exemplar rows (renderMetricExemplarsChartConfig) → Exemplar[]. */ +function mapClickhouseExemplars(rows: Record[]): Exemplar[] { + return rows + .filter(r => r.traceId) + .map(r => ({ + timestamp: Number(r.timestamp), + value: Number(r.value), + traceId: String(r.traceId), + spanId: r.spanId ? String(r.spanId) : undefined, + })); +} + +/** + * Fetches exemplars for a chart in parallel with the main series query. A no-op + * (disabled query) unless `config.enableExemplars` is set and the source kind + * supports exemplars, so it adds zero cost to charts that don't use the overlay. + */ +export function useExemplars( + config: ChartConfigWithOptDateRange, + source: TSource | undefined, +) { + const clickhouseClient = useClickhouseClient(); + const metadata = useMetadataWithSettings(); + + const supported = !!source && EXEMPLAR_SUPPORTED_KINDS.includes(source.kind); + // Global feature gate: even a config with enableExemplars set fetches nothing + // while the feature is disabled for the deployment. + const enabled = + IS_EXEMPLARS_ENABLED && config.enableExemplars === true && supported; + + const query = useQuery({ + queryKey: ['exemplars', config], + queryFn: async context => { + // PromQL → native Prometheus exemplars via the API proxy. + if (isPromqlChartConfig(config) && config.dateRange) { + const [startDate, endDate] = config.dateRange; + const resp = await prometheusApi.queryExemplars({ + query: config.promqlExpression, + start: startDate.getTime() / 1000, + end: endDate.getTime() / 1000, + connectionId: config.connection, + database: config.from?.databaseName, + table: config.from?.tableName, + }); + if (resp.status !== 'success') { + throw new Error(resp.error ?? 'query_exemplars failed'); + } + // Native Prometheus /query_exemplars has no result-limit parameter, so + // bound the set client-side to keep an unbounded upstream response from + // ballooning downstream thinning/render work. Mirror the ClickHouse + // path's EXEMPLAR_QUERY_LIMIT and keep the highest-value exemplars. + return normalizePrometheusExemplars(resp.data) + .sort((a, b) => b.value - a.value) + .slice(0, EXEMPLAR_QUERY_LIMIT); + } + + // Structured metric source → exemplars stored on the OTel metric table. + const exemplarSql = await renderMetricExemplarsChartConfig( + config, + metadata, + ); + if (!exemplarSql) return []; + + const resp = await clickhouseClient.query({ + query: exemplarSql.sql, + query_params: exemplarSql.params, + format: 'JSON', + abort_signal: context.signal, + connectionId: config.connection, + }); + const json = await resp.json>(); + return mapClickhouseExemplars(json.data ?? []); + }, + enabled, + retry: 1, + refetchOnWindowFocus: false, + }); + + return { + exemplars: enabled ? (query.data ?? []) : [], + isLoading: query.isLoading, + isError: query.isError, + }; +} + +export type ExemplarTraceMeta = { + service?: string; + spanName?: string; + statusCode?: string; + durationMs?: number; + timestamp?: string; +}; + +/** + * Fetches a one-row summary of a trace (root/first span) from the given trace + * source, for the exemplar hover card. Enabled only while a trace id is hovered + * and a trace source is configured. + */ +export function useExemplarTraceMeta( + traceId: string | undefined, + traceSource: TSource | undefined, +) { + const clickhouseClient = useClickhouseClient(); + const isTrace = !!traceSource && traceSource.kind === SourceKind.Trace; + + return useQuery({ + queryKey: ['exemplarTraceMeta', traceId, traceSource?.id], + enabled: !!traceId && isTrace, + staleTime: 5 * 60 * 1000, + queryFn: async context => { + if (!traceId || !traceSource || traceSource.kind !== SourceKind.Trace) { + return null; + } + const s = traceSource; + const from = s.from.databaseName + ? `\`${s.from.databaseName}\`.\`${s.from.tableName}\`` + : `\`${s.from.tableName}\``; + const traceIdExpr = s.traceIdExpression || 'TraceId'; + const parentExpr = s.parentSpanIdExpression || 'ParentSpanId'; + const tsExpr = s.timestampValueExpression || 'Timestamp'; + const sql = ` + SELECT + ${s.serviceNameExpression || 'ServiceName'} AS service, + ${s.spanNameExpression || 'SpanName'} AS spanName, + ${s.statusCodeExpression || 'StatusCode'} AS statusCode, + ${getDurationMsExpression(s)} AS durationMs, + ${tsExpr} AS timestamp + FROM ${from} + WHERE ${traceIdExpr} = {traceId:String} + ORDER BY (${parentExpr} = '') DESC, ${tsExpr} ASC + LIMIT 1`; + const resp = await clickhouseClient.query({ + query: sql, + query_params: { traceId }, + format: 'JSON', + abort_signal: context.signal, + connectionId: s.connection, + }); + const json = await resp.json(); + const row = json.data?.[0]; + if (!row) return null; + return { + ...row, + durationMs: row.durationMs != null ? Number(row.durationMs) : undefined, + }; + }, + }); +} diff --git a/packages/common-utils/src/__tests__/renderChartConfig.test.ts b/packages/common-utils/src/__tests__/renderChartConfig.test.ts index c3592f0b78..17a13518ea 100644 --- a/packages/common-utils/src/__tests__/renderChartConfig.test.ts +++ b/packages/common-utils/src/__tests__/renderChartConfig.test.ts @@ -2,7 +2,10 @@ import { chSql, ColumnMeta, parameterizedQueryToSql } from '@/clickhouse'; import { Metadata } from '@/core/metadata'; import { ChartConfigWithOptDateRangeEx, + EXEMPLAR_QUERY_LIMIT, + isExemplarEligible, renderChartConfig, + renderMetricExemplarsChartConfig, timeFilterExpr, } from '@/core/renderChartConfig'; import { @@ -3508,3 +3511,251 @@ describe('renderChartConfig', () => { // gauge / sum / histogram snapshots earlier in this file plus the // cross-scope integration test in packages/api/src/clickhouse/__tests__. }); + +describe('isExemplarEligible', () => { + // The single rule the chart-editor toggle and the SQL renderer both delegate + // to; each clause below is a case where a marker can't be attributed to a + // point on the chart's y-axis. + const eligible = { + seriesCount: 1, + seriesReturnType: 'column' as const, + metricType: MetricsDataType.Histogram, + hasGroupBy: false, + }; + + it('accepts a single, non-ratio, non-grouped histogram series', () => { + expect(isExemplarEligible(eligible)).toBe(true); + }); + + it('accepts an unset seriesReturnType (defaults to a plain column series)', () => { + expect( + isExemplarEligible({ ...eligible, seriesReturnType: undefined }), + ).toBe(true); + }); + + it('rejects multi-series and zero-series charts', () => { + expect(isExemplarEligible({ ...eligible, seriesCount: 2 })).toBe(false); + expect(isExemplarEligible({ ...eligible, seriesCount: 0 })).toBe(false); + }); + + it('rejects a ratio chart', () => { + expect(isExemplarEligible({ ...eligible, seriesReturnType: 'ratio' })).toBe( + false, + ); + }); + + it('rejects a grouped chart', () => { + expect(isExemplarEligible({ ...eligible, hasGroupBy: true })).toBe(false); + }); + + it('rejects non-histogram and missing metric types', () => { + expect( + isExemplarEligible({ ...eligible, metricType: MetricsDataType.Gauge }), + ).toBe(false); + expect(isExemplarEligible({ ...eligible, metricType: undefined })).toBe( + false, + ); + }); + + it('rejects a differently-cased metric type', () => { + // The comparison is by value, so a caller passing the app package's legacy + // MetricsDataType ('Histogram') must not silently read as eligible. + expect(isExemplarEligible({ ...eligible, metricType: 'Histogram' })).toBe( + false, + ); + }); +}); + +describe('renderMetricExemplarsChartConfig', () => { + let mockMetadata: jest.Mocked; + + beforeAll(() => { + jest.spyOn(console, 'warn').mockImplementation(() => {}); + jest.spyOn(console, 'error').mockImplementation(() => {}); + }); + afterAll(() => { + jest.restoreAllMocks(); + }); + + beforeEach(() => { + mockMetadata = { + getColumns: jest.fn().mockResolvedValue([]), + getMaterializedColumnsLookupTable: jest.fn().mockResolvedValue(null), + getColumn: jest.fn().mockResolvedValue(undefined), + getTableMetadata: jest + .fn() + .mockResolvedValue({ primary_key: 'TimeUnix' }), + getSkipIndices: jest.fn().mockResolvedValue([]), + getSetting: jest.fn().mockResolvedValue(undefined), + isClickHouseCloud: jest.fn().mockResolvedValue(false), + } as unknown as jest.Mocked; + }); + + const histogramConfig: ChartConfigWithOptDateRange = { + displayType: DisplayType.Line, + connection: 'test-connection', + metricTables: { + gauge: 'otel_metrics_gauge', + histogram: 'otel_metrics_histogram', + sum: 'otel_metrics_sum', + summary: 'otel_metrics_summary', + 'exponential histogram': 'otel_metrics_exponential_histogram', + }, + from: { databaseName: 'default', tableName: '' }, + select: [ + { + aggFn: 'quantile', + aggCondition: '', + aggConditionLanguage: 'lucene', + valueExpression: 'Value', + level: 0.95, + metricName: 'http.server.duration', + metricType: MetricsDataType.Histogram, + }, + ], + where: '', + whereLanguage: 'lucene', + timestampValueExpression: 'TimeUnix', + dateRange: [new Date('2025-02-12'), new Date('2025-02-14')], + granularity: '1 minute', + }; + + it('builds an ARRAY JOIN exemplar query against the metric-type table', async () => { + const generated = await renderMetricExemplarsChartConfig( + histogramConfig, + mockMetadata, + ); + expect(generated).not.toBeNull(); + const sql = parameterizedQueryToSql(generated!); + + // Surfaces the exemplar columns + expect(sql).toContain('ARRAY JOIN'); + expect(sql).toContain('`Exemplars.TraceId` AS ex_TraceId'); + expect(sql).toContain('toUnixTimestamp64Milli(ex_TimeUnix)'); + // Points at the histogram table and filters by metric name + time + expect(sql).toContain('otel_metrics_histogram'); + expect(sql).toContain("MetricName = 'http.server.duration'"); + expect(sql).toContain('TimeUnix'); + // Drops empty trace ids and caps the result set + expect(sql).toContain('notEmpty(ex_TraceId)'); + expect(sql).toContain(`LIMIT ${EXEMPLAR_QUERY_LIMIT}`); + }); + + it("scopes exemplars to the series' aggCondition so markers match the plotted line", async () => { + const filteredConfig = { + ...histogramConfig, + select: [ + { + aggFn: 'quantile', + level: 0.95, + valueExpression: 'Value', + metricName: 'http.server.duration', + metricType: MetricsDataType.Histogram, + aggCondition: "ServiceName = 'api'", + aggConditionLanguage: 'sql', + }, + ], + } as ChartConfigWithOptDateRange; + const generated = await renderMetricExemplarsChartConfig( + filteredConfig, + mockMetadata, + ); + expect(generated).not.toBeNull(); + const sql = parameterizedQueryToSql(generated!); + expect(sql).toContain("ServiceName = 'api'"); + }); + + it('ANDs the metric-name predicate even when the chart uses OR filters', async () => { + const orConfig = { + ...histogramConfig, + filtersLogicalOperator: 'OR', + filters: [ + { type: 'sql', condition: "ServiceName = 'api'" }, + { type: 'sql', condition: "ServiceName = 'web'" }, + ], + } as ChartConfigWithOptDateRange; + const generated = await renderMetricExemplarsChartConfig( + orConfig, + mockMetadata, + ); + expect(generated).not.toBeNull(); + const sql = parameterizedQueryToSql(generated!); + // The required metric-name check must be ANDed as its own group, never + // folded into the user OR group (which would let the scan match other + // metrics whenever a user filter matches). + expect(sql).toContain("AND (MetricName = 'http.server.duration')"); + // The user filters are still present and OR'd within their own group. + expect(sql).toContain("ServiceName = 'api'"); + expect(sql).toContain("ServiceName = 'web'"); + expect(sql).not.toMatch(/OR\s*\(?\s*MetricName/); + }); + + it('returns null for a ratio config (exemplars are meaningless on a ratio axis)', async () => { + const ratioConfig = { + ...histogramConfig, + seriesReturnType: 'ratio', + select: [histogramConfig.select[0], histogramConfig.select[0]], + } as ChartConfigWithOptDateRange; + expect( + await renderMetricExemplarsChartConfig(ratioConfig, mockMetadata), + ).toBeNull(); + }); + + it('returns null for a multi-series config', async () => { + const multiConfig = { + ...histogramConfig, + select: [histogramConfig.select[0], histogramConfig.select[0]], + } as ChartConfigWithOptDateRange; + expect( + await renderMetricExemplarsChartConfig(multiConfig, mockMetadata), + ).toBeNull(); + }); + + it('returns null when the chart has a Group By', async () => { + // A grouped chart draws one line per group, but the exemplar scan is not + // group-aware: it would pool exemplars from every group into one + // unattributable set. Drop the overlay instead. + const groupedConfig = { + ...histogramConfig, + groupBy: 'ServiceName', + } as ChartConfigWithOptDateRange; + expect( + await renderMetricExemplarsChartConfig(groupedConfig, mockMetadata), + ).toBeNull(); + }); + + it('returns null for a non-histogram metric (value is on a different scale)', async () => { + const gaugeConfig = { + ...histogramConfig, + select: [ + { + aggFn: 'avg', + aggCondition: '', + aggConditionLanguage: 'lucene', + valueExpression: 'Value', + metricName: 'system.memory.usage', + metricType: MetricsDataType.Gauge, + }, + ], + } as ChartConfigWithOptDateRange; + expect( + await renderMetricExemplarsChartConfig(gaugeConfig, mockMetadata), + ).toBeNull(); + }); + + it('returns null for a non-metric config', async () => { + const logConfig: ChartConfigWithOptDateRange = { + displayType: DisplayType.Line, + connection: 'test-connection', + from: { databaseName: 'default', tableName: 'otel_logs' }, + select: [{ aggFn: 'count', valueExpression: '' }], + where: '', + timestampValueExpression: 'Timestamp', + dateRange: [new Date('2025-02-12'), new Date('2025-02-14')], + granularity: '1 minute', + }; + expect( + await renderMetricExemplarsChartConfig(logConfig, mockMetadata), + ).toBeNull(); + }); +}); diff --git a/packages/common-utils/src/core/renderChartConfig.ts b/packages/common-utils/src/core/renderChartConfig.ts index 2f22cbabcd..f3bbad6dbf 100644 --- a/packages/common-utils/src/core/renderChartConfig.ts +++ b/packages/common-utils/src/core/renderChartConfig.ts @@ -2283,6 +2283,137 @@ export async function renderChartConfig( ]); } +/** Overall cap on exemplar markers returned for a single chart, so a wide + * time range can't flood the chart overlay with thousands of points. */ +export const EXEMPLAR_QUERY_LIMIT = 200; + +/** + * The exemplar-eligibility rule, in one place. An exemplar marker sits at a + * single trace's raw measurement on the chart's y-axis, so it is only + * attributable when the chart draws exactly one series of a compatible unit: + * + * - single series, no Group By — a marker can't be attributed (or scaled) across + * series, and a grouped scan pools exemplars from every group. + * - not a ratio — the exemplar value isn't on a ratio axis. + * - histogram — the exemplar value is a duration, which only shares the y-axis + * unit on a latency histogram (counts/gauges/rates are a different scale). + * + * The chart-editor toggle and the SQL renderer both delegate here so they can't + * drift apart. + */ +export function isExemplarEligible({ + seriesCount, + seriesReturnType, + metricType, + hasGroupBy, +}: { + seriesCount: number; + seriesReturnType?: 'ratio' | 'column'; + // Deliberately `string`, and compared by value below. tsup emits this + // declaration without importing MetricsDataType, so any annotation naming the + // enum resolves against the *consumer's* scope: in packages/app it binds to + // that package's unrelated legacy MetricsDataType, whose values are + // capitalised ('Histogram'), and the build fails on the mismatch. + // Callers must pass a common-utils MetricsDataType value (lowercase + // 'histogram'); a legacy-enum value would compile but never match. + metricType?: string; + hasGroupBy: boolean; +}): boolean { + return ( + seriesCount === 1 && + seriesReturnType !== 'ratio' && + metricType === MetricsDataType.Histogram && + !hasGroupBy + ); +} + +/** + * Builds a ClickHouse query that surfaces native exemplars stored on an OTel + * metric table (`Exemplars.TraceId/SpanId/Value/TimeUnix`). Returns null when + * the config is not a single-metric chart we can resolve a table for. + * + * Reuses `renderWhere` so the exemplar scan honors the exact same time range, + * metric-name, and user filters as the rendered series. Exemplars are kept as + * their own raw points (the marker sits at the exemplar's own value/time), not + * bucketed — so no `timeBucketExpr` here. + */ +export async function renderMetricExemplarsChartConfig( + chartConfig: ChartConfigWithOptDateRangeEx, + metadata: Metadata, +): Promise { + if ( + isRawSqlChartConfig(chartConfig) || + isPromqlChartConfig(chartConfig) || + !isMetricChartConfig(chartConfig) || + !Array.isArray(chartConfig.select) + ) { + return null; + } + const { metricTables, select } = chartConfig; + const { metricType, metricName, metricNameSql } = select[0] ?? {}; + // Shared eligibility rule (single, non-ratio, non-grouped histogram series) — + // see isExemplarEligible. A Group By in particular would pool exemplars from + // every group into one unattributable set. + if ( + !isExemplarEligible({ + seriesCount: select.length, + seriesReturnType: chartConfig.seriesReturnType, + metricType, + hasGroupBy: isUsingGroupBy(chartConfig), + }) + ) { + return null; + } + const table = + metricType && metricTables ? metricTables[metricType] : undefined; + if (!metricType || !metricName || !table) { + return null; + } + + // Build a config that points at the concrete metric-type table and carries + // the metric-name predicate alongside the user filters, then let renderWhere + // assemble the time filter + filters exactly as the main query does. The + // guards above narrow chartConfig to the metric builder config, so no cast. + const whereConfig: BuilderChartConfigWithOptDateRangeEx = { + ...chartConfig, + from: { ...chartConfig.from, tableName: table }, + timestampValueExpression: + chartConfig.timestampValueExpression || DEFAULT_METRIC_TABLE_TIME_COLUMN, + // Keep the original select so renderWhere applies the series' aggCondition — + // otherwise the exemplar scan would surface traces from other series (e.g. + // other services/routes/tenants) that share the same metric name. + }; + + const where = await renderWhere(whereConfig, metadata); + const from = renderFrom({ from: whereConfig.from }); + + // The metric-name predicate is REQUIRED and must always be ANDed. Appending it + // to `chartConfig.filters` would subject it to the chart's + // `filtersLogicalOperator`, so an 'OR' filter group would produce + // `userFilterA OR userFilterB OR MetricName = ...` and let the exemplar scan + // surface traces from other metrics. AND it separately from the user filters. + const metricNameCondition = createMetricNameFilter(metricName, metricNameSql); + + return concatChSql(' ', [ + chSql`SELECT + toUnixTimestamp64Milli(ex_TimeUnix) AS timestamp, + ex_Value AS value, + ex_TraceId AS traceId, + ex_SpanId AS spanId`, + chSql`FROM ${from}`, + chSql`ARRAY JOIN + \`Exemplars.TimeUnix\` AS ex_TimeUnix, + \`Exemplars.Value\` AS ex_Value, + \`Exemplars.TraceId\` AS ex_TraceId, + \`Exemplars.SpanId\` AS ex_SpanId`, + chSql`WHERE ${where.sql ? where : chSql`1 = 1`} AND (${metricNameCondition}) AND notEmpty(ex_TraceId)`, + // Native exemplars carry no interestingness signal; keep the highest-value + // ones as a stable cap. ponytail: value-desc cap, revisit if even sampling + // across buckets is wanted. + chSql`ORDER BY value DESC LIMIT ${{ Int32: EXEMPLAR_QUERY_LIMIT }}`, + ]); +} + // EditForm -> translateToQueriedChartConfig -> QueriedChartConfig // renderFn(QueriedChartConfig) -> sql // query(sql) -> data diff --git a/packages/common-utils/src/types.ts b/packages/common-utils/src/types.ts index 9ef47c1da4..c16ca2ab25 100644 --- a/packages/common-utils/src/types.ts +++ b/packages/common-utils/src/types.ts @@ -1229,6 +1229,15 @@ const SharedChartSettingsSchema = z.object({ // number tiles have no time dimension to bucket). Other display types // ignore the field. Kept at shared level mirroring `color` / `colorRules`. backgroundChart: BackgroundChartSchema.optional(), + // Opt-in: overlay exemplar markers (individual traces linked to the series) + // on time charts. Sourced natively for metric/PromQL sources and generated + // per time bucket for trace sources. The UI gates the toggle on supported + // source kinds; off by default so the extra exemplar query never runs. + enableExemplars: z.boolean().optional(), + // Trace source that an exemplar's trace id resolves against — used to fetch + // hover metadata and to deep-link straight to the trace. When unset, the + // chart source's linked `traceSourceId` is used as a fallback. + exemplarTraceSourceId: z.string().optional(), // Zebra striping for table tiles: when true, the renderer tints alternating // rows so wide tables are easier to scan across. Applies to any table tile // (builder or raw SQL); the striping is purely presentational and keys off @@ -1366,6 +1375,24 @@ export const ChartConfigSchema = z.union([ export type ChartConfig = z.infer; +/** + * A single exemplar: an individual data point overlaid on a time chart that + * links back to a trace. Shared shape for both backends — metric/PromQL + * sources surface native exemplars, trace sources generate them per bucket. + */ +export const ExemplarSchema = z.object({ + timestamp: z.number(), // epoch ms, x-position on the chart + value: z.number(), // exemplar's own value (metric Value / trace Duration in ms) + traceId: z.string(), + spanId: z.string().optional(), + // Matches the series the exemplar belongs to (LineData.displayName) so the + // overlay can attach markers to the right line. Undefined => applies to all. + groupKey: z.string().optional(), + attributes: z.record(z.string()).optional(), +}); + +export type Exemplar = z.infer; + export type DateRange = { dateRange: [Date, Date]; dateRangeStartInclusive?: boolean; // default true @@ -1669,6 +1696,9 @@ export const TeamClickHouseSettingsSchema = z.object({ metadataMaxRowsToRead: z.number().optional(), parallelizeWhenPossible: z.boolean().optional(), filterKeysFetchLimit: z.number().optional(), + // Target number of exemplar markers shown per chart (0 = unlimited). Not a + // ClickHouse setting, but lives in the same team-config bag for reuse. + maxExemplars: z.number().optional(), }); /** Accepts null to unset (reset to default) a setting. */ @@ -1679,6 +1709,7 @@ export const TeamClickHouseSettingsUpdateSchema = z.object({ metadataMaxRowsToRead: z.number().nullish(), parallelizeWhenPossible: z.boolean().nullish(), filterKeysFetchLimit: z.number().nullish(), + maxExemplars: z.number().nullish(), }); export type TeamClickHouseSettingsUpdate = z.infer< typeof TeamClickHouseSettingsUpdateSchema diff --git a/packages/otel-collector/builder-config.yaml b/packages/otel-collector/builder-config.yaml index ecd794a456..e84cad931a 100644 --- a/packages/otel-collector/builder-config.yaml +++ b/packages/otel-collector/builder-config.yaml @@ -135,6 +135,9 @@ connectors: - gomod: github.com/open-telemetry/opentelemetry-collector-contrib/connector/routingconnector v__OTEL_COLLECTOR_VERSION__ + - gomod: + github.com/open-telemetry/opentelemetry-collector-contrib/connector/spanmetricsconnector + v__OTEL_COLLECTOR_VERSION__ extensions: # Core diff --git a/scripts/dev-env.sh b/scripts/dev-env.sh index 938aaeb4d7..135eca7ef5 100755 --- a/scripts/dev-env.sh +++ b/scripts/dev-env.sh @@ -55,6 +55,7 @@ HDX_DEV_OTEL_GRPC_PORT=$((30800 + HDX_DEV_SLOT)) HDX_DEV_OTEL_HTTP_PORT=$((30900 + HDX_DEV_SLOT)) HDX_DEV_OTEL_METRICS_PORT=$((31000 + HDX_DEV_SLOT)) HDX_DEV_OTEL_JSON_HTTP_PORT=$((31100 + HDX_DEV_SLOT)) +HDX_DEV_PROMETHEUS_PORT=$((31200 + HDX_DEV_SLOT)) # --- Docker Compose project name (unique per slot) --- HDX_DEV_PROJECT="hdx-dev-${HDX_DEV_SLOT}" @@ -80,6 +81,7 @@ export HDX_DEV_OTEL_GRPC_PORT export HDX_DEV_OTEL_HTTP_PORT export HDX_DEV_OTEL_METRICS_PORT export HDX_DEV_OTEL_JSON_HTTP_PORT +export HDX_DEV_PROMETHEUS_PORT export HDX_DEV_PROJECT export NX_CACHE_DIRECTORY @@ -111,6 +113,7 @@ cat > "${HDX_DEV_SLOTS_DIR}/${HDX_DEV_SLOT}.json" < ClickHouse). Metric + * exemplars are written to ClickHouse directly because OTel JS (1.30) does not + * yet emit metric exemplars and our collector has no spanmetrics connector. + * See README for the spanmetrics upgrade path. + */ +'use strict'; + +const { + trace, + ROOT_CONTEXT, + SpanStatusCode, + SpanKind, + diag, + DiagConsoleLogger, + DiagLogLevel, +} = require('@opentelemetry/api'); +// Surface OTLP export failures (auth, connection) instead of dropping silently. +diag.setLogger(new DiagConsoleLogger(), DiagLogLevel.ERROR); +const { NodeTracerProvider } = require('@opentelemetry/sdk-trace-node'); +const { BatchSpanProcessor } = require('@opentelemetry/sdk-trace-base'); +const { + OTLPTraceExporter, +} = require('@opentelemetry/exporter-trace-otlp-proto'); +const { Resource } = require('@opentelemetry/resources'); +const { ATTR_SERVICE_NAME } = require('@opentelemetry/semantic-conventions'); +const net = require('net'); + +// ── Config ────────────────────────────────────────────────────────── +const cfg = { + // OTLP HTTP base (port 4318). Traces are POSTed to /v1/traces. + otlpEndpoint: + process.env.OTEL_EXPORTER_OTLP_ENDPOINT || 'http://otel-collector:4318', + backfillMinutes: Number(process.env.GEN_BACKFILL_MINUTES || 30), + ratePerSec: Number(process.env.GEN_RATE_PER_SEC || 20), + // HyperDX collectors enforce bearer-token auth on OTLP ingest. Default to the + // dev INGESTION_API_KEY (packages/api/.env.development); override per env. + otlpApiKey: + process.env.GEN_OTLP_API_KEY || 'super-secure-ingestion-api-key', +}; + +// ── Weighted / uniform random helpers ─────────────────────────────── +function pickWeighted(choices) { + const total = choices.reduce((s, c) => s + c[1], 0); + let r = Math.random() * total; + for (const [value, weight] of choices) { + r -= weight; + if (r <= 0) return value; + } + return choices[choices.length - 1][0]; +} +function pickUniform(arr) { + return arr[Math.floor(Math.random() * arr.length)]; +} +// Box-Muller gaussian, clamped to >= 1ms +function gaussianMs(mean, stddev) { + let u = 0; + let v = 0; + while (u === 0) u = Math.random(); + while (v === 0) v = Math.random(); + const z = Math.sqrt(-2 * Math.log(u)) * Math.cos(2 * Math.PI * v); + return Math.max(1, mean + z * stddev); +} + +// ── Attribute pools (cardinality + diversity) ─────────────────────── +const routes = [ + ['/api/users', 25], + ['/api/orders', 15], + ['/cart/checkout', 10], + ['/api/search', 25], + ['/api/products', 15], + ['/api/auth', 10], +]; +const methods = [ + ['GET', 60], + ['POST', 25], + ['PUT', 10], + ['DELETE', 5], +]; +const regions = [ + ['us-east-1', 40], + ['us-west-2', 30], + ['eu-west-1', 20], + ['ap-southeast-1', 10], +]; +const buildIDs = [ + ['build-7a1', 35], + ['build-7a2', 35], + ['build-7a3', 30], +]; +const platforms = [ + ['web', 50], + ['ios', 30], + ['android', 20], +]; +const featureFlags = [ + ['new-checkout-flow', 15], + ['dark-launch-search', 10], + ['legacy', 75], +]; +const tenants = [ + 'tenant-acme', + 'tenant-globex', + 'tenant-initech', + 'tenant-umbrella', +]; +const pods = Array.from({ length: 8 }, (_, i) => `pod-abc-${i + 1}`); +const userId = () => `user-${String(Math.floor(Math.random() * 500) + 1).padStart(4, '0')}`; + +function routeToService(route) { + switch (route) { + case '/api/orders': + case '/cart/checkout': + return 'order-service'; + case '/api/users': + case '/api/auth': + return 'user-service'; + case '/api/search': + case '/api/products': + return 'search-service'; + default: + return 'unknown-service'; + } +} +function serviceToDb(svc) { + if (svc === 'order-service' || svc === 'user-service') return 'postgres'; + if (svc === 'search-service') return 'elasticsearch'; + return 'none'; +} + +// ── Per-service tracer providers (ServiceName = resource attr) ─────── +const SERVICE_NAMES = [ + 'api-gateway', + 'order-service', + 'user-service', + 'search-service', + 'payment-service', + 'notification-service', +]; +const exporter = new OTLPTraceExporter({ + url: `${cfg.otlpEndpoint.replace(/\/$/, '')}/v1/traces`, + // HyperDX's bearertokenauth uses scheme:'' — send the raw token, no prefix. + headers: cfg.otlpApiKey ? { authorization: cfg.otlpApiKey } : undefined, +}); +const providers = new Map(); +const tracers = new Map(); +for (const name of SERVICE_NAMES) { + const provider = new NodeTracerProvider({ + resource: new Resource({ + [ATTR_SERVICE_NAME]: name, + 'service.version': '1.0.0', + }), + spanProcessors: [ + // Large queue so the backfill burst isn't dropped before export. + new BatchSpanProcessor(exporter, { + maxExportBatchSize: 512, + maxQueueSize: 32768, + scheduledDelayMillis: 500, + }), + ], + }); + providers.set(name, provider); + tracers.set(name, provider.getTracer('telemetry-generator')); +} +const tracer = name => tracers.get(name); + +const jitter = () => Math.random() * 2; // ms + +function commonAttrs(a) { + return { + 'http.method': a.method, + 'http.route': a.route, + 'user.id': a.uid, + 'app.tenant_id': a.tenant, + 'host.region': a.region, + 'app.build_id': a.buildID, + 'app.platform': a.platform, + 'app.feature_flag': a.featureFlag, + 'k8s.pod.name': a.pod, + }; +} +function markError(span, type, message) { + span.setStatus({ code: SpanStatusCode.ERROR, message }); + span.addEvent('exception', { + 'exception.type': type, + 'exception.message': message, + }); +} + +// ── Trace emission ────────────────────────────────────────────────── +// Emits a trace; the collector's spanmetrics connector derives the +// request-duration metric (with exemplars) from these spans. +function emitTrace(tsMs) { + const a = { + route: pickWeighted(routes), + method: pickWeighted(methods), + region: pickWeighted(regions), + buildID: pickWeighted(buildIDs), + platform: pickWeighted(platforms), + featureFlag: pickWeighted(featureFlags), + tenant: pickUniform(tenants), + uid: userId(), + pod: pickUniform(pods), + }; + const svc = routeToService(a.route); + const sc = detectScenario(a); + const plan = SCENARIOS[sc](a, svc); + + const root = tracer('api-gateway').startSpan( + `${a.method} ${a.route}`, + { + kind: SpanKind.SERVER, + startTime: tsMs, + attributes: { ...commonAttrs(a), 'http.status_code': plan.statusCode }, + }, + ROOT_CONTEXT, + ); + if (plan.error) markError(root, plan.error.type, plan.error.message); + else root.setStatus({ code: SpanStatusCode.OK }); + + const rootCtx = trace.setSpan(ROOT_CONTEXT, root); + const svcStart = tsMs + jitter(); + const svcSpan = tracer(svc).startSpan( + `${svc}.handle`, + { kind: SpanKind.INTERNAL, startTime: svcStart, attributes: svcAttrs(svc, a) }, + rootCtx, + ); + if (plan.error) markError(svcSpan, plan.error.type, plan.error.message); + const svcCtx = trace.setSpan(rootCtx, svcSpan); + + // Leaf spans (db / cache / downstream) per the scenario plan. + let cursor = svcStart + jitter(); + for (const leaf of plan.leaves) { + const leafSvc = leaf.service || svc; + const ls = tracer(leafSvc).startSpan( + leaf.name, + { + kind: SpanKind.CLIENT, + startTime: cursor, + attributes: { + [ATTR_SERVICE_NAME]: leafSvc, + 'host.region': a.region, + ...(leaf.attrs || {}), + }, + }, + svcCtx, + ); + if (leaf.error) markError(ls, leaf.error.type, leaf.error.message); + ls.end(cursor + leaf.durMs); + cursor += leaf.durMs + 1; + } + + svcSpan.end(svcStart + plan.svcDurMs); + root.end(tsMs + plan.rootDurMs); +} + +function svcAttrs(svc, a) { + return { + [ATTR_SERVICE_NAME]: svc, + 'http.route': a.route, + 'host.region': a.region, + 'app.build_id': a.buildID, + 'app.feature_flag': a.featureFlag, + 'app.platform': a.platform, + 'user.id': a.uid, + 'app.tenant_id': a.tenant, + 'k8s.pod.name': a.pod, + }; +} + +// ── Scenarios ─────────────────────────────────────────────────────── +// Each returns { rootDurMs, svcDurMs, statusCode, error?, leaves:[{name,durMs,service?,attrs?,error?}] } +const PAYMENT_TIMEOUT_RATE = 0.05; +const AUTH_LEAK_ERROR_RATE = 0.1; + +function detectScenario(a) { + const svc = routeToService(a.route); + if ( + a.route === '/cart/checkout' && + a.region === 'us-west-2' && + Math.random() < PAYMENT_TIMEOUT_RATE + ) + return 'paymentTimeout'; + if ( + a.route === '/cart/checkout' && + a.featureFlag === 'new-checkout-flow' && + a.region === 'eu-west-1' + ) + return 'slowCheckout'; + if (a.route === '/api/orders' && a.platform === 'ios' && a.buildID === 'build-7a3') + return 'iosOrderErrors'; + if ( + a.tenant === 'tenant-initech' && + a.featureFlag === 'dark-launch-search' && + a.route === '/api/search' + ) + return 'initechSearch'; + if ( + a.route === '/api/auth' && + a.buildID === 'build-7a3' && + (a.pod === 'pod-abc-7' || a.pod === 'pod-abc-8') + ) + return 'authMemoryLeak'; + if (a.region === 'ap-southeast-1' && svc === 'user-service') + return 'redisTimeoutApac'; + if (a.tenant === 'tenant-umbrella' && a.region === 'eu-west-1') + return 'umbrellaCompliance'; + return 'normal'; +} + +function normalStatus() { + const r = Math.random(); + if (r < 0.95) return 200; + if (r < 0.98) return 201; + return 404; +} + +const SCENARIOS = { + normal: (a, svc) => { + const db = serviceToDb(svc); + const leaves = []; + if (db === 'postgres') + leaves.push({ + name: 'postgres.query', + durMs: gaussianMs(10, 5), + attrs: { 'db.system': 'postgres', 'db.statement': `SELECT * FROM ${a.route.slice(5)}` }, + }); + else if (db === 'elasticsearch') + leaves.push({ + name: 'elasticsearch.search', + durMs: gaussianMs(10, 5), + attrs: { 'db.system': 'elasticsearch' }, + }); + if (a.route === '/cart/checkout') + leaves.push({ + name: 'payment-service.charge', + service: 'payment-service', + durMs: gaussianMs(10, 4), + }); + return { rootDurMs: gaussianMs(40, 20), svcDurMs: gaussianMs(25, 12), statusCode: normalStatus(), leaves }; + }, + slowCheckout: a => { + const leaves = []; + const n = 3 + Math.floor(Math.random() * 3); + for (let i = 0; i < n; i++) + leaves.push({ + name: 'postgres.query', + durMs: gaussianMs(200, 40), + attrs: { 'db.system': 'postgres', 'db.statement': 'SELECT * FROM orders WHERE id = ?' }, + }); + leaves.push({ name: 'payment-service.charge', service: 'payment-service', durMs: gaussianMs(200, 50) }); + return { rootDurMs: gaussianMs(1500, 400), svcDurMs: gaussianMs(1200, 350), statusCode: 200, leaves }; + }, + iosOrderErrors: () => ({ + rootDurMs: gaussianMs(250, 60), + svcDurMs: gaussianMs(100, 30), + statusCode: 500, + error: { type: 'ValidationError', message: 'malformed request body' }, + leaves: [], + }), + redisTimeoutApac: a => ({ + rootDurMs: gaussianMs(650, 120), + svcDurMs: gaussianMs(580, 100), + statusCode: 200, + leaves: [ + { name: 'redis.get', durMs: gaussianMs(550, 100), attrs: { 'db.system': 'redis', 'db.statement': `GET user:session:${a.uid}` } }, + { name: 'postgres.query', durMs: gaussianMs(30, 10), attrs: { 'db.system': 'postgres' } }, + ], + }), + initechSearch: () => ({ + rootDurMs: gaussianMs(3000, 500), + svcDurMs: gaussianMs(2800, 450), + statusCode: 500, + error: { type: 'TimeoutError', message: 'elasticsearch timeout' }, + leaves: [ + { + name: 'elasticsearch.search', + durMs: gaussianMs(2500, 400), + attrs: { 'db.system': 'elasticsearch' }, + error: { type: 'IOException', message: 'read tcp: i/o timeout' }, + }, + ], + }), + authMemoryLeak: a => { + const err = Math.random() < AUTH_LEAK_ERROR_RATE; + return { + rootDurMs: gaussianMs(800, 200), + svcDurMs: gaussianMs(700, 180), + statusCode: err ? 503 : 200, + error: err ? { type: 'ServiceUnavailableError', message: 'GC overhead' } : undefined, + leaves: [{ name: 'redis.get', durMs: gaussianMs(600, 150), attrs: { 'db.system': 'redis', 'db.statement': `GET auth:token:${a.uid}` } }], + }; + }, + paymentTimeout: () => ({ + rootDurMs: gaussianMs(5000, 500), + svcDurMs: gaussianMs(4800, 450), + statusCode: 504, + error: { type: 'TimeoutError', message: 'gateway timeout' }, + leaves: [ + { name: 'postgres.query', durMs: gaussianMs(20, 8), attrs: { 'db.system': 'postgres' } }, + { + name: 'payment-service.charge', + service: 'payment-service', + durMs: gaussianMs(4500, 300), + error: { type: 'TimeoutError', message: 'context deadline exceeded' }, + }, + ], + }), + umbrellaCompliance: (a, svc) => { + const overhead = gaussianMs(150, 40); + const db = serviceToDb(svc); + const leaves = [{ name: 'compliance.data_residency_check', durMs: overhead, attrs: { 'app.tenant_id': a.tenant } }]; + if (db === 'postgres') leaves.push({ name: 'postgres.query', durMs: gaussianMs(10, 5), attrs: { 'db.system': 'postgres' } }); + else if (db === 'elasticsearch') leaves.push({ name: 'elasticsearch.search', durMs: gaussianMs(10, 5), attrs: { 'db.system': 'elasticsearch' } }); + return { rootDurMs: gaussianMs(40, 20) + overhead, svcDurMs: gaussianMs(40, 20) + overhead, statusCode: normalStatus(), leaves }; + }, +}; + +// The collector's OTLP receiver is configured dynamically via OpAMP, so the +// port isn't open the instant the container starts — it appears only after the +// collector fetches its remote config. Wait for it before backfilling, else the +// initial burst hits ECONNREFUSED and is lost. +function waitForCollector(timeoutMs = 120_000) { + const u = new URL(cfg.otlpEndpoint); + const port = Number(u.port) || (u.protocol === 'https:' ? 443 : 80); + const host = u.hostname; + const deadline = Date.now() + timeoutMs; + const tryOnce = () => + new Promise(resolve => { + const sock = net.connect({ host, port }, () => { + sock.destroy(); + resolve(true); + }); + sock.on('error', () => { + sock.destroy(); + resolve(false); + }); + sock.setTimeout(2000, () => { + sock.destroy(); + resolve(false); + }); + }); + return (async () => { + while (Date.now() < deadline) { + if (await tryOnce()) return true; + await new Promise(r => setTimeout(r, 2000)); + } + return false; + })(); +} + +// ── Main ──────────────────────────────────────────────────────────── +async function main() { + console.log( + `telemetry-generator starting: otlp=${cfg.otlpEndpoint} ` + + `backfill=${cfg.backfillMinutes}m rate=${cfg.ratePerSec}/s`, + ); + + console.log('waiting for collector OTLP endpoint...'); + const ready = await waitForCollector(); + console.log(ready ? 'collector reachable' : 'collector wait timed out, proceeding anyway'); + + // Backfill historical traces. The spanmetrics connector derives the + // request-duration metric (with exemplars) from these spans. + const now = Date.now(); + const backfillStart = now - cfg.backfillMinutes * 60_000; + const backfillCount = cfg.backfillMinutes * 60 * cfg.ratePerSec; + console.log(`backfilling ~${backfillCount} requests over ${cfg.backfillMinutes}m...`); + for (let i = 0; i < backfillCount; i++) { + const ts = backfillStart + Math.random() * (now - backfillStart); + emitTrace(ts); + // Periodically yield so the span exporter can drain its queue instead of + // overflowing (which would drop traces, and thus their exemplars). + if (i % 500 === 0) await new Promise(r => setTimeout(r, 25)); + } + // Let the final span batches export before we declare backfill done. + await new Promise(r => setTimeout(r, 3000)); + console.log('backfill complete, starting live emission...'); + + // Live emission. + const intervalMs = Math.max(1, Math.floor(1000 / cfg.ratePerSec)); + setInterval(() => emitTrace(Date.now()), intervalMs); + + const shutdown = async () => { + console.log('shutting down...'); + await Promise.all( + [...providers.values()].map(p => p.shutdown().catch(() => {})), + ); + process.exit(0); + }; + process.on('SIGINT', shutdown); + process.on('SIGTERM', shutdown); +} + +main().catch(e => { + console.error(e); + process.exit(1); +});