From 536e37bb123815fdf7c9e53477f178e4eef71c08 Mon Sep 17 00:00:00 2001 From: Jordan Simonovski Date: Mon, 29 Jun 2026 15:49:07 +1000 Subject: [PATCH 01/10] feat(HDX) Added support for exemplars --- packages/api/src/config.ts | 11 + packages/api/src/models/team.ts | 1 + .../__tests__/prometheus.integration.test.ts | 58 +++++ packages/api/src/routers/api/prometheus.ts | 80 +++++++ packages/app/src/HDXMultiSeriesTimeChart.tsx | 151 ++++++++++++- packages/app/src/api.ts | 29 +++ .../ChartEditor/PromqlChartEditor.tsx | 34 ++- .../app/src/components/ChartEditor/utils.ts | 4 + .../ChartEditorControls.tsx | 59 ++++- packages/app/src/components/DBTimeChart.tsx | 212 ++++++++++++++++- .../TeamSettings/TeamQueryConfigSection.tsx | 21 ++ .../components/__tests__/DBTimeChart.test.tsx | 9 + packages/app/src/defaults.ts | 2 + .../src/hooks/__tests__/useExemplars.test.ts | 61 +++++ packages/app/src/hooks/useExemplars.tsx | 213 ++++++++++++++++++ .../src/__tests__/renderChartConfig.test.ts | 115 ++++++++++ .../src/core/renderChartConfig.ts | 88 ++++++++ packages/common-utils/src/types.ts | 31 +++ 18 files changed, 1169 insertions(+), 10 deletions(-) create mode 100644 packages/app/src/hooks/__tests__/useExemplars.test.ts create mode 100644 packages/app/src/hooks/useExemplars.tsx diff --git a/packages/api/src/config.ts b/packages/api/src/config.ts index 6677989124..5a34941e33 100644 --- a/packages/api/src/config.ts +++ b/packages/api/src/config.ts @@ -51,6 +51,17 @@ export const DEFAULT_SOURCES = env.DEFAULT_SOURCES; export const IS_PROMQL_ENABLED = env.ENABLE_PROMQL === 'true'; +// Opt-in: have the collector derive request metrics (with trace exemplars) from +// spans via the spanmetrics connector. Off by default; enabled in dev so the +// telemetry-generator's traces produce coherent metric exemplars end-to-end. +export const IS_SPAN_METRICS_ENABLED = env.ENABLE_SPAN_METRICS === 'true'; + +// Opt-in: also remote-write the span-derived metrics (with exemplars) to a +// Prometheus endpoint (SPAN_METRICS_PROM_RW_ENDPOINT on the collector) so the +// native Prometheus query_exemplars path can be tested against real data. +export const IS_SPAN_METRICS_PROM_RW_ENABLED = + env.ENABLE_SPAN_METRICS_PROM_RW === 'true'; + // FOR CI ONLY export const CLICKHOUSE_HOST = env.CLICKHOUSE_HOST as string; export const CLICKHOUSE_USER = env.CLICKHOUSE_USER as string; diff --git a/packages/api/src/models/team.ts b/packages/api/src/models/team.ts index 4d6b620162..259c695f2c 100644 --- a/packages/api/src/models/team.ts +++ b/packages/api/src/models/team.ts @@ -42,6 +42,7 @@ export default mongoose.model( fieldMetadataDisabled: Boolean, parallelizeWhenPossible: Boolean, filterKeysFetchLimit: Number, + maxExemplars: Number, }, { timestamps: true, diff --git a/packages/api/src/routers/api/__tests__/prometheus.integration.test.ts b/packages/api/src/routers/api/__tests__/prometheus.integration.test.ts index dc48dd8499..dcfee80a83 100644 --- a/packages/api/src/routers/api/__tests__/prometheus.integration.test.ts +++ b/packages/api/src/routers/api/__tests__/prometheus.integration.test.ts @@ -270,4 +270,62 @@ describe('prometheus router', () => { expect(calledUrl).toContain('/api/v1/label/__name__/values'); }); }); + + describe('GET /v1/prometheus/query_exemplars', () => { + it('returns 400 when query parameter is missing', async () => { + const { agent } = await getLoggedInAgent(server); + const res = await agent + .get('/v1/prometheus/query_exemplars') + .query({ connectionId: new Types.ObjectId().toString() }) + .expect(400); + expect(res.body).toMatchObject({ + status: 'error', + errorType: 'bad_data', + error: expect.stringContaining('query'), + }); + }); + + it('proxies to upstream Prometheus when connection isPrometheusEndpoint', async () => { + const { agent, team } = await getLoggedInAgent(server); + const conn = await seedPrometheusConnection(team._id); + + const promResponse = { status: 'success', data: [] }; + mockFetch.mockResolvedValueOnce( + fakeUpstreamResponse(promResponse) as any, + ); + + const res = await agent + .get('/v1/prometheus/query_exemplars') + .query({ + query: 'up', + start: '1700000000', + end: '1700000060', + connectionId: conn._id.toString(), + }) + .expect(200); + + expect(res.body).toEqual(promResponse); + const calledUrl = mockFetch.mock.calls[0][0] as string; + expect(calledUrl).toContain('/api/v1/query_exemplars'); + expect(calledUrl).toContain('query=up'); + }); + + it('returns an empty result for ClickHouse-backed connections (no native exemplar table function)', async () => { + const { agent, team } = await getLoggedInAgent(server); + const conn = await seedClickHouseConnection(team._id); + + const res = await agent + .get('/v1/prometheus/query_exemplars') + .query({ + query: 'up', + start: '1700000000', + end: '1700000060', + connectionId: conn._id.toString(), + }) + .expect(200); + + expect(res.body).toEqual({ status: 'success', data: [] }); + expect(mockFetch).not.toHaveBeenCalled(); + }); + }); }); diff --git a/packages/api/src/routers/api/prometheus.ts b/packages/api/src/routers/api/prometheus.ts index 07f446816f..ad35357206 100644 --- a/packages/api/src/routers/api/prometheus.ts +++ b/packages/api/src/routers/api/prometheus.ts @@ -478,6 +478,86 @@ const queryHandler: express.RequestHandler = async (req, res) => { router.get('/query', queryHandler); router.post('/query', queryHandler); +// -------------------------- +// GET|POST /query_exemplars +// -------------------------- + +// Native Prometheus exposes exemplars via /api/v1/query_exemplars. We proxy +// straight through for Prometheus-backed connections. ClickHouse-backed metric +// exemplars are read directly from the OTel metric tables' `Exemplars.*` +// columns in the app (via renderMetricExemplarsChartConfig), so there is no +// ClickHouse table function to call here — return an empty result instead. +const queryExemplarsHandler: express.RequestHandler = async (req, res) => { + const startedAt = performance.now(); + let backend: PrometheusBackend = 'unknown'; + try { + const { teamId } = getNonNullUserWithTeam(req); + const params = getParams(req); + + const query = params.query; + if (!query) { + return res.status(400).json({ + status: 'error', + errorType: 'bad_data', + error: 'missing required parameter: query', + }); + } + + const connectionId = params.connectionId; + if (!connectionId) { + return res.status(400).json({ + status: 'error', + errorType: 'bad_data', + error: 'missing required parameter: connectionId', + }); + } + + const connection = await getConnectionById( + teamId.toString(), + connectionId, + true, + ); + if (!connection) { + return res.status(404).json({ + status: 'error', + errorType: 'bad_data', + error: 'Connection not found', + }); + } + + if (connection.isPrometheusEndpoint) { + backend = 'prometheus'; + await proxyToPrometheus( + connection.host, + '/api/v1/query_exemplars', + params, + res, + ); + return; + } + + // ClickHouse-backed PromQL: no native exemplar table function. Exemplars + // for structured metric charts are fetched app-side from the metric table. + backend = 'clickhouse'; + return res.json({ status: 'success', data: [] }); + } catch (e) { + prometheusQueryErrors.add(1, { endpoint: 'query_exemplars', backend }); + logger.error(e, 'Prometheus query_exemplars error'); + return res.status(400).json({ + status: 'error', + errorType: 'bad_data', + error: e instanceof Error ? e.message : String(e), + }); + } finally { + prometheusQueryDuration.record(performance.now() - startedAt, { + endpoint: 'query_exemplars', + backend, + }); + } +}; +router.get('/query_exemplars', queryExemplarsHandler); +router.post('/query_exemplars', queryExemplarsHandler); + // -------------------------- // GET /label/:name/values // -------------------------- diff --git a/packages/app/src/HDXMultiSeriesTimeChart.tsx b/packages/app/src/HDXMultiSeriesTimeChart.tsx index 06b2c1a61f..878fc6489a 100644 --- a/packages/app/src/HDXMultiSeriesTimeChart.tsx +++ b/packages/app/src/HDXMultiSeriesTimeChart.tsx @@ -11,6 +11,7 @@ import { CartesianGrid, Legend, ReferenceArea, + ReferenceDot, ReferenceLine, ResponsiveContainer, Tooltip, @@ -19,7 +20,7 @@ import { } from 'recharts'; import { AxisDomain } from 'recharts/types/util/types'; import { convertGranularityToSeconds } from '@hyperdx/common-utils/dist/core/utils'; -import { DisplayType } from '@hyperdx/common-utils/dist/types'; +import { DisplayType, Exemplar } from '@hyperdx/common-utils/dist/types'; import { Popover } from '@mantine/core'; import type { NumberFormat } from '@/types'; @@ -428,6 +429,39 @@ function CaptureActiveDot({ ); } +function numOrNull(v: unknown): number | null { + return typeof v === 'number' && !isNaN(v) ? v : null; +} + +/** + * Diamond marker for an exemplar, drawn via . + * Recharts injects cx/cy. Hovering opens a floating menu (handled by the parent + * via onHoverStart/onHoverEnd) to inspect the linked trace — the marker itself + * is not a click target. A larger transparent hit circle eases hovering. + */ +function ExemplarDot(props: any) { + const { cx, cy, exemplar, onHoverStart, onHoverEnd } = props; + if (typeof cx !== 'number' || typeof cy !== 'number') { + return null; + } + const s = 4; + return ( + onHoverStart?.(exemplar, cx, cy)} + onMouseLeave={() => onHoverEnd?.()} + > + + + + ); +} + /** * Compute the unique set of hexes referenced by `` defs * inside MemoChart. Exported so a unit test can pin the dedup-and-union @@ -476,6 +510,10 @@ export const MemoChart = memo(function MemoChart({ granularity, dateRangeEndInclusive = true, fitYAxisToData = false, + exemplars, + maxExemplars = 12, + onExemplarHover, + onExemplarHoverEnd, }: { graphResults: any[]; setIsClickActive: (v: any) => void; @@ -502,6 +540,14 @@ export const MemoChart = memo(function MemoChart({ * (with padding) instead of zero. **/ fitYAxisToData?: boolean; + /** Exemplar markers to overlay on the chart (linked to traces). */ + exemplars?: Exemplar[]; + /** Target number of exemplar markers to show (0 = unlimited). */ + maxExemplars?: number; + /** Invoked when the cursor enters an exemplar marker, with its pixel coords. */ + onExemplarHover?: (exemplar: Exemplar, cx: number, cy: number) => void; + /** Invoked when the cursor leaves an exemplar marker. */ + onExemplarHoverEnd?: () => void; }) { const _id = useId(); const id = _id.replace(/:/g, ''); @@ -610,17 +656,39 @@ export const MemoChart = memo(function MemoChart({ const shouldFitYAxis = fitYAxisToData && displayType !== DisplayType.StackedBar; - // The data min/max is only needed to either zoom into a selection or to - // fit the lower bound to the data. When neither applies, let Recharts - // auto-calculate the upper bound while pinning the lower bound to zero. + // Exemplars plot at the trace's own value, which can exceed the series + // (a slow request above the avg line), so they must influence the upper + // bound or they'd be clipped off-chart. + const exemplarValues = (exemplars ?? []) + .map(e => e.value) + .filter((v): v is number => typeof v === 'number' && !isNaN(v)); + const exemplarMax = exemplarValues.length + ? Math.max(...exemplarValues) + : -Infinity; + + // The data min/max is only needed to either zoom into a selection, fit the + // lower bound to the data, or make room for exemplar markers. When none + // apply, let Recharts auto-calculate the upper bound (lower pinned to zero). if (!hasSelection && !shouldFitYAxis) { - return [0, 'auto']; + if (exemplarMax === -Infinity) return [0, 'auto']; + // Need an explicit upper bound to include exemplars; derive it from the + // visible series max and the exemplar max. + let seriesMax = -Infinity; + graphResults.forEach(dataPoint => { + lineData.forEach(ld => { + const value = dataPoint[ld.dataKey]; + if (typeof value === 'number' && !isNaN(value)) { + seriesMax = Math.max(seriesMax, value); + } + }); + }); + return [0, Math.max(seriesMax, exemplarMax) * 1.05]; } // Calculate domain based on visible series (all series when there's no // explicit selection). let minValue = Infinity; - let maxValue = -Infinity; + let maxValue = exemplarMax; graphResults.forEach(dataPoint => { lineData.forEach(ld => { @@ -653,6 +721,7 @@ export const MemoChart = memo(function MemoChart({ return ['auto', 'auto']; }, [ + exemplars, graphResults, lineData, selectedSeriesNames, @@ -721,6 +790,61 @@ export const MemoChart = memo(function MemoChart({ return map; }, [lineData]); + // Place each exemplar at its own value (the trace/span's actual measurement), + // never remapped onto the series line — the marker's height must match what + // the linked trace reports. Thinned to keep ~maxExemplars markers across the + // visible range: the highest-value (most notable, e.g. slowest) trace per + // window. The window is coarser than the chart granularity so the count stays + // readable even when every fine-grained bucket has an exemplar. + // maxExemplars <= 0 means "unlimited" — show every exemplar (deduped). + const exemplarPoints = useMemo(() => { + type ExemplarPoint = { + x: number; + y: number; + exemplar: Exemplar; + key: string; + }; + if (!exemplars?.length) return [] as ExemplarPoint[]; + + const toPoint = (exemplar: Exemplar, value: number): ExemplarPoint => ({ + x: exemplar.timestamp / 1000, // ms -> seconds (chart x unit) + y: value, + exemplar, + key: `exemplar-${exemplar.traceId}-${exemplar.timestamp}`, + }); + + if (maxExemplars <= 0) { + const all = new Map(); + for (const exemplar of exemplars) { + const value = numOrNull(exemplar.value); + if (value == null) continue; + const p = toPoint(exemplar, value); + all.set(p.key, p); // dedupe identical trace+time + } + return Array.from(all.values()); + } + + const granMs = convertGranularityToSeconds(granularity) * 1000; + const rangeMs = dateRange[1].getTime() - dateRange[0].getTime(); + const bucketMs = Math.max( + granMs || 1, + rangeMs > 0 ? Math.floor(rangeMs / maxExemplars) : granMs || 1, + ); + + const bestPerBucket = new Map(); + for (const exemplar of exemplars) { + const value = numOrNull(exemplar.value); + if (value == null) continue; + const bucket = Math.floor(exemplar.timestamp / bucketMs); + const key = `${exemplar.groupKey ?? ''}@${bucket}`; + const existing = bestPerBucket.get(key); + if (!existing || value > existing.y) { + bestPerBucket.set(key, toPoint(exemplar, value)); + } + } + return Array.from(bestPerBucket.values()); + }, [exemplars, maxExemplars, granularity, dateRange]); + const xAxisDomain: AxisDomain = useMemo(() => { let startTime = toStartOfInterval(dateRange[0], granularity); let endTime = toStartOfInterval(dateRange[1], granularity); @@ -946,6 +1070,21 @@ export const MemoChart = memo(function MemoChart({ /> )} {referenceLines} + {exemplarPoints.map(p => ( + + } + /> + ))} {highlightStart && highlightEnd ? ( ( path: string, @@ -554,6 +566,23 @@ export const prometheusApi = { ...(params.table ? { table: params.table } : {}), }), + queryExemplars: (params: { + query: string; + start: number; + end: number; + connectionId: string; + database?: string; + table?: string; + }): Promise => + prometheusFetch('v1/prometheus/query_exemplars', { + query: params.query, + start: String(params.start), + end: String(params.end), + connectionId: params.connectionId, + ...(params.database ? { database: params.database } : {}), + ...(params.table ? { table: params.table } : {}), + }), + labelValues: (params: { label: string; connectionId: string; diff --git a/packages/app/src/components/ChartEditor/PromqlChartEditor.tsx b/packages/app/src/components/ChartEditor/PromqlChartEditor.tsx index 1b6a7b346d..c5c592a562 100644 --- a/packages/app/src/components/ChartEditor/PromqlChartEditor.tsx +++ b/packages/app/src/components/ChartEditor/PromqlChartEditor.tsx @@ -1,6 +1,6 @@ import { Control, useController, useWatch } from 'react-hook-form'; import { SourceKind } from '@hyperdx/common-utils/dist/types'; -import { Box, Button, Flex, Stack, Text } from '@mantine/core'; +import { Box, Button, Flex, Stack, Switch, Text } from '@mantine/core'; import PromQLEditor from '@/components/PromQLEditor/PromQLEditor'; import { SourceSelectControlled } from '@/components/SourceSelect'; @@ -22,6 +22,10 @@ export default function PromqlChartEditor({ control, name: 'promqlExpression', }); + const { field: exemplarsField } = useController({ + control, + name: 'enableExemplars', + }); const sourceId = useWatch({ control, name: 'source' }); const { data: source } = useSource({ id: sourceId }); @@ -57,7 +61,33 @@ export default function PromqlChartEditor({ metricNames={metricNames} /> - + + + { + exemplarsField.onChange(exemplarsField.value !== true); + onSubmit(); + }} + /> + {exemplarsField.value === true && ( + + + Trace source + + + + )} + + + + + ); +} + function ActiveTimeTooltip({ activeClickPayload, buildSearchUrl, @@ -367,6 +498,69 @@ function DBTimeChartComponent({ id: sourceId || config.source, }); + // Exemplar overlay is configured per-chart via `enableExemplars` (set in the + // chart editor next to "As Ratio"), not a runtime toolbar toggle. The hook is + // a no-op unless the flag is set and the source kind supports exemplars. + const { exemplars } = useExemplars(queriedConfig, source); + + // Trace source an exemplar resolves against: the chart's explicit + // `exemplarTraceSourceId`, else the chart source's linked trace source. + const exemplarTraceSourceId = + queriedConfig.exemplarTraceSourceId || + (source as { traceSourceId?: string } | undefined)?.traceSourceId; + const { data: exemplarTraceSource } = useSource({ + id: exemplarTraceSourceId, + }); + + // Hover card state. A short close delay lets the cursor travel from the SVG + // marker into the HTML card without it closing. + const [hoveredExemplar, setHoveredExemplar] = useState<{ + exemplar: Exemplar; + x: number; + y: number; + } | null>(null); + const exemplarCloseTimer = useRef | null>(null); + const openExemplarCard = useCallback( + (exemplar: Exemplar, x: number, y: number) => { + if (exemplarCloseTimer.current) clearTimeout(exemplarCloseTimer.current); + setHoveredExemplar({ exemplar, x, y }); + }, + [], + ); + const scheduleCloseExemplarCard = useCallback(() => { + if (exemplarCloseTimer.current) clearTimeout(exemplarCloseTimer.current); + exemplarCloseTimer.current = setTimeout( + () => setHoveredExemplar(null), + 150, + ); + }, []); + useEffect( + () => () => { + if (exemplarCloseTimer.current) clearTimeout(exemplarCloseTimer.current); + }, + [], + ); + + const { data: hoveredTraceMeta, isLoading: isHoveredTraceMetaLoading } = + useExemplarTraceMeta( + hoveredExemplar?.exemplar.traceId, + exemplarTraceSource, + ); + + const navigateToExemplarTrace = useCallback( + (exemplar: Exemplar) => { + if (exemplarTraceSourceId) { + const params = new URLSearchParams(); + params.set('source', exemplarTraceSourceId); + params.set('traceId', exemplar.traceId); + Router.push(`/search?${params.toString()}`); + } else { + Router.push(`/trace/${encodeURIComponent(exemplar.traceId)}`); + } + }, + [exemplarTraceSourceId], + ); + const { formatByColumn, chartFormat: axisNumberFormat } = useChartNumberFormats(queriedConfig, data?.meta); @@ -720,6 +914,18 @@ function DBTimeChartComponent({ buildSearchUrl={buildSearchUrl} onDismiss={() => setActiveClickPayload(undefined)} /> + { + if (exemplarCloseTimer.current) + clearTimeout(exemplarCloseTimer.current); + }} + onMouseLeave={scheduleCloseExemplarCard} + /> )} diff --git a/packages/app/src/components/TeamSettings/TeamQueryConfigSection.tsx b/packages/app/src/components/TeamSettings/TeamQueryConfigSection.tsx index de850d2d6f..5d3af8715d 100644 --- a/packages/app/src/components/TeamSettings/TeamQueryConfigSection.tsx +++ b/packages/app/src/components/TeamSettings/TeamQueryConfigSection.tsx @@ -22,6 +22,7 @@ import SelectControlled from '@/components/SelectControlled'; import { DEFAULT_FILTER_KEYS_FETCH_LIMIT, DEFAULT_FILTER_KEYS_FETCH_LIMIT_WITH_MVS, + DEFAULT_MAX_EXEMPLARS, DEFAULT_QUERY_TIMEOUT, DEFAULT_SEARCH_ROW_LIMIT, } from '@/defaults'; @@ -345,6 +346,26 @@ export default function TeamQueryConfigSection() { /> + + + Chart Settings + + + + + + + ); } diff --git a/packages/app/src/components/__tests__/DBTimeChart.test.tsx b/packages/app/src/components/__tests__/DBTimeChart.test.tsx index 6d851cffe2..beaa9b7b95 100644 --- a/packages/app/src/components/__tests__/DBTimeChart.test.tsx +++ b/packages/app/src/components/__tests__/DBTimeChart.test.tsx @@ -35,6 +35,15 @@ jest.mock('@/source', () => ({ .mockReturnValue({ formatByColumn: new Map(), chartFormat: undefined }), })); +jest.mock('@/hooks/useExemplars', () => ({ + useExemplars: jest + .fn() + .mockReturnValue({ exemplars: [], isLoading: false, isError: false }), + useExemplarTraceMeta: jest + .fn() + .mockReturnValue({ data: null, isLoading: false }), +})); + jest.mock('../MaterializedViews/MVOptimizationIndicator', () => jest.fn(() => null), ); diff --git a/packages/app/src/defaults.ts b/packages/app/src/defaults.ts index c3537c5911..57f94e7139 100644 --- a/packages/app/src/defaults.ts +++ b/packages/app/src/defaults.ts @@ -6,6 +6,8 @@ export const DEFAULT_QUERY_TIMEOUT = 60; // max_execution_time, seconds export const DEFAULT_FILTER_KEYS_FETCH_LIMIT = 20; export const DEFAULT_FILTER_KEYS_FETCH_LIMIT_WITH_MVS = 100; export const DEFAULT_SERIES_LIMIT = 100; +// Target number of exemplar markers shown per chart (0 = unlimited). +export const DEFAULT_MAX_EXEMPLARS = 12; export function searchChartConfigDefaults( team: any | undefined | null, diff --git a/packages/app/src/hooks/__tests__/useExemplars.test.ts b/packages/app/src/hooks/__tests__/useExemplars.test.ts new file mode 100644 index 0000000000..7a0ac340cc --- /dev/null +++ b/packages/app/src/hooks/__tests__/useExemplars.test.ts @@ -0,0 +1,61 @@ +import { normalizePrometheusExemplars } from '@/hooks/useExemplars'; + +describe('normalizePrometheusExemplars', () => { + it('returns [] for undefined/empty input', () => { + expect(normalizePrometheusExemplars(undefined)).toEqual([]); + expect(normalizePrometheusExemplars([])).toEqual([]); + }); + + it('maps trace/span ids, value, and seconds→ms timestamp', () => { + const result = normalizePrometheusExemplars([ + { + seriesLabels: { __name__: 'http_latency', service: 'api' }, + exemplars: [ + { + labels: { trace_id: 'abc', span_id: 'def' }, + value: '1.5', + timestamp: 1700000000, + }, + ], + }, + ]); + expect(result).toEqual([ + { + timestamp: 1700000000 * 1000, + value: 1.5, + traceId: 'abc', + spanId: 'def', + groupKey: 'service="api"', + }, + ]); + }); + + it('accepts alternate label spellings (traceID/spanID)', () => { + const [ex] = normalizePrometheusExemplars([ + { + seriesLabels: {}, + exemplars: [ + { + labels: { traceID: 'xyz', spanID: 's1' }, + value: '2', + timestamp: 1, + }, + ], + }, + ]); + expect(ex.traceId).toBe('xyz'); + expect(ex.spanId).toBe('s1'); + expect(ex.groupKey).toBeUndefined(); + }); + + it('skips exemplars without a trace id', () => { + expect( + normalizePrometheusExemplars([ + { + seriesLabels: {}, + exemplars: [{ labels: { foo: 'bar' }, value: '1', timestamp: 1 }], + }, + ]), + ).toEqual([]); + }); +}); diff --git a/packages/app/src/hooks/useExemplars.tsx b/packages/app/src/hooks/useExemplars.tsx new file mode 100644 index 0000000000..bf5260f198 --- /dev/null +++ b/packages/app/src/hooks/useExemplars.tsx @@ -0,0 +1,213 @@ +import { renderMetricExemplarsChartConfig } from '@hyperdx/common-utils/dist/core/renderChartConfig'; +import { isPromqlChartConfig } from '@hyperdx/common-utils/dist/guards'; +import { + ChartConfigWithOptDateRange, + Exemplar, + SourceKind, + TSource, +} from '@hyperdx/common-utils/dist/types'; +import { useQuery } from '@tanstack/react-query'; + +import { prometheusApi } from '@/api'; +import { useClickhouseClient } from '@/clickhouse'; +import { useMetadataWithSettings } from '@/hooks/useMetadata'; +import { getDurationMsExpression } from '@/source'; + +// Source kinds that can produce exemplars today: native metric/promql sources. +// Trace-generated exemplars are added in a follow-up. +const EXEMPLAR_SUPPORTED_KINDS: SourceKind[] = [ + SourceKind.Metric, + SourceKind.Promql, +]; + +// Native Prometheus exporters disagree on the trace/span id label name; accept +// the common spellings. +const TRACE_ID_LABELS = ['trace_id', 'traceID', 'traceId', 'trace.id']; +const SPAN_ID_LABELS = ['span_id', 'spanID', 'spanId', 'span.id']; + +function pick(labels: Record, keys: string[]) { + for (const k of keys) { + if (labels[k]) return labels[k]; + } + return undefined; +} + +/** + * Normalize a native Prometheus /query_exemplars response into the shared + * Exemplar shape. Exported for testing — label naming varies by exporter. + */ +export function normalizePrometheusExemplars( + data: + | { + seriesLabels: Record; + exemplars: { + labels: Record; + value: string; + timestamp: number; + }[]; + }[] + | undefined, +): Exemplar[] { + if (!data) return []; + const out: Exemplar[] = []; + for (const series of data) { + const seriesLabels = series.seriesLabels ?? {}; + const groupKey = + Object.entries(seriesLabels) + .filter(([k]) => k !== '__name__') + .map(([k, v]) => `${k}="${v}"`) + .join(', ') || undefined; + for (const ex of series.exemplars ?? []) { + const traceId = pick(ex.labels ?? {}, TRACE_ID_LABELS); + if (!traceId) continue; + out.push({ + timestamp: ex.timestamp * 1000, // prometheus exemplar ts is unix seconds + value: Number(ex.value), + traceId, + spanId: pick(ex.labels ?? {}, SPAN_ID_LABELS), + groupKey, + }); + } + } + return out; +} + +/** Map raw ClickHouse exemplar rows (renderMetricExemplarsChartConfig) → Exemplar[]. */ +function mapClickhouseExemplars(rows: Record[]): Exemplar[] { + return rows + .filter(r => r.traceId) + .map(r => ({ + timestamp: Number(r.timestamp), + value: Number(r.value), + traceId: String(r.traceId), + spanId: r.spanId ? String(r.spanId) : undefined, + })); +} + +/** + * Fetches exemplars for a chart in parallel with the main series query. A no-op + * (disabled query) unless `config.enableExemplars` is set and the source kind + * supports exemplars, so it adds zero cost to charts that don't use the overlay. + */ +export function useExemplars( + config: ChartConfigWithOptDateRange, + source: TSource | undefined, +) { + const clickhouseClient = useClickhouseClient(); + const metadata = useMetadataWithSettings(); + + const supported = !!source && EXEMPLAR_SUPPORTED_KINDS.includes(source.kind); + const enabled = config.enableExemplars === true && supported; + + const query = useQuery({ + queryKey: ['exemplars', config], + queryFn: async context => { + // PromQL → native Prometheus exemplars via the API proxy. + if (isPromqlChartConfig(config) && config.dateRange) { + const [startDate, endDate] = config.dateRange; + const resp = await prometheusApi.queryExemplars({ + query: config.promqlExpression, + start: startDate.getTime() / 1000, + end: endDate.getTime() / 1000, + connectionId: config.connection, + database: config.from?.databaseName, + table: config.from?.tableName, + }); + if (resp.status !== 'success') { + throw new Error(resp.error ?? 'query_exemplars failed'); + } + return normalizePrometheusExemplars(resp.data); + } + + // Structured metric source → exemplars stored on the OTel metric table. + const exemplarSql = await renderMetricExemplarsChartConfig( + config, + metadata, + ); + if (!exemplarSql) return []; + + const resp = await clickhouseClient.query({ + query: exemplarSql.sql, + query_params: exemplarSql.params, + format: 'JSON', + abort_signal: context.signal, + connectionId: config.connection, + }); + const json = await resp.json>(); + return mapClickhouseExemplars(json.data ?? []); + }, + enabled, + retry: 1, + refetchOnWindowFocus: false, + }); + + return { + exemplars: enabled ? (query.data ?? []) : [], + isLoading: query.isLoading, + isError: query.isError, + }; +} + +export type ExemplarTraceMeta = { + service?: string; + spanName?: string; + statusCode?: string; + durationMs?: number; + timestamp?: string; +}; + +/** + * Fetches a one-row summary of a trace (root/first span) from the given trace + * source, for the exemplar hover card. Enabled only while a trace id is hovered + * and a trace source is configured. + */ +export function useExemplarTraceMeta( + traceId: string | undefined, + traceSource: TSource | undefined, +) { + const clickhouseClient = useClickhouseClient(); + const isTrace = !!traceSource && traceSource.kind === SourceKind.Trace; + + return useQuery({ + queryKey: ['exemplarTraceMeta', traceId, traceSource?.id], + enabled: !!traceId && isTrace, + staleTime: 5 * 60 * 1000, + queryFn: async context => { + if (!traceId || !traceSource || traceSource.kind !== SourceKind.Trace) { + return null; + } + const s = traceSource; + const from = s.from.databaseName + ? `\`${s.from.databaseName}\`.\`${s.from.tableName}\`` + : `\`${s.from.tableName}\``; + const traceIdExpr = s.traceIdExpression || 'TraceId'; + const parentExpr = s.parentSpanIdExpression || 'ParentSpanId'; + const tsExpr = s.timestampValueExpression || 'Timestamp'; + const sql = ` + SELECT + ${s.serviceNameExpression || 'ServiceName'} AS service, + ${s.spanNameExpression || 'SpanName'} AS spanName, + ${s.statusCodeExpression || 'StatusCode'} AS statusCode, + ${getDurationMsExpression(s)} AS durationMs, + ${tsExpr} AS timestamp + FROM ${from} + WHERE ${traceIdExpr} = {traceId:String} + ORDER BY (${parentExpr} = '') DESC, ${tsExpr} ASC + LIMIT 1`; + const resp = await clickhouseClient.query({ + query: sql, + query_params: { traceId }, + format: 'JSON', + abort_signal: context.signal, + connectionId: s.connection, + }); + const json = await resp.json(); + const row = json.data?.[0]; + if (!row) return null; + return { + ...row, + durationMs: row.durationMs != null ? Number(row.durationMs) : undefined, + }; + }, + }); +} diff --git a/packages/common-utils/src/__tests__/renderChartConfig.test.ts b/packages/common-utils/src/__tests__/renderChartConfig.test.ts index 9159687546..58cad9f8d5 100644 --- a/packages/common-utils/src/__tests__/renderChartConfig.test.ts +++ b/packages/common-utils/src/__tests__/renderChartConfig.test.ts @@ -2,7 +2,9 @@ import { chSql, ColumnMeta, parameterizedQueryToSql } from '@/clickhouse'; import { Metadata } from '@/core/metadata'; import { ChartConfigWithOptDateRangeEx, + EXEMPLAR_QUERY_LIMIT, renderChartConfig, + renderMetricExemplarsChartConfig, timeFilterExpr, } from '@/core/renderChartConfig'; import { @@ -3436,3 +3438,116 @@ describe('renderChartConfig', () => { // gauge / sum / histogram snapshots earlier in this file plus the // cross-scope integration test in packages/api/src/clickhouse/__tests__. }); + +describe('renderMetricExemplarsChartConfig', () => { + let mockMetadata: jest.Mocked; + + beforeAll(() => { + jest.spyOn(console, 'warn').mockImplementation(() => {}); + jest.spyOn(console, 'error').mockImplementation(() => {}); + }); + afterAll(() => { + jest.restoreAllMocks(); + }); + + beforeEach(() => { + mockMetadata = { + getColumns: jest.fn().mockResolvedValue([]), + getMaterializedColumnsLookupTable: jest.fn().mockResolvedValue(null), + getColumn: jest.fn().mockResolvedValue(undefined), + getTableMetadata: jest + .fn() + .mockResolvedValue({ primary_key: 'TimeUnix' }), + getSkipIndices: jest.fn().mockResolvedValue([]), + getSetting: jest.fn().mockResolvedValue(undefined), + isClickHouseCloud: jest.fn().mockResolvedValue(false), + } as unknown as jest.Mocked; + }); + + const histogramConfig: ChartConfigWithOptDateRange = { + displayType: DisplayType.Line, + connection: 'test-connection', + metricTables: { + gauge: 'otel_metrics_gauge', + histogram: 'otel_metrics_histogram', + sum: 'otel_metrics_sum', + summary: 'otel_metrics_summary', + 'exponential histogram': 'otel_metrics_exponential_histogram', + }, + from: { databaseName: 'default', tableName: '' }, + select: [ + { + aggFn: 'quantile', + aggCondition: '', + aggConditionLanguage: 'lucene', + valueExpression: 'Value', + level: 0.95, + metricName: 'http.server.duration', + metricType: MetricsDataType.Histogram, + }, + ], + where: '', + whereLanguage: 'lucene', + timestampValueExpression: 'TimeUnix', + dateRange: [new Date('2025-02-12'), new Date('2025-02-14')], + granularity: '1 minute', + }; + + it('builds an ARRAY JOIN exemplar query against the metric-type table', async () => { + const generated = await renderMetricExemplarsChartConfig( + histogramConfig, + mockMetadata, + ); + expect(generated).not.toBeNull(); + const sql = parameterizedQueryToSql(generated!); + + // Surfaces the exemplar columns + expect(sql).toContain('ARRAY JOIN'); + expect(sql).toContain('`Exemplars.TraceId` AS ex_TraceId'); + expect(sql).toContain('toUnixTimestamp64Milli(ex_TimeUnix)'); + // Points at the histogram table and filters by metric name + time + expect(sql).toContain('otel_metrics_histogram'); + expect(sql).toContain("MetricName = 'http.server.duration'"); + expect(sql).toContain('TimeUnix'); + // Drops empty trace ids and caps the result set + expect(sql).toContain('notEmpty(ex_TraceId)'); + expect(sql).toContain(`LIMIT ${EXEMPLAR_QUERY_LIMIT}`); + }); + + it('returns null for a ratio config (exemplars are meaningless on a ratio axis)', async () => { + const ratioConfig = { + ...histogramConfig, + seriesReturnType: 'ratio', + select: [histogramConfig.select[0], histogramConfig.select[0]], + } as ChartConfigWithOptDateRange; + expect( + await renderMetricExemplarsChartConfig(ratioConfig, mockMetadata), + ).toBeNull(); + }); + + it('returns null for a multi-series config', async () => { + const multiConfig = { + ...histogramConfig, + select: [histogramConfig.select[0], histogramConfig.select[0]], + } as ChartConfigWithOptDateRange; + expect( + await renderMetricExemplarsChartConfig(multiConfig, mockMetadata), + ).toBeNull(); + }); + + it('returns null for a non-metric config', async () => { + const logConfig: ChartConfigWithOptDateRange = { + displayType: DisplayType.Line, + connection: 'test-connection', + from: { databaseName: 'default', tableName: 'otel_logs' }, + select: [{ aggFn: 'count', valueExpression: '' }], + where: '', + timestampValueExpression: 'Timestamp', + dateRange: [new Date('2025-02-12'), new Date('2025-02-14')], + granularity: '1 minute', + }; + expect( + await renderMetricExemplarsChartConfig(logConfig, mockMetadata), + ).toBeNull(); + }); +}); diff --git a/packages/common-utils/src/core/renderChartConfig.ts b/packages/common-utils/src/core/renderChartConfig.ts index 4f5c8a48d5..2245aec457 100644 --- a/packages/common-utils/src/core/renderChartConfig.ts +++ b/packages/common-utils/src/core/renderChartConfig.ts @@ -2178,6 +2178,94 @@ export async function renderChartConfig( ]); } +/** Overall cap on exemplar markers returned for a single chart, so a wide + * time range can't flood the chart overlay with thousands of points. */ +export const EXEMPLAR_QUERY_LIMIT = 200; + +/** + * Builds a ClickHouse query that surfaces native exemplars stored on an OTel + * metric table (`Exemplars.TraceId/SpanId/Value/TimeUnix`). Returns null when + * the config is not a single-metric chart we can resolve a table for. + * + * Reuses `renderWhere` so the exemplar scan honors the exact same time range, + * metric-name, and user filters as the rendered series. Exemplars are kept as + * their own raw points (the marker sits at the exemplar's own value/time), not + * bucketed — so no `timeBucketExpr` here. + */ +export async function renderMetricExemplarsChartConfig( + chartConfig: ChartConfigWithOptDateRangeEx, + metadata: Metadata, +): Promise { + if ( + isRawSqlChartConfig(chartConfig) || + isPromqlChartConfig(chartConfig) || + !isMetricChartConfig(chartConfig) || + !Array.isArray(chartConfig.select) || + // Exemplars carry a single series' raw measurement (e.g. latency). They are + // meaningless on a ratio axis and ambiguous across multiple series, so only + // surface them for a single, non-ratio metric series. + chartConfig.select.length !== 1 || + chartConfig.seriesReturnType === 'ratio' + ) { + return null; + } + const { metricTables, select } = chartConfig; + const { metricType, metricName, metricNameSql } = select[0] ?? {}; + const table = + metricType && metricTables ? metricTables[metricType] : undefined; + if (!metricType || !metricName || !table) { + return null; + } + // Keep exemplars to latency metrics for now: a histogram's exemplar value is a + // request duration, which shares the chart's y-axis unit. Other metric types + // (counts/gauges/rates) put exemplars on an incompatible scale. + if (metricType !== MetricsDataType.Histogram) { + return null; + } + + // Build a config that points at the concrete metric-type table and carries + // the metric-name predicate alongside the user filters, then let renderWhere + // assemble the time filter + filters exactly as the main query does. The + // guards above narrow chartConfig to the metric builder config, so no cast. + const whereConfig: BuilderChartConfigWithOptDateRangeEx = { + ...chartConfig, + from: { ...chartConfig.from, tableName: table }, + timestampValueExpression: + chartConfig.timestampValueExpression || DEFAULT_METRIC_TABLE_TIME_COLUMN, + // Drop the metric select so renderWhere doesn't push aggCondition predicates. + select: [], + filters: [ + ...(chartConfig.filters ?? []), + { + type: 'sql', + condition: createMetricNameFilter(metricName, metricNameSql), + }, + ], + }; + + const where = await renderWhere(whereConfig, metadata); + const from = renderFrom({ from: whereConfig.from }); + + return concatChSql(' ', [ + chSql`SELECT + toUnixTimestamp64Milli(ex_TimeUnix) AS timestamp, + ex_Value AS value, + ex_TraceId AS traceId, + ex_SpanId AS spanId`, + chSql`FROM ${from}`, + chSql`ARRAY JOIN + \`Exemplars.TimeUnix\` AS ex_TimeUnix, + \`Exemplars.Value\` AS ex_Value, + \`Exemplars.TraceId\` AS ex_TraceId, + \`Exemplars.SpanId\` AS ex_SpanId`, + chSql`WHERE ${where.sql ? where : chSql`1 = 1`} AND notEmpty(ex_TraceId)`, + // Native exemplars carry no interestingness signal; keep the highest-value + // ones as a stable cap. ponytail: value-desc cap, revisit if even sampling + // across buckets is wanted. + chSql`ORDER BY value DESC LIMIT ${{ Int32: EXEMPLAR_QUERY_LIMIT }}`, + ]); +} + // EditForm -> translateToQueriedChartConfig -> QueriedChartConfig // renderFn(QueriedChartConfig) -> sql // query(sql) -> data diff --git a/packages/common-utils/src/types.ts b/packages/common-utils/src/types.ts index f7ea805eda..5ab84b6a0a 100644 --- a/packages/common-utils/src/types.ts +++ b/packages/common-utils/src/types.ts @@ -1185,6 +1185,15 @@ const SharedChartSettingsSchema = z.object({ // number tiles have no time dimension to bucket). Other display types // ignore the field. Kept at shared level mirroring `color` / `colorRules`. backgroundChart: BackgroundChartSchema.optional(), + // Opt-in: overlay exemplar markers (individual traces linked to the series) + // on time charts. Sourced natively for metric/PromQL sources and generated + // per time bucket for trace sources. The UI gates the toggle on supported + // source kinds; off by default so the extra exemplar query never runs. + enableExemplars: z.boolean().optional(), + // Trace source that an exemplar's trace id resolves against — used to fetch + // hover metadata and to deep-link straight to the trace. When unset, the + // chart source's linked `traceSourceId` is used as a fallback. + exemplarTraceSourceId: z.string().optional(), }); export const _ChartConfigSchema = SharedChartSettingsSchema.extend({ @@ -1301,6 +1310,24 @@ export const ChartConfigSchema = z.union([ export type ChartConfig = z.infer; +/** + * A single exemplar: an individual data point overlaid on a time chart that + * links back to a trace. Shared shape for both backends — metric/PromQL + * sources surface native exemplars, trace sources generate them per bucket. + */ +export const ExemplarSchema = z.object({ + timestamp: z.number(), // epoch ms, x-position on the chart + value: z.number(), // exemplar's own value (metric Value / trace Duration in ms) + traceId: z.string(), + spanId: z.string().optional(), + // Matches the series the exemplar belongs to (LineData.displayName) so the + // overlay can attach markers to the right line. Undefined => applies to all. + groupKey: z.string().optional(), + attributes: z.record(z.string()).optional(), +}); + +export type Exemplar = z.infer; + export type DateRange = { dateRange: [Date, Date]; dateRangeStartInclusive?: boolean; // default true @@ -1604,6 +1631,9 @@ export const TeamClickHouseSettingsSchema = z.object({ metadataMaxRowsToRead: z.number().optional(), parallelizeWhenPossible: z.boolean().optional(), filterKeysFetchLimit: z.number().optional(), + // Target number of exemplar markers shown per chart (0 = unlimited). Not a + // ClickHouse setting, but lives in the same team-config bag for reuse. + maxExemplars: z.number().optional(), }); /** Accepts null to unset (reset to default) a setting. */ @@ -1614,6 +1644,7 @@ export const TeamClickHouseSettingsUpdateSchema = z.object({ metadataMaxRowsToRead: z.number().nullish(), parallelizeWhenPossible: z.boolean().nullish(), filterKeysFetchLimit: z.number().nullish(), + maxExemplars: z.number().nullish(), }); export type TeamClickHouseSettingsUpdate = z.infer< typeof TeamClickHouseSettingsUpdateSchema From c38448a3d0afa416927157359b2ab4a7a8c38ca4 Mon Sep 17 00:00:00 2001 From: Jordan Simonovski Date: Mon, 29 Jun 2026 15:49:48 +1000 Subject: [PATCH 02/10] feat: Created a telemetry generator for local dev Covers more complex metric/trace use cases. --- telemetry-generator/Dockerfile | 11 + telemetry-generator/README.md | 62 ++++ telemetry-generator/package.json | 19 ++ telemetry-generator/src/index.js | 508 +++++++++++++++++++++++++++++++ 4 files changed, 600 insertions(+) create mode 100644 telemetry-generator/Dockerfile create mode 100644 telemetry-generator/README.md create mode 100644 telemetry-generator/package.json create mode 100644 telemetry-generator/src/index.js diff --git a/telemetry-generator/Dockerfile b/telemetry-generator/Dockerfile new file mode 100644 index 0000000000..f700c816e9 --- /dev/null +++ b/telemetry-generator/Dockerfile @@ -0,0 +1,11 @@ +FROM node:22-alpine + +WORKDIR /app + +# Install deps first for layer caching. +COPY package.json ./ +RUN npm install --omit=dev --no-package-lock + +COPY src ./src + +CMD ["node", "src/index.js"] diff --git a/telemetry-generator/README.md b/telemetry-generator/README.md new file mode 100644 index 0000000000..679ce52459 --- /dev/null +++ b/telemetry-generator/README.md @@ -0,0 +1,62 @@ +# telemetry-generator + +Synthetic telemetry for local dev and e2e testing of HyperDX. Emits **realistic +OTLP traces** and **coherent metric exemplars** so exemplar features (and +anything else needing good test data) can be built and validated against +data that actually agrees with itself. + +## What it produces + +- **Traces** (via OTLP gRPC → the dev collector → ClickHouse): six services + (`api-gateway`, `order-service`, `user-service`, `search-service`, + `payment-service`, `notification-service`), weighted attribute pools + (`http.route`, `http.method`, `host.region`, `app.tenant_id`, `app.build_id`, + `app.platform`, `app.feature_flag`, `k8s.pod.name`, `user.id`), nested + db/cache/downstream spans, and several failure scenarios (slow checkout, + iOS order errors, Redis timeout, Elasticsearch timeout, auth memory leak, + payment timeout, compliance overhead) with gaussian latency distributions. +- **Metrics, fully via OTLP**: the generator emits *only* traces. The + collector's **`spanmetrics` connector** derives `calls` (sum) and `duration` + (histogram, ms) from those spans, **with exemplars enabled** — so the + histogram data points carry `trace_id`/`span_id` pointing back at the exact + spans they were measured from. The ClickHouse exporter writes them to + `otel_metrics_histogram` / `otel_metrics_sum` with `Exemplars.*` populated. + Nothing is written to ClickHouse directly; everything flows through the real + pipeline. + +Backfills `GEN_BACKFILL_MINUTES` of history on startup, then emits live at +`GEN_RATE_PER_SEC`. + +## How the metrics get exemplars + +The `spanmetrics` connector is bundled into the collector +(`packages/otel-collector/builder-config.yaml`) and wired up in +`packages/api/src/opamp/controllers/opampController.ts`, gated on the +`ENABLE_SPAN_METRICS` env flag (on in dev via `packages/api/.env.development`, +off by default so production behavior is unchanged). It reads span attributes as +metric dimensions (`http.route`, `http.method`, `host.region`, `app.tenant_id`, +`http.status_code`) and emits a duration histogram with `exemplars.enabled`. + +Chart the **`traces.span.metrics.duration`** metric (histogram) on a Metric +source and the exemplar overlay resolves against the trace source — coherent by +construction because the metric *is* derived from the traces. (`span_metrics` +also emits `traces.span.metrics.calls`, a request-count sum.) + +## Config (env) + +| Var | Default | Meaning | +|-----|---------|---------| +| `OTEL_EXPORTER_OTLP_ENDPOINT` | `http://otel-collector:4318` | OTLP HTTP base (`/v1/traces` appended) | +| `GEN_OTLP_API_KEY` | `super-secure-ingestion-api-key` | ingest token (HyperDX `bearertokenauth`, raw scheme) | +| `GEN_BACKFILL_MINUTES` | `30` | history to backfill on startup | +| `GEN_RATE_PER_SEC` | `20` | live request rate | + +## Run + +Runs automatically with `yarn dev` (see the `telemetry-generator` service in +`docker-compose.dev.yml`). Standalone: + +```bash +cd telemetry-generator && npm install +OTEL_EXPORTER_OTLP_ENDPOINT=http://localhost:30996 npm start +``` diff --git a/telemetry-generator/package.json b/telemetry-generator/package.json new file mode 100644 index 0000000000..18a83158af --- /dev/null +++ b/telemetry-generator/package.json @@ -0,0 +1,19 @@ +{ + "name": "telemetry-generator", + "version": "0.1.0", + "private": true, + "description": "Synthetic OTLP traces + coherent metric exemplars for local dev and e2e testing of HyperDX.", + "type": "commonjs", + "main": "src/index.js", + "scripts": { + "start": "node src/index.js" + }, + "dependencies": { + "@opentelemetry/api": "^1.9.0", + "@opentelemetry/exporter-trace-otlp-proto": "^0.57.2", + "@opentelemetry/resources": "^1.30.1", + "@opentelemetry/sdk-trace-base": "^1.30.1", + "@opentelemetry/sdk-trace-node": "^1.30.1", + "@opentelemetry/semantic-conventions": "^1.30.0" + } +} diff --git a/telemetry-generator/src/index.js b/telemetry-generator/src/index.js new file mode 100644 index 0000000000..de6264fa33 --- /dev/null +++ b/telemetry-generator/src/index.js @@ -0,0 +1,508 @@ +/* + * Telemetry generator — synthetic OTLP traces + coherent metric exemplars. + * + * Why this exists: hand-seeding ClickHouse produces incoherent data (metric + * values, exemplar values, and the traces they point at don't agree). This + * service emits realistic traces over OTLP (diverse attributes, failure + * scenarios, nested service spans, backfill + live) and, for each request, + * writes ONE metric data point whose value AND exemplar are derived from that + * exact trace — so the metric line, the exemplar markers, the hover metadata, + * and the linked trace all agree by construction. + * + * Traces go through the real OTLP pipeline (collector -> ClickHouse). Metric + * exemplars are written to ClickHouse directly because OTel JS (1.30) does not + * yet emit metric exemplars and our collector has no spanmetrics connector. + * See README for the spanmetrics upgrade path. + */ +'use strict'; + +const { + trace, + ROOT_CONTEXT, + SpanStatusCode, + SpanKind, + diag, + DiagConsoleLogger, + DiagLogLevel, +} = require('@opentelemetry/api'); +// Surface OTLP export failures (auth, connection) instead of dropping silently. +diag.setLogger(new DiagConsoleLogger(), DiagLogLevel.ERROR); +const { NodeTracerProvider } = require('@opentelemetry/sdk-trace-node'); +const { BatchSpanProcessor } = require('@opentelemetry/sdk-trace-base'); +const { + OTLPTraceExporter, +} = require('@opentelemetry/exporter-trace-otlp-proto'); +const { Resource } = require('@opentelemetry/resources'); +const { ATTR_SERVICE_NAME } = require('@opentelemetry/semantic-conventions'); +const net = require('net'); + +// ── Config ────────────────────────────────────────────────────────── +const cfg = { + // OTLP HTTP base (port 4318). Traces are POSTed to /v1/traces. + otlpEndpoint: + process.env.OTEL_EXPORTER_OTLP_ENDPOINT || 'http://otel-collector:4318', + backfillMinutes: Number(process.env.GEN_BACKFILL_MINUTES || 30), + ratePerSec: Number(process.env.GEN_RATE_PER_SEC || 20), + // HyperDX collectors enforce bearer-token auth on OTLP ingest. Default to the + // dev INGESTION_API_KEY (packages/api/.env.development); override per env. + otlpApiKey: + process.env.GEN_OTLP_API_KEY || 'super-secure-ingestion-api-key', +}; + +// ── Weighted / uniform random helpers ─────────────────────────────── +function pickWeighted(choices) { + const total = choices.reduce((s, c) => s + c[1], 0); + let r = Math.random() * total; + for (const [value, weight] of choices) { + r -= weight; + if (r <= 0) return value; + } + return choices[choices.length - 1][0]; +} +function pickUniform(arr) { + return arr[Math.floor(Math.random() * arr.length)]; +} +// Box-Muller gaussian, clamped to >= 1ms +function gaussianMs(mean, stddev) { + let u = 0; + let v = 0; + while (u === 0) u = Math.random(); + while (v === 0) v = Math.random(); + const z = Math.sqrt(-2 * Math.log(u)) * Math.cos(2 * Math.PI * v); + return Math.max(1, mean + z * stddev); +} + +// ── Attribute pools (cardinality + diversity) ─────────────────────── +const routes = [ + ['/api/users', 25], + ['/api/orders', 15], + ['/cart/checkout', 10], + ['/api/search', 25], + ['/api/products', 15], + ['/api/auth', 10], +]; +const methods = [ + ['GET', 60], + ['POST', 25], + ['PUT', 10], + ['DELETE', 5], +]; +const regions = [ + ['us-east-1', 40], + ['us-west-2', 30], + ['eu-west-1', 20], + ['ap-southeast-1', 10], +]; +const buildIDs = [ + ['build-7a1', 35], + ['build-7a2', 35], + ['build-7a3', 30], +]; +const platforms = [ + ['web', 50], + ['ios', 30], + ['android', 20], +]; +const featureFlags = [ + ['new-checkout-flow', 15], + ['dark-launch-search', 10], + ['legacy', 75], +]; +const tenants = [ + 'tenant-acme', + 'tenant-globex', + 'tenant-initech', + 'tenant-umbrella', +]; +const pods = Array.from({ length: 8 }, (_, i) => `pod-abc-${i + 1}`); +const userId = () => `user-${String(Math.floor(Math.random() * 500) + 1).padStart(4, '0')}`; + +function routeToService(route) { + switch (route) { + case '/api/orders': + case '/cart/checkout': + return 'order-service'; + case '/api/users': + case '/api/auth': + return 'user-service'; + case '/api/search': + case '/api/products': + return 'search-service'; + default: + return 'unknown-service'; + } +} +function serviceToDb(svc) { + if (svc === 'order-service' || svc === 'user-service') return 'postgres'; + if (svc === 'search-service') return 'elasticsearch'; + return 'none'; +} + +// ── Per-service tracer providers (ServiceName = resource attr) ─────── +const SERVICE_NAMES = [ + 'api-gateway', + 'order-service', + 'user-service', + 'search-service', + 'payment-service', + 'notification-service', +]; +const exporter = new OTLPTraceExporter({ + url: `${cfg.otlpEndpoint.replace(/\/$/, '')}/v1/traces`, + // HyperDX's bearertokenauth uses scheme:'' — send the raw token, no prefix. + headers: cfg.otlpApiKey ? { authorization: cfg.otlpApiKey } : undefined, +}); +const providers = new Map(); +const tracers = new Map(); +for (const name of SERVICE_NAMES) { + const provider = new NodeTracerProvider({ + resource: new Resource({ + [ATTR_SERVICE_NAME]: name, + 'service.version': '1.0.0', + }), + spanProcessors: [ + // Large queue so the backfill burst isn't dropped before export. + new BatchSpanProcessor(exporter, { + maxExportBatchSize: 512, + maxQueueSize: 32768, + scheduledDelayMillis: 500, + }), + ], + }); + providers.set(name, provider); + tracers.set(name, provider.getTracer('telemetry-generator')); +} +const tracer = name => tracers.get(name); + +const jitter = () => Math.random() * 2; // ms + +function commonAttrs(a) { + return { + 'http.method': a.method, + 'http.route': a.route, + 'user.id': a.uid, + 'app.tenant_id': a.tenant, + 'host.region': a.region, + 'app.build_id': a.buildID, + 'app.platform': a.platform, + 'app.feature_flag': a.featureFlag, + 'k8s.pod.name': a.pod, + }; +} +function markError(span, type, message) { + span.setStatus({ code: SpanStatusCode.ERROR, message }); + span.addEvent('exception', { + 'exception.type': type, + 'exception.message': message, + }); +} + +// ── Trace emission ────────────────────────────────────────────────── +// Emits a trace; the collector's spanmetrics connector derives the +// request-duration metric (with exemplars) from these spans. +function emitTrace(tsMs) { + const a = { + route: pickWeighted(routes), + method: pickWeighted(methods), + region: pickWeighted(regions), + buildID: pickWeighted(buildIDs), + platform: pickWeighted(platforms), + featureFlag: pickWeighted(featureFlags), + tenant: pickUniform(tenants), + uid: userId(), + pod: pickUniform(pods), + }; + const svc = routeToService(a.route); + const sc = detectScenario(a); + const plan = SCENARIOS[sc](a, svc); + + const root = tracer('api-gateway').startSpan( + `${a.method} ${a.route}`, + { + kind: SpanKind.SERVER, + startTime: tsMs, + attributes: { ...commonAttrs(a), 'http.status_code': plan.statusCode }, + }, + ROOT_CONTEXT, + ); + if (plan.error) markError(root, plan.error.type, plan.error.message); + else root.setStatus({ code: SpanStatusCode.OK }); + + const rootCtx = trace.setSpan(ROOT_CONTEXT, root); + const svcStart = tsMs + jitter(); + const svcSpan = tracer(svc).startSpan( + `${svc}.handle`, + { kind: SpanKind.INTERNAL, startTime: svcStart, attributes: svcAttrs(svc, a) }, + rootCtx, + ); + if (plan.error) markError(svcSpan, plan.error.type, plan.error.message); + const svcCtx = trace.setSpan(rootCtx, svcSpan); + + // Leaf spans (db / cache / downstream) per the scenario plan. + let cursor = svcStart + jitter(); + for (const leaf of plan.leaves) { + const leafSvc = leaf.service || svc; + const ls = tracer(leafSvc).startSpan( + leaf.name, + { + kind: SpanKind.CLIENT, + startTime: cursor, + attributes: { + [ATTR_SERVICE_NAME]: leafSvc, + 'host.region': a.region, + ...(leaf.attrs || {}), + }, + }, + svcCtx, + ); + if (leaf.error) markError(ls, leaf.error.type, leaf.error.message); + ls.end(cursor + leaf.durMs); + cursor += leaf.durMs + 1; + } + + svcSpan.end(svcStart + plan.svcDurMs); + root.end(tsMs + plan.rootDurMs); +} + +function svcAttrs(svc, a) { + return { + [ATTR_SERVICE_NAME]: svc, + 'http.route': a.route, + 'host.region': a.region, + 'app.build_id': a.buildID, + 'app.feature_flag': a.featureFlag, + 'app.platform': a.platform, + 'user.id': a.uid, + 'app.tenant_id': a.tenant, + 'k8s.pod.name': a.pod, + }; +} + +// ── Scenarios ─────────────────────────────────────────────────────── +// Each returns { rootDurMs, svcDurMs, statusCode, error?, leaves:[{name,durMs,service?,attrs?,error?}] } +const PAYMENT_TIMEOUT_RATE = 0.05; +const AUTH_LEAK_ERROR_RATE = 0.1; + +function detectScenario(a) { + const svc = routeToService(a.route); + if ( + a.route === '/cart/checkout' && + a.region === 'us-west-2' && + Math.random() < PAYMENT_TIMEOUT_RATE + ) + return 'paymentTimeout'; + if ( + a.route === '/cart/checkout' && + a.featureFlag === 'new-checkout-flow' && + a.region === 'eu-west-1' + ) + return 'slowCheckout'; + if (a.route === '/api/orders' && a.platform === 'ios' && a.buildID === 'build-7a3') + return 'iosOrderErrors'; + if ( + a.tenant === 'tenant-initech' && + a.featureFlag === 'dark-launch-search' && + a.route === '/api/search' + ) + return 'initechSearch'; + if ( + a.route === '/api/auth' && + a.buildID === 'build-7a3' && + (a.pod === 'pod-abc-7' || a.pod === 'pod-abc-8') + ) + return 'authMemoryLeak'; + if (a.region === 'ap-southeast-1' && svc === 'user-service') + return 'redisTimeoutApac'; + if (a.tenant === 'tenant-umbrella' && a.region === 'eu-west-1') + return 'umbrellaCompliance'; + return 'normal'; +} + +function normalStatus() { + const r = Math.random(); + if (r < 0.95) return 200; + if (r < 0.98) return 201; + return 404; +} + +const SCENARIOS = { + normal: (a, svc) => { + const db = serviceToDb(svc); + const leaves = []; + if (db === 'postgres') + leaves.push({ + name: 'postgres.query', + durMs: gaussianMs(10, 5), + attrs: { 'db.system': 'postgres', 'db.statement': `SELECT * FROM ${a.route.slice(5)}` }, + }); + else if (db === 'elasticsearch') + leaves.push({ + name: 'elasticsearch.search', + durMs: gaussianMs(10, 5), + attrs: { 'db.system': 'elasticsearch' }, + }); + if (a.route === '/cart/checkout') + leaves.push({ + name: 'payment-service.charge', + service: 'payment-service', + durMs: gaussianMs(10, 4), + }); + return { rootDurMs: gaussianMs(40, 20), svcDurMs: gaussianMs(25, 12), statusCode: normalStatus(), leaves }; + }, + slowCheckout: a => { + const leaves = []; + const n = 3 + Math.floor(Math.random() * 3); + for (let i = 0; i < n; i++) + leaves.push({ + name: 'postgres.query', + durMs: gaussianMs(200, 40), + attrs: { 'db.system': 'postgres', 'db.statement': 'SELECT * FROM orders WHERE id = ?' }, + }); + leaves.push({ name: 'payment-service.charge', service: 'payment-service', durMs: gaussianMs(200, 50) }); + return { rootDurMs: gaussianMs(1500, 400), svcDurMs: gaussianMs(1200, 350), statusCode: 200, leaves }; + }, + iosOrderErrors: () => ({ + rootDurMs: gaussianMs(250, 60), + svcDurMs: gaussianMs(100, 30), + statusCode: 500, + error: { type: 'ValidationError', message: 'malformed request body' }, + leaves: [], + }), + redisTimeoutApac: a => ({ + rootDurMs: gaussianMs(650, 120), + svcDurMs: gaussianMs(580, 100), + statusCode: 200, + leaves: [ + { name: 'redis.get', durMs: gaussianMs(550, 100), attrs: { 'db.system': 'redis', 'db.statement': `GET user:session:${a.uid}` } }, + { name: 'postgres.query', durMs: gaussianMs(30, 10), attrs: { 'db.system': 'postgres' } }, + ], + }), + initechSearch: () => ({ + rootDurMs: gaussianMs(3000, 500), + svcDurMs: gaussianMs(2800, 450), + statusCode: 500, + error: { type: 'TimeoutError', message: 'elasticsearch timeout' }, + leaves: [ + { + name: 'elasticsearch.search', + durMs: gaussianMs(2500, 400), + attrs: { 'db.system': 'elasticsearch' }, + error: { type: 'IOException', message: 'read tcp: i/o timeout' }, + }, + ], + }), + authMemoryLeak: a => { + const err = Math.random() < AUTH_LEAK_ERROR_RATE; + return { + rootDurMs: gaussianMs(800, 200), + svcDurMs: gaussianMs(700, 180), + statusCode: err ? 503 : 200, + error: err ? { type: 'ServiceUnavailableError', message: 'GC overhead' } : undefined, + leaves: [{ name: 'redis.get', durMs: gaussianMs(600, 150), attrs: { 'db.system': 'redis', 'db.statement': `GET auth:token:${a.uid}` } }], + }; + }, + paymentTimeout: () => ({ + rootDurMs: gaussianMs(5000, 500), + svcDurMs: gaussianMs(4800, 450), + statusCode: 504, + error: { type: 'TimeoutError', message: 'gateway timeout' }, + leaves: [ + { name: 'postgres.query', durMs: gaussianMs(20, 8), attrs: { 'db.system': 'postgres' } }, + { + name: 'payment-service.charge', + service: 'payment-service', + durMs: gaussianMs(4500, 300), + error: { type: 'TimeoutError', message: 'context deadline exceeded' }, + }, + ], + }), + umbrellaCompliance: (a, svc) => { + const overhead = gaussianMs(150, 40); + const db = serviceToDb(svc); + const leaves = [{ name: 'compliance.data_residency_check', durMs: overhead, attrs: { 'app.tenant_id': a.tenant } }]; + if (db === 'postgres') leaves.push({ name: 'postgres.query', durMs: gaussianMs(10, 5), attrs: { 'db.system': 'postgres' } }); + else if (db === 'elasticsearch') leaves.push({ name: 'elasticsearch.search', durMs: gaussianMs(10, 5), attrs: { 'db.system': 'elasticsearch' } }); + return { rootDurMs: gaussianMs(40, 20) + overhead, svcDurMs: gaussianMs(40, 20) + overhead, statusCode: normalStatus(), leaves }; + }, +}; + +// The collector's OTLP receiver is configured dynamically via OpAMP, so the +// port isn't open the instant the container starts — it appears only after the +// collector fetches its remote config. Wait for it before backfilling, else the +// initial burst hits ECONNREFUSED and is lost. +function waitForCollector(timeoutMs = 120_000) { + const u = new URL(cfg.otlpEndpoint); + const port = Number(u.port) || (u.protocol === 'https:' ? 443 : 80); + const host = u.hostname; + const deadline = Date.now() + timeoutMs; + const tryOnce = () => + new Promise(resolve => { + const sock = net.connect({ host, port }, () => { + sock.destroy(); + resolve(true); + }); + sock.on('error', () => { + sock.destroy(); + resolve(false); + }); + sock.setTimeout(2000, () => { + sock.destroy(); + resolve(false); + }); + }); + return (async () => { + while (Date.now() < deadline) { + if (await tryOnce()) return true; + await new Promise(r => setTimeout(r, 2000)); + } + return false; + })(); +} + +// ── Main ──────────────────────────────────────────────────────────── +async function main() { + console.log( + `telemetry-generator starting: otlp=${cfg.otlpEndpoint} ` + + `backfill=${cfg.backfillMinutes}m rate=${cfg.ratePerSec}/s`, + ); + + console.log('waiting for collector OTLP endpoint...'); + const ready = await waitForCollector(); + console.log(ready ? 'collector reachable' : 'collector wait timed out, proceeding anyway'); + + // Backfill historical traces. The spanmetrics connector derives the + // request-duration metric (with exemplars) from these spans. + const now = Date.now(); + const backfillStart = now - cfg.backfillMinutes * 60_000; + const backfillCount = cfg.backfillMinutes * 60 * cfg.ratePerSec; + console.log(`backfilling ~${backfillCount} requests over ${cfg.backfillMinutes}m...`); + for (let i = 0; i < backfillCount; i++) { + const ts = backfillStart + Math.random() * (now - backfillStart); + emitTrace(ts); + // Periodically yield so the span exporter can drain its queue instead of + // overflowing (which would drop traces, and thus their exemplars). + if (i % 500 === 0) await new Promise(r => setTimeout(r, 25)); + } + // Let the final span batches export before we declare backfill done. + await new Promise(r => setTimeout(r, 3000)); + console.log('backfill complete, starting live emission...'); + + // Live emission. + const intervalMs = Math.max(1, Math.floor(1000 / cfg.ratePerSec)); + setInterval(() => emitTrace(Date.now()), intervalMs); + + const shutdown = async () => { + console.log('shutting down...'); + await Promise.all( + [...providers.values()].map(p => p.shutdown().catch(() => {})), + ); + process.exit(0); + }; + process.on('SIGINT', shutdown); + process.on('SIGTERM', shutdown); +} + +main().catch(e => { + console.error(e); + process.exit(1); +}); From 62bcb9c28920a1e66c67e2a685011054b3b7815b Mon Sep 17 00:00:00 2001 From: Jordan Simonovski Date: Mon, 29 Jun 2026 15:50:14 +1000 Subject: [PATCH 03/10] feat: configured spanmetric generation for local testing --- packages/api/.env.development | 6 +- .../src/opamp/controllers/opampController.ts | 78 +++++++++++++++++++ packages/otel-collector/builder-config.yaml | 3 + scripts/dev-env.sh | 3 + 4 files changed, 88 insertions(+), 2 deletions(-) diff --git a/packages/api/.env.development b/packages/api/.env.development index 8ba0a6c0c8..a8ec2bd9ca 100644 --- a/packages/api/.env.development +++ b/packages/api/.env.development @@ -20,9 +20,11 @@ REDIS_URL=redis://localhost:6379 USAGE_STATS_ENABLED=false NODE_OPTIONS="--max-http-header-size=131072" ENABLE_SWAGGER=true -DEFAULT_CONNECTIONS=[{"name":"Local ClickHouse","host":"http://localhost:${HDX_DEV_CH_HTTP_PORT}","username":"default","password":""}] -DEFAULT_SOURCES=[{"from":{"databaseName":"default","tableName":"otel_logs"},"kind":"log","timestampValueExpression":"Timestamp","name":"Logs","displayedTimestampValueExpression":"Timestamp","implicitColumnExpression":"Body","serviceNameExpression":"ServiceName","bodyExpression":"Body","eventAttributesExpression":"LogAttributes","resourceAttributesExpression":"ResourceAttributes","defaultTableSelectExpression":"Timestamp,ServiceName,SeverityText,Body","severityTextExpression":"SeverityText","traceIdExpression":"TraceId","spanIdExpression":"SpanId","metadataMaterializedViews":{"keyRollupTable":"otel_logs_key_rollup_15m","kvRollupTable":"otel_logs_kv_rollup_15m","granularity":"15 minute"},"connection":"Local ClickHouse","traceSourceId":"Traces","sessionSourceId":"Sessions","metricSourceId":"Metrics"},{"from":{"databaseName":"default","tableName":"otel_traces"},"kind":"trace","timestampValueExpression":"Timestamp","name":"Traces","displayedTimestampValueExpression":"Timestamp","implicitColumnExpression":"SpanName","serviceNameExpression":"ServiceName","eventAttributesExpression":"SpanAttributes","resourceAttributesExpression":"ResourceAttributes","defaultTableSelectExpression":"Timestamp,ServiceName,StatusCode,round(Duration/1e6),SpanName","traceIdExpression":"TraceId","spanIdExpression":"SpanId","durationExpression":"Duration","durationPrecision":9,"parentSpanIdExpression":"ParentSpanId","spanNameExpression":"SpanName","spanKindExpression":"SpanKind","statusCodeExpression":"StatusCode","statusMessageExpression":"StatusMessage","metadataMaterializedViews":{"keyRollupTable":"otel_traces_key_rollup_15m","kvRollupTable":"otel_traces_kv_rollup_15m","granularity":"15 minute"},"connection":"Local ClickHouse","logSourceId":"Logs","sessionSourceId":"Sessions","metricSourceId":"Metrics"},{"from":{"databaseName":"default","tableName":""},"kind":"metric","timestampValueExpression":"TimeUnix","name":"Metrics","resourceAttributesExpression":"ResourceAttributes","metricTables":{"gauge":"otel_metrics_gauge","histogram":"otel_metrics_histogram","sum":"otel_metrics_sum","_id":"682586a8b1f81924e628e808","id":"682586a8b1f81924e628e808"},"connection":"Local ClickHouse","logSourceId":"Logs","traceSourceId":"Traces","sessionSourceId":"Sessions"},{"from":{"databaseName":"default","tableName":"hyperdx_sessions"},"kind":"session","timestampValueExpression":"TimestampTime","name":"Sessions","displayedTimestampValueExpression":"Timestamp","implicitColumnExpression":"Body","serviceNameExpression":"ServiceName","bodyExpression":"Body","eventAttributesExpression":"LogAttributes","resourceAttributesExpression":"ResourceAttributes","defaultTableSelectExpression":"Timestamp,ServiceName,SeverityText,Body","severityTextExpression":"SeverityText","traceIdExpression":"TraceId","spanIdExpression":"SpanId","connection":"Local ClickHouse","logSourceId":"Logs","traceSourceId":"Traces","metricSourceId":"Metrics"},{"from":{"databaseName":"otel_json","tableName":"otel_logs"},"kind":"log","timestampValueExpression":"Timestamp","name":"JSON Logs","displayedTimestampValueExpression":"Timestamp","implicitColumnExpression":"Body","serviceNameExpression":"ServiceName","bodyExpression":"Body","eventAttributesExpression":"LogAttributes","resourceAttributesExpression":"ResourceAttributes","defaultTableSelectExpression":"Timestamp,ServiceName,SeverityText,Body","severityTextExpression":"SeverityText","traceIdExpression":"TraceId","spanIdExpression":"SpanId","connection":"Local ClickHouse","traceSourceId":"JSON Traces","metricSourceId":"JSON Metrics"},{"from":{"databaseName":"otel_json","tableName":"otel_traces"},"kind":"trace","timestampValueExpression":"Timestamp","name":"JSON Traces","displayedTimestampValueExpression":"Timestamp","implicitColumnExpression":"SpanName","serviceNameExpression":"ServiceName","eventAttributesExpression":"SpanAttributes","resourceAttributesExpression":"ResourceAttributes","defaultTableSelectExpression":"Timestamp,ServiceName,StatusCode,round(Duration/1e6),SpanName","traceIdExpression":"TraceId","spanIdExpression":"SpanId","durationExpression":"Duration","durationPrecision":9,"parentSpanIdExpression":"ParentSpanId","spanNameExpression":"SpanName","spanKindExpression":"SpanKind","statusCodeExpression":"StatusCode","statusMessageExpression":"StatusMessage","connection":"Local ClickHouse","logSourceId":"JSON Logs","metricSourceId":"JSON Metrics"},{"from":{"databaseName":"otel_json","tableName":""},"kind":"metric","timestampValueExpression":"TimeUnix","name":"JSON Metrics","resourceAttributesExpression":"ResourceAttributes","metricTables":{"gauge":"otel_metrics_gauge","histogram":"otel_metrics_histogram","sum":"otel_metrics_sum"},"connection":"Local ClickHouse","logSourceId":"JSON Logs","traceSourceId":"JSON Traces"}] +DEFAULT_CONNECTIONS=[{"name":"Local ClickHouse","host":"http://localhost:${HDX_DEV_CH_HTTP_PORT}","username":"default","password":""},{"name":"Local Prometheus","host":"http://localhost:${HDX_DEV_PROMETHEUS_PORT}","username":"","password":"","isPrometheusEndpoint":true}] +DEFAULT_SOURCES=[{"from":{"databaseName":"default","tableName":"otel_logs"},"kind":"log","timestampValueExpression":"Timestamp","name":"Logs","displayedTimestampValueExpression":"Timestamp","implicitColumnExpression":"Body","serviceNameExpression":"ServiceName","bodyExpression":"Body","eventAttributesExpression":"LogAttributes","resourceAttributesExpression":"ResourceAttributes","defaultTableSelectExpression":"Timestamp,ServiceName,SeverityText,Body","severityTextExpression":"SeverityText","traceIdExpression":"TraceId","spanIdExpression":"SpanId","metadataMaterializedViews":{"keyRollupTable":"otel_logs_key_rollup_15m","kvRollupTable":"otel_logs_kv_rollup_15m","granularity":"15 minute"},"connection":"Local ClickHouse","traceSourceId":"Traces","sessionSourceId":"Sessions","metricSourceId":"Metrics"},{"from":{"databaseName":"default","tableName":"otel_traces"},"kind":"trace","timestampValueExpression":"Timestamp","name":"Traces","displayedTimestampValueExpression":"Timestamp","implicitColumnExpression":"SpanName","serviceNameExpression":"ServiceName","eventAttributesExpression":"SpanAttributes","resourceAttributesExpression":"ResourceAttributes","defaultTableSelectExpression":"Timestamp,ServiceName,StatusCode,round(Duration/1e6),SpanName","traceIdExpression":"TraceId","spanIdExpression":"SpanId","durationExpression":"Duration","durationPrecision":9,"parentSpanIdExpression":"ParentSpanId","spanNameExpression":"SpanName","spanKindExpression":"SpanKind","statusCodeExpression":"StatusCode","statusMessageExpression":"StatusMessage","metadataMaterializedViews":{"keyRollupTable":"otel_traces_key_rollup_15m","kvRollupTable":"otel_traces_kv_rollup_15m","granularity":"15 minute"},"connection":"Local ClickHouse","logSourceId":"Logs","sessionSourceId":"Sessions","metricSourceId":"Metrics"},{"from":{"databaseName":"default","tableName":""},"kind":"metric","timestampValueExpression":"TimeUnix","name":"Metrics","resourceAttributesExpression":"ResourceAttributes","metricTables":{"gauge":"otel_metrics_gauge","histogram":"otel_metrics_histogram","sum":"otel_metrics_sum","_id":"682586a8b1f81924e628e808","id":"682586a8b1f81924e628e808"},"connection":"Local ClickHouse","logSourceId":"Logs","traceSourceId":"Traces","sessionSourceId":"Sessions"},{"from":{"databaseName":"default","tableName":"hyperdx_sessions"},"kind":"session","timestampValueExpression":"TimestampTime","name":"Sessions","displayedTimestampValueExpression":"Timestamp","implicitColumnExpression":"Body","serviceNameExpression":"ServiceName","bodyExpression":"Body","eventAttributesExpression":"LogAttributes","resourceAttributesExpression":"ResourceAttributes","defaultTableSelectExpression":"Timestamp,ServiceName,SeverityText,Body","severityTextExpression":"SeverityText","traceIdExpression":"TraceId","spanIdExpression":"SpanId","connection":"Local ClickHouse","logSourceId":"Logs","traceSourceId":"Traces","metricSourceId":"Metrics"},{"from":{"databaseName":"otel_json","tableName":"otel_logs"},"kind":"log","timestampValueExpression":"Timestamp","name":"JSON Logs","displayedTimestampValueExpression":"Timestamp","implicitColumnExpression":"Body","serviceNameExpression":"ServiceName","bodyExpression":"Body","eventAttributesExpression":"LogAttributes","resourceAttributesExpression":"ResourceAttributes","defaultTableSelectExpression":"Timestamp,ServiceName,SeverityText,Body","severityTextExpression":"SeverityText","traceIdExpression":"TraceId","spanIdExpression":"SpanId","connection":"Local ClickHouse","traceSourceId":"JSON Traces","metricSourceId":"JSON Metrics"},{"from":{"databaseName":"otel_json","tableName":"otel_traces"},"kind":"trace","timestampValueExpression":"Timestamp","name":"JSON Traces","displayedTimestampValueExpression":"Timestamp","implicitColumnExpression":"SpanName","serviceNameExpression":"ServiceName","eventAttributesExpression":"SpanAttributes","resourceAttributesExpression":"ResourceAttributes","defaultTableSelectExpression":"Timestamp,ServiceName,StatusCode,round(Duration/1e6),SpanName","traceIdExpression":"TraceId","spanIdExpression":"SpanId","durationExpression":"Duration","durationPrecision":9,"parentSpanIdExpression":"ParentSpanId","spanNameExpression":"SpanName","spanKindExpression":"SpanKind","statusCodeExpression":"StatusCode","statusMessageExpression":"StatusMessage","connection":"Local ClickHouse","logSourceId":"JSON Logs","metricSourceId":"JSON Metrics"},{"from":{"databaseName":"otel_json","tableName":""},"kind":"metric","timestampValueExpression":"TimeUnix","name":"JSON Metrics","resourceAttributesExpression":"ResourceAttributes","metricTables":{"gauge":"otel_metrics_gauge","histogram":"otel_metrics_histogram","sum":"otel_metrics_sum"},"connection":"Local ClickHouse","logSourceId":"JSON Logs","traceSourceId":"JSON Traces"},{"from":{"databaseName":"prometheus","tableName":"prometheus"},"kind":"promql","timestampValueExpression":"timestamp","name":"Prometheus","connection":"Local Prometheus","traceSourceId":"Traces"}] INGESTION_API_KEY="super-secure-ingestion-api-key" HYPERDX_API_KEY=$INGESTION_API_KEY ANTHROPIC_API_KEY="your-anthropic-api-key-here" ENABLE_PROMQL=true +ENABLE_SPAN_METRICS=true +ENABLE_SPAN_METRICS_PROM_RW=true diff --git a/packages/api/src/opamp/controllers/opampController.ts b/packages/api/src/opamp/controllers/opampController.ts index 5d726622a1..d785fe2562 100644 --- a/packages/api/src/opamp/controllers/opampController.ts +++ b/packages/api/src/opamp/controllers/opampController.ts @@ -82,6 +82,13 @@ type CollectorConfig = { pipelines: string[]; }>; }; + span_metrics?: { + histogram: { unit: string; explicit: { buckets: string[] } }; + dimensions: Array<{ name: string }>; + exemplars: { enabled: boolean }; + metrics_flush_interval: string; + namespace?: string; + }; }; exporters?: { nop?: null; @@ -132,6 +139,15 @@ type CollectorConfig = { enabled: boolean; }; }; + 'prometheusremotewrite/spanmetrics'?: { + endpoint: string; + tls: { + insecure: boolean; + }; + resource_to_telemetry_conversion: { + enabled: boolean; + }; + }; }; service: { extensions: string[]; @@ -322,6 +338,68 @@ export const buildOtelCollectorConfig = ( }; } + if ( + config.IS_SPAN_METRICS_ENABLED && + otelCollectorConfig.connectors && + otelCollectorConfig.exporters + ) { + // Derive request metrics (with trace exemplars) from spans. The connector + // consumes the traces pipeline and feeds a dedicated metrics pipeline, so + // the resulting `traces.span.metrics.*` land in ClickHouse with + // `Exemplars.*` pointing back at the spans they were measured from. + otelCollectorConfig.connectors.span_metrics = { + histogram: { + unit: 'ms', + explicit: { + buckets: [ + '2ms', + '5ms', + '10ms', + '25ms', + '50ms', + '100ms', + '250ms', + '500ms', + '1s', + '2.5s', + '5s', + '10s', + ], + }, + }, + dimensions: [ + { name: 'http.route' }, + { name: 'http.method' }, + { name: 'host.region' }, + { name: 'app.tenant_id' }, + { name: 'http.status_code' }, + ], + exemplars: { enabled: true }, + metrics_flush_interval: '15s', + }; + otelCollectorConfig.service.pipelines.traces.exporters.push( + 'span_metrics', + ); + + const spanMetricsExporters = ['clickhouse']; + // Optionally also remote-write the derived metrics (with exemplars) to a + // Prometheus endpoint, so the native Prometheus `query_exemplars` path can + // be exercised against the same real, generated data. + if (config.IS_SPAN_METRICS_PROM_RW_ENABLED) { + otelCollectorConfig.exporters['prometheusremotewrite/spanmetrics'] = { + endpoint: '${env:SPAN_METRICS_PROM_RW_ENDPOINT}', + tls: { insecure: true }, + resource_to_telemetry_conversion: { enabled: true }, + }; + spanMetricsExporters.push('prometheusremotewrite/spanmetrics'); + } + otelCollectorConfig.service.pipelines['metrics/spanmetrics'] = { + receivers: ['span_metrics'], + processors: ['memory_limiter', 'batch'], + exporters: spanMetricsExporters, + }; + } + if (collectorAuthenticationEnforced) { if (otelCollectorConfig.receivers['otlp/hyperdx'] == null) { // should never happen diff --git a/packages/otel-collector/builder-config.yaml b/packages/otel-collector/builder-config.yaml index 5b8e0c8f94..23ef4ee2d5 100644 --- a/packages/otel-collector/builder-config.yaml +++ b/packages/otel-collector/builder-config.yaml @@ -132,6 +132,9 @@ connectors: - gomod: github.com/open-telemetry/opentelemetry-collector-contrib/connector/routingconnector v__OTEL_COLLECTOR_VERSION__ + - gomod: + github.com/open-telemetry/opentelemetry-collector-contrib/connector/spanmetricsconnector + v__OTEL_COLLECTOR_VERSION__ extensions: # Core diff --git a/scripts/dev-env.sh b/scripts/dev-env.sh index 938aaeb4d7..135eca7ef5 100755 --- a/scripts/dev-env.sh +++ b/scripts/dev-env.sh @@ -55,6 +55,7 @@ HDX_DEV_OTEL_GRPC_PORT=$((30800 + HDX_DEV_SLOT)) HDX_DEV_OTEL_HTTP_PORT=$((30900 + HDX_DEV_SLOT)) HDX_DEV_OTEL_METRICS_PORT=$((31000 + HDX_DEV_SLOT)) HDX_DEV_OTEL_JSON_HTTP_PORT=$((31100 + HDX_DEV_SLOT)) +HDX_DEV_PROMETHEUS_PORT=$((31200 + HDX_DEV_SLOT)) # --- Docker Compose project name (unique per slot) --- HDX_DEV_PROJECT="hdx-dev-${HDX_DEV_SLOT}" @@ -80,6 +81,7 @@ export HDX_DEV_OTEL_GRPC_PORT export HDX_DEV_OTEL_HTTP_PORT export HDX_DEV_OTEL_METRICS_PORT export HDX_DEV_OTEL_JSON_HTTP_PORT +export HDX_DEV_PROMETHEUS_PORT export HDX_DEV_PROJECT export NX_CACHE_DIRECTORY @@ -111,6 +113,7 @@ cat > "${HDX_DEV_SLOTS_DIR}/${HDX_DEV_SLOT}.json" < Date: Mon, 29 Jun 2026 15:50:43 +1000 Subject: [PATCH 04/10] Configured spanmetric generation and changesets --- .changeset/exemplar-mode-metrics.md | 22 +++++++++++++++++++ .changeset/span-metrics-connector.md | 15 +++++++++++++ docker-compose.dev.yml | 33 ++++++++++++++++++++++++++-- docker/prometheus/prometheus.yml | 4 ++++ 4 files changed, 72 insertions(+), 2 deletions(-) create mode 100644 .changeset/exemplar-mode-metrics.md create mode 100644 .changeset/span-metrics-connector.md diff --git a/.changeset/exemplar-mode-metrics.md b/.changeset/exemplar-mode-metrics.md new file mode 100644 index 0000000000..dda84d862c --- /dev/null +++ b/.changeset/exemplar-mode-metrics.md @@ -0,0 +1,22 @@ +--- +"@hyperdx/common-utils": minor +"@hyperdx/api": minor +"@hyperdx/app": minor +--- + +feat: add exemplar overlay for metric and PromQL charts + +Time charts on metric and PromQL sources can now overlay exemplars — +individual data points linked to a trace — via the "Exemplars" toggle in the +chart editor (next to "As Ratio" for metric charts, in the PromQL editor for +PromQL charts). Markers snap onto the series line so the chart stays honest; +hovering a marker shows trace metadata (service, span, duration, status) from a +configurable exemplar trace source, with a button to open the trace directly. + +For structured metric sources, exemplars are read directly from the OTel metric +tables' `Exemplars.*` columns (`renderMetricExemplarsChartConfig`), honoring the +chart's time range, metric name, and filters. For PromQL sources backed by a +real Prometheus endpoint, the new `/v1/prometheus/query_exemplars` route proxies +to Prometheus's native `/api/v1/query_exemplars`. The overlay is opt-in and runs +its query in parallel only when enabled, so charts that don't use it are +unaffected. Trace-source exemplar generation lands in a follow-up. diff --git a/.changeset/span-metrics-connector.md b/.changeset/span-metrics-connector.md new file mode 100644 index 0000000000..002e942870 --- /dev/null +++ b/.changeset/span-metrics-connector.md @@ -0,0 +1,15 @@ +--- +"@hyperdx/api": minor +"@hyperdx/otel-collector": minor +--- + +feat: optional spanmetrics connector for metric exemplars + +Adds the `spanmetricsconnector` to the collector build and wires it into the +OpAMP-generated collector config, gated on the `ENABLE_SPAN_METRICS` env flag +(off by default). When enabled, the collector derives `traces.span.metrics.*` +(calls + duration histogram) from spans with **exemplars enabled**, so the +duration histogram lands in ClickHouse with `Exemplars.*` pointing back at the +spans they were measured from — giving coherent, fully-OTLP metric exemplars +without any direct ClickHouse writes. Enabled in local dev to back the new +`telemetry-generator` service. diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml index 95e882eca0..7ec6efc04a 100644 --- a/docker-compose.dev.yml +++ b/docker-compose.dev.yml @@ -41,6 +41,9 @@ services: OTEL_SUPERVISOR_LOGS: 'true' HYPERDX_OTEL_EXPORTER_TABLES_TTL: '24h' ENABLE_PROMQL: 'true' + # Remote-write the span-derived metrics (with exemplars) to the dev + # Prometheus so the native query_exemplars path is testable with real data. + SPAN_METRICS_PROM_RW_ENDPOINT: 'http://prometheus:9090/api/v1/write' volumes: - ./docker/otel-collector/config.yaml:/etc/otelcol-contrib/config.yaml - ./docker/otel-collector/supervisor_docker.yaml.tmpl:/etc/otel/supervisor.yaml.tmpl @@ -157,9 +160,15 @@ services: hdx.dev.service: prometheus hdx.dev.port: '${HDX_DEV_PROMETHEUS_PORT:-9090}' hdx.dev.url: 'http://localhost:${HDX_DEV_PROMETHEUS_PORT:-9090}' - profiles: - - prometheus image: prom/prometheus:latest + # exemplar-storage: query_exemplars data; remote-write receiver: ingest the + # collector's span-derived metrics (with exemplars) from real generated data. + command: + - '--config.file=/etc/prometheus/prometheus.yml' + - '--storage.tsdb.path=/prometheus' + - '--enable-feature=exemplar-storage' + - '--web.enable-remote-write-receiver' + - '--web.enable-lifecycle' ports: - '${HDX_DEV_PROMETHEUS_PORT:-9090}:9090' volumes: @@ -169,5 +178,25 @@ services: - internal restart: on-failure + # Synthetic traces (via OTLP) + coherent metric exemplars (to ClickHouse) for + # local dev + e2e. See telemetry-generator/README.md. + telemetry-generator: + labels: + <<: *hdx-labels + hdx.dev.service: telemetry-generator + build: + context: ./telemetry-generator + environment: + OTEL_EXPORTER_OTLP_ENDPOINT: 'http://otel-collector:4318' + GEN_OTLP_API_KEY: '${INGESTION_API_KEY:-super-secure-ingestion-api-key}' + GEN_BACKFILL_MINUTES: '30' + GEN_RATE_PER_SEC: '20' + networks: + - internal + restart: on-failure + depends_on: + otel-collector: + condition: service_started + networks: internal: diff --git a/docker/prometheus/prometheus.yml b/docker/prometheus/prometheus.yml index 94877ac65d..ff6643b21f 100644 --- a/docker/prometheus/prometheus.yml +++ b/docker/prometheus/prometheus.yml @@ -18,3 +18,7 @@ scrape_configs: static_configs: - targets: ['ch-server:9363'] metrics_path: '/metrics' + + # Note: the span-derived request metrics (with exemplars) arrive via + # remote-write from the OTel collector, not a scrape — see the collector's + # SPAN_METRICS_PROM_RW_ENDPOINT and the spanmetrics connector. From d2bcfc4ebb11fa6b9d443b11f7896ca4934c5f18 Mon Sep 17 00:00:00 2001 From: Jordan Simonovski Date: Mon, 29 Jun 2026 22:10:03 +1000 Subject: [PATCH 05/10] Resolved knip and greptile stuff --- knip.json | 3 ++- packages/api/src/config.ts | 5 +++- .../__tests__/DBSearchPageQueryKey.test.tsx | 7 ++++++ .../src/__tests__/renderChartConfig.test.ts | 24 +++++++++++++++++++ .../src/core/renderChartConfig.ts | 5 ++-- 5 files changed, 40 insertions(+), 4 deletions(-) diff --git a/knip.json b/knip.json index 7d7837b24b..e5ff8325ea 100644 --- a/knip.json +++ b/knip.json @@ -35,7 +35,8 @@ "ignore": [ "scripts/dev-portal/**", ".github/scripts/**", - "docker/hyperdx/**" + "docker/hyperdx/**", + "telemetry-generator/**" ], "ignoreBinaries": ["make", "migrate", "playwright"], "ignoreDependencies": [ diff --git a/packages/api/src/config.ts b/packages/api/src/config.ts index 5a34941e33..bbbfc10130 100644 --- a/packages/api/src/config.ts +++ b/packages/api/src/config.ts @@ -59,8 +59,11 @@ export const IS_SPAN_METRICS_ENABLED = env.ENABLE_SPAN_METRICS === 'true'; // Opt-in: also remote-write the span-derived metrics (with exemplars) to a // Prometheus endpoint (SPAN_METRICS_PROM_RW_ENDPOINT on the collector) so the // native Prometheus query_exemplars path can be tested against real data. +// Requires the endpoint to be set; without it the generated collector config +// would fail to resolve ${env:SPAN_METRICS_PROM_RW_ENDPOINT} and not start. export const IS_SPAN_METRICS_PROM_RW_ENABLED = - env.ENABLE_SPAN_METRICS_PROM_RW === 'true'; + env.ENABLE_SPAN_METRICS_PROM_RW === 'true' && + !!env.SPAN_METRICS_PROM_RW_ENDPOINT; // FOR CI ONLY export const CLICKHOUSE_HOST = env.CLICKHOUSE_HOST as string; diff --git a/packages/app/src/__tests__/DBSearchPageQueryKey.test.tsx b/packages/app/src/__tests__/DBSearchPageQueryKey.test.tsx index 210e970c26..7b829f8687 100644 --- a/packages/app/src/__tests__/DBSearchPageQueryKey.test.tsx +++ b/packages/app/src/__tests__/DBSearchPageQueryKey.test.tsx @@ -19,6 +19,13 @@ jest.mock('@/api', () => ({ }, })); +// Exemplar hooks need a QueryClientProvider (via useMetadataWithSettings) and +// are irrelevant to queryKey consistency, so stub them out. +jest.mock('@/hooks/useExemplars', () => ({ + useExemplars: () => ({ exemplars: [], isLoading: false, isError: false }), + useExemplarTraceMeta: () => ({ data: null, isLoading: false }), +})); + jest.mock('@/hooks/useMVOptimizationExplanation', () => ({ useMVOptimizationExplanation: jest.fn().mockReturnValue({ data: undefined, diff --git a/packages/common-utils/src/__tests__/renderChartConfig.test.ts b/packages/common-utils/src/__tests__/renderChartConfig.test.ts index 58cad9f8d5..8c480634fd 100644 --- a/packages/common-utils/src/__tests__/renderChartConfig.test.ts +++ b/packages/common-utils/src/__tests__/renderChartConfig.test.ts @@ -3514,6 +3514,30 @@ describe('renderMetricExemplarsChartConfig', () => { expect(sql).toContain(`LIMIT ${EXEMPLAR_QUERY_LIMIT}`); }); + it("scopes exemplars to the series' aggCondition so markers match the plotted line", async () => { + const filteredConfig = { + ...histogramConfig, + select: [ + { + aggFn: 'quantile', + level: 0.95, + valueExpression: 'Value', + metricName: 'http.server.duration', + metricType: MetricsDataType.Histogram, + aggCondition: "ServiceName = 'api'", + aggConditionLanguage: 'sql', + }, + ], + } as ChartConfigWithOptDateRange; + const generated = await renderMetricExemplarsChartConfig( + filteredConfig, + mockMetadata, + ); + expect(generated).not.toBeNull(); + const sql = parameterizedQueryToSql(generated!); + expect(sql).toContain("ServiceName = 'api'"); + }); + it('returns null for a ratio config (exemplars are meaningless on a ratio axis)', async () => { const ratioConfig = { ...histogramConfig, diff --git a/packages/common-utils/src/core/renderChartConfig.ts b/packages/common-utils/src/core/renderChartConfig.ts index 2245aec457..4c695b06de 100644 --- a/packages/common-utils/src/core/renderChartConfig.ts +++ b/packages/common-utils/src/core/renderChartConfig.ts @@ -2232,8 +2232,9 @@ export async function renderMetricExemplarsChartConfig( from: { ...chartConfig.from, tableName: table }, timestampValueExpression: chartConfig.timestampValueExpression || DEFAULT_METRIC_TABLE_TIME_COLUMN, - // Drop the metric select so renderWhere doesn't push aggCondition predicates. - select: [], + // Keep the original select so renderWhere applies the series' aggCondition — + // otherwise the exemplar scan would surface traces from other series (e.g. + // other services/routes/tenants) that share the same metric name. filters: [ ...(chartConfig.filters ?? []), { From 30fca770c519845e20e841ec935cbded3aed8e0b Mon Sep 17 00:00:00 2001 From: Jordan Simonovski Date: Tue, 30 Jun 2026 09:37:05 +1000 Subject: [PATCH 06/10] fix: The endpoint is now resolved API-side and inlined into the generated collector config, so the collector container no longer needs SPAN_METRICS_PROM_RW_ENDPOINT in its own environment. --- packages/api/src/config.ts | 13 +++++++------ .../api/src/opamp/controllers/opampController.ts | 3 ++- 2 files changed, 9 insertions(+), 7 deletions(-) diff --git a/packages/api/src/config.ts b/packages/api/src/config.ts index bbbfc10130..20bfff0b0c 100644 --- a/packages/api/src/config.ts +++ b/packages/api/src/config.ts @@ -57,13 +57,14 @@ export const IS_PROMQL_ENABLED = env.ENABLE_PROMQL === 'true'; export const IS_SPAN_METRICS_ENABLED = env.ENABLE_SPAN_METRICS === 'true'; // Opt-in: also remote-write the span-derived metrics (with exemplars) to a -// Prometheus endpoint (SPAN_METRICS_PROM_RW_ENDPOINT on the collector) so the -// native Prometheus query_exemplars path can be tested against real data. -// Requires the endpoint to be set; without it the generated collector config -// would fail to resolve ${env:SPAN_METRICS_PROM_RW_ENDPOINT} and not start. +// Prometheus endpoint so the native Prometheus query_exemplars path can be +// tested against real data. The endpoint is resolved here (API side) and +// inlined into the generated collector config, so the collector container does +// not need SPAN_METRICS_PROM_RW_ENDPOINT in its own environment. Requires the +// endpoint to be set; without it the feature stays disabled. +export const SPAN_METRICS_PROM_RW_ENDPOINT = env.SPAN_METRICS_PROM_RW_ENDPOINT; export const IS_SPAN_METRICS_PROM_RW_ENABLED = - env.ENABLE_SPAN_METRICS_PROM_RW === 'true' && - !!env.SPAN_METRICS_PROM_RW_ENDPOINT; + env.ENABLE_SPAN_METRICS_PROM_RW === 'true' && !!SPAN_METRICS_PROM_RW_ENDPOINT; // FOR CI ONLY export const CLICKHOUSE_HOST = env.CLICKHOUSE_HOST as string; diff --git a/packages/api/src/opamp/controllers/opampController.ts b/packages/api/src/opamp/controllers/opampController.ts index d785fe2562..2774b385a4 100644 --- a/packages/api/src/opamp/controllers/opampController.ts +++ b/packages/api/src/opamp/controllers/opampController.ts @@ -387,7 +387,8 @@ export const buildOtelCollectorConfig = ( // be exercised against the same real, generated data. if (config.IS_SPAN_METRICS_PROM_RW_ENABLED) { otelCollectorConfig.exporters['prometheusremotewrite/spanmetrics'] = { - endpoint: '${env:SPAN_METRICS_PROM_RW_ENDPOINT}', + // Guaranteed set by IS_SPAN_METRICS_PROM_RW_ENABLED above. + endpoint: config.SPAN_METRICS_PROM_RW_ENDPOINT!, tls: { insecure: true }, resource_to_telemetry_conversion: { enabled: true }, }; From 72db1ee900dbb85acfb08ad88cb791648f50018a Mon Sep 17 00:00:00 2001 From: Jordan Simonovski Date: Fri, 24 Jul 2026 11:56:50 +1000 Subject: [PATCH 07/10] refactor(app): extract exemplar chart components with Storybook stories Move the inline ExemplarDot marker and ExemplarHoverCard popover out of the oversized HDXMultiSeriesTimeChart and DBTimeChart into a focused components/Exemplars/ directory, and add a Storybook story for each so the marker and every hover-card state (full/partial metadata, loading, not-found, no-trace-source) can be developed and reviewed in isolation across light/dark and both brand themes. Behaviour-preserving: the extracted component bodies are unchanged; ExemplarDot also gains a real props type in place of `any`. --- packages/app/src/HDXMultiSeriesTimeChart.tsx | 30 +--- packages/app/src/components/DBTimeChart.tsx | 131 +----------------- .../Exemplars/ExemplarDot.stories.tsx | 69 +++++++++ .../src/components/Exemplars/ExemplarDot.tsx | 49 +++++++ .../Exemplars/ExemplarHoverCard.stories.tsx | 103 ++++++++++++++ .../Exemplars/ExemplarHoverCard.tsx | 125 +++++++++++++++++ .../app/src/components/Exemplars/index.ts | 2 + 7 files changed, 352 insertions(+), 157 deletions(-) create mode 100644 packages/app/src/components/Exemplars/ExemplarDot.stories.tsx create mode 100644 packages/app/src/components/Exemplars/ExemplarDot.tsx create mode 100644 packages/app/src/components/Exemplars/ExemplarHoverCard.stories.tsx create mode 100644 packages/app/src/components/Exemplars/ExemplarHoverCard.tsx create mode 100644 packages/app/src/components/Exemplars/index.ts diff --git a/packages/app/src/HDXMultiSeriesTimeChart.tsx b/packages/app/src/HDXMultiSeriesTimeChart.tsx index 26ce495231..bb50911545 100644 --- a/packages/app/src/HDXMultiSeriesTimeChart.tsx +++ b/packages/app/src/HDXMultiSeriesTimeChart.tsx @@ -46,6 +46,7 @@ import { toViewportPoint, useChartTooltipZIndex, } from './components/charts/ChartTooltip'; +import { ExemplarDot } from './components/Exemplars'; import { useChartSyncId } from './chartSync'; import { findNearestSeriesKey, @@ -598,35 +599,6 @@ function numOrNull(v: unknown): number | null { return typeof v === 'number' && !isNaN(v) ? v : null; } -/** - * Diamond marker for an exemplar, drawn via . - * Recharts injects cx/cy. Hovering opens a floating menu (handled by the parent - * via onHoverStart/onHoverEnd) to inspect the linked trace — the marker itself - * is not a click target. A larger transparent hit circle eases hovering. - */ -function ExemplarDot(props: any) { - const { cx, cy, exemplar, onHoverStart, onHoverEnd } = props; - if (typeof cx !== 'number' || typeof cy !== 'number') { - return null; - } - const s = 4; - return ( - onHoverStart?.(exemplar, cx, cy)} - onMouseLeave={() => onHoverEnd?.()} - > - - - - ); -} - /** * Compute the unique set of hexes referenced by `` defs * inside MemoChart. Exported so a unit test can pin the dedup-and-union diff --git a/packages/app/src/components/DBTimeChart.tsx b/packages/app/src/components/DBTimeChart.tsx index 43db6b8d24..ea393e87c7 100644 --- a/packages/app/src/components/DBTimeChart.tsx +++ b/packages/app/src/components/DBTimeChart.tsx @@ -3,7 +3,6 @@ import React, { useCallback, useEffect, useId, - useLayoutEffect, useMemo, useRef, useState, @@ -25,15 +24,7 @@ import { DisplayType, Exemplar, } from '@hyperdx/common-utils/dist/types'; -import { - Button, - Group, - Paper, - Popover, - Portal, - Stack, - Text, -} from '@mantine/core'; +import { Popover, Portal } from '@mantine/core'; import { IconChartBar, IconChartLine } from '@tabler/icons-react'; import api from '@/api'; @@ -50,14 +41,11 @@ import { import { ChartAnnotation } from '@/components/charts/chartAnnotations'; import { ChartSeriesTooltip } from '@/components/charts/ChartSeriesTooltip'; import { useChartTooltipZIndex } from '@/components/charts/ChartTooltip'; +import { ExemplarHoverCard } from '@/components/Exemplars'; import { DEFAULT_MAX_EXEMPLARS } from '@/defaults'; import { type ActiveClickPayload, MemoChart } from '@/HDXMultiSeriesTimeChart'; import { useQueriedChartConfig } from '@/hooks/useChartConfig'; -import { - ExemplarTraceMeta, - useExemplars, - useExemplarTraceMeta, -} from '@/hooks/useExemplars'; +import { useExemplars, useExemplarTraceMeta } from '@/hooks/useExemplars'; import { useMVOptimizationExplanation } from '@/hooks/useMVOptimizationExplanation'; import { useChartNumberFormats, useSource } from '@/source'; import type { NumberFormat } from '@/types'; @@ -73,119 +61,6 @@ import MVOptimizationIndicator from './MaterializedViews/MVOptimizationIndicator /** A single group column / value pair decoded from a chart series key. */ export type SeriesGroupFilter = { column: string; value: string }; -// Floating card shown when hovering an exemplar marker: trace metadata (from the -// configured exemplar trace source) plus a button to open the trace directly. -function ExemplarHoverCard({ - hovered, - meta, - isLoading, - traceSourceConfigured, - onInspect, - onMouseEnter, - onMouseLeave, -}: { - hovered: { exemplar: Exemplar; x: number; y: number } | null; - meta?: ExemplarTraceMeta; - isLoading: boolean; - traceSourceConfigured: boolean; - onInspect: (exemplar: Exemplar) => void; - onMouseEnter: () => void; - onMouseLeave: () => void; -}) { - const ref = useRef(null); - const [pos, setPos] = useState<{ left: number; top: number } | null>(null); - - // Position the card next to the marker, but flip to the left / clamp upward - // when it would overflow the chart container, so it's never cut off. Measured - // after render (size depends on the async-loaded metadata). - useLayoutEffect(() => { - if (!hovered || !ref.current) { - setPos(null); - return; - } - const el = ref.current; - const parent = el.offsetParent as HTMLElement | null; - const pW = parent?.clientWidth ?? window.innerWidth; - const pH = parent?.clientHeight ?? window.innerHeight; - const cardW = el.offsetWidth; - const cardH = el.offsetHeight; - const margin = 12; - - let left = hovered.x + margin; - if (left + cardW > pW) left = hovered.x - margin - cardW; // flip left - left = Math.max(4, Math.min(left, pW - cardW - 4)); - - let top = hovered.y - margin; - if (top + cardH > pH) top = pH - cardH - 4; // shift up to stay in view - top = Math.max(4, top); - - setPos({ left, top }); - }, [hovered, meta, isLoading, traceSourceConfigured]); - - if (!hovered) return null; - const { exemplar } = hovered; - return ( -
- - - - - Exemplar - - - {exemplar.traceId.slice(0, 16)}… - - - {!traceSourceConfigured ? ( - - Set an exemplar trace source in the chart editor to see trace - details. - - ) : isLoading ? ( - - Loading trace… - - ) : meta ? ( - - {meta.service && Service: {meta.service}} - {meta.spanName && Span: {meta.spanName}} - {meta.durationMs != null && ( - Duration: {meta.durationMs.toFixed(1)} ms - )} - {meta.statusCode && ( - Status: {meta.statusCode} - )} - - ) : ( - - Trace not found in source. - - )} - - - -
- ); -} - // Only one pinned tooltip at a time across all charts. Module-level (not // context) because charts can be scattered with no common provider, and their // onClick stopPropagation hides cross-chart clicks from Mantine's click-outside. diff --git a/packages/app/src/components/Exemplars/ExemplarDot.stories.tsx b/packages/app/src/components/Exemplars/ExemplarDot.stories.tsx new file mode 100644 index 0000000000..13db0628f1 --- /dev/null +++ b/packages/app/src/components/Exemplars/ExemplarDot.stories.tsx @@ -0,0 +1,69 @@ +import type { ReactNode } from 'react'; +import { Exemplar } from '@hyperdx/common-utils/dist/types'; +import type { Meta, StoryObj } from '@storybook/nextjs'; + +import { ExemplarDot } from './ExemplarDot'; + +/** + * Diamond marker overlaid on a time chart to mark an individual exemplar trace. + * It is rendered by recharts as a `} />`, so + * recharts injects `cx`/`cy`; here we place it inside a plain `` to show the + * marker in isolation. The fill uses the `--color-chart-warning` token — use the + * Theme (Light / Dark) and Brand toolbar toggles to review both. + */ +const meta = { + title: 'Components/Exemplars/ExemplarDot', + component: ExemplarDot, + parameters: { layout: 'centered' }, +} satisfies Meta; + +export default meta; +type Story = StoryObj; + +const mockExemplar: Exemplar = { + timestamp: 1_700_000_000_000, + value: 128.4, + traceId: '4bf92f3577b34da6a3ce929d0e0e4736', + spanId: '00f067aa0ba902b7', +}; + +// A small chart-like canvas so the marker has a baseline for context. +function SvgCanvas({ children }: { children: ReactNode }) { + return ( + + + {children} + + ); +} + +export const Default: Story = { + args: { cx: 140, cy: 80, exemplar: mockExemplar }, + render: args => ( + + + + ), +}; + +export const AlongASeries: Story = { + args: { exemplar: mockExemplar }, + render: args => ( + + {[40, 90, 140, 190, 240].map((cx, i) => ( + + ))} + + ), +}; diff --git a/packages/app/src/components/Exemplars/ExemplarDot.tsx b/packages/app/src/components/Exemplars/ExemplarDot.tsx new file mode 100644 index 0000000000..bfc64b4bbb --- /dev/null +++ b/packages/app/src/components/Exemplars/ExemplarDot.tsx @@ -0,0 +1,49 @@ +import { Exemplar } from '@hyperdx/common-utils/dist/types'; + +// Half-diagonal of the diamond marker, in px. +const DIAMOND_HALF_SIZE = 4; +// Radius of the transparent hit target that eases hovering the small marker. +const HIT_RADIUS = 9; + +type ExemplarDotProps = { + // cx/cy are injected by recharts when this is used as a . + cx?: number; + cy?: number; + exemplar: Exemplar; + onHoverStart?: (exemplar: Exemplar, cx: number, cy: number) => void; + onHoverEnd?: () => void; +}; + +/** + * Diamond marker for an exemplar, drawn via . + * Recharts injects cx/cy. Hovering opens a floating menu (handled by the parent + * via onHoverStart/onHoverEnd) to inspect the linked trace — the marker itself + * is not a click target. A larger transparent hit circle eases hovering. + */ +export function ExemplarDot({ + cx, + cy, + exemplar, + onHoverStart, + onHoverEnd, +}: ExemplarDotProps) { + if (typeof cx !== 'number' || typeof cy !== 'number') { + return null; + } + const s = DIAMOND_HALF_SIZE; + return ( + onHoverStart?.(exemplar, cx, cy)} + onMouseLeave={() => onHoverEnd?.()} + > + + + + ); +} diff --git a/packages/app/src/components/Exemplars/ExemplarHoverCard.stories.tsx b/packages/app/src/components/Exemplars/ExemplarHoverCard.stories.tsx new file mode 100644 index 0000000000..d8c755c87d --- /dev/null +++ b/packages/app/src/components/Exemplars/ExemplarHoverCard.stories.tsx @@ -0,0 +1,103 @@ +import type { ReactNode } from 'react'; +import { Exemplar } from '@hyperdx/common-utils/dist/types'; +import type { Meta, StoryObj } from '@storybook/nextjs'; + +import { ExemplarHoverCard } from './ExemplarHoverCard'; + +/** + * Floating card shown when hovering an exemplar marker on a time chart. It shows + * the linked trace's metadata (resolved from the configured exemplar trace + * source) plus an "Inspect trace" button. The card is `position: absolute` and + * flips / clamps against its offset parent so it never overflows the chart, so + * every story wraps it in a relative, chart-sized container. Use the Theme + * (Light / Dark) and Brand toolbar toggles to review each state. + */ +const meta = { + title: 'Components/Exemplars/ExemplarHoverCard', + component: ExemplarHoverCard, + parameters: { layout: 'centered' }, +} satisfies Meta; + +export default meta; +type Story = StoryObj; + +const mockExemplar: Exemplar = { + timestamp: 1_700_000_000_000, + value: 128.4, + traceId: '4bf92f3577b34da6a3ce929d0e0e4736', + spanId: '00f067aa0ba902b7', +}; + +const hovered = { exemplar: mockExemplar, x: 60, y: 40 }; + +const noop = () => {}; + +// A relative, chart-sized container so the card's absolute positioning and +// flip/clamp logic resolve against a realistic offset parent. +function ChartArea({ children }: { children: ReactNode }) { + return ( +
+ {children} +
+ ); +} + +const baseArgs = { + hovered, + isLoading: false, + traceSourceConfigured: true, + onInspect: noop, + onMouseEnter: noop, + onMouseLeave: noop, +}; + +const render: Story['render'] = args => ( + + + +); + +export const FullMetadata: Story = { + render, + args: { + ...baseArgs, + meta: { + service: 'checkout-api', + spanName: 'POST /checkout', + durationMs: 128.42, + statusCode: 'OK', + }, + }, +}; + +export const PartialMetadata: Story = { + render, + args: { + ...baseArgs, + meta: { service: 'checkout-api', durationMs: 128.42 }, + }, +}; + +export const Loading: Story = { + render, + args: { ...baseArgs, isLoading: true }, +}; + +export const TraceNotFound: Story = { + render, + args: { ...baseArgs, meta: undefined }, +}; + +export const NoTraceSourceConfigured: Story = { + render, + args: { ...baseArgs, traceSourceConfigured: false }, +}; diff --git a/packages/app/src/components/Exemplars/ExemplarHoverCard.tsx b/packages/app/src/components/Exemplars/ExemplarHoverCard.tsx new file mode 100644 index 0000000000..c698987df8 --- /dev/null +++ b/packages/app/src/components/Exemplars/ExemplarHoverCard.tsx @@ -0,0 +1,125 @@ +import { useLayoutEffect, useRef, useState } from 'react'; +import { Exemplar } from '@hyperdx/common-utils/dist/types'; +import { Button, Group, Paper, Stack, Text } from '@mantine/core'; + +import type { ExemplarTraceMeta } from '@/hooks/useExemplars'; + +type ExemplarHoverCardProps = { + /** The hovered exemplar plus its on-screen position; null hides the card. */ + hovered: { exemplar: Exemplar; x: number; y: number } | null; + /** Trace metadata resolved from the configured exemplar trace source. */ + meta?: ExemplarTraceMeta; + isLoading: boolean; + /** Whether an exemplar trace source is configured for this chart. */ + traceSourceConfigured: boolean; + onInspect: (exemplar: Exemplar) => void; + onMouseEnter: () => void; + onMouseLeave: () => void; +}; + +/** + * Floating card shown when hovering an exemplar marker: trace metadata (from the + * configured exemplar trace source) plus a button to open the trace directly. + */ +export function ExemplarHoverCard({ + hovered, + meta, + isLoading, + traceSourceConfigured, + onInspect, + onMouseEnter, + onMouseLeave, +}: ExemplarHoverCardProps) { + const ref = useRef(null); + const [pos, setPos] = useState<{ left: number; top: number } | null>(null); + + // Position the card next to the marker, but flip to the left / clamp upward + // when it would overflow the chart container, so it's never cut off. Measured + // after render (size depends on the async-loaded metadata). + useLayoutEffect(() => { + if (!hovered || !ref.current) { + setPos(null); + return; + } + const el = ref.current; + const parent = el.offsetParent as HTMLElement | null; + const pW = parent?.clientWidth ?? window.innerWidth; + const pH = parent?.clientHeight ?? window.innerHeight; + const cardW = el.offsetWidth; + const cardH = el.offsetHeight; + const margin = 12; + + let left = hovered.x + margin; + if (left + cardW > pW) left = hovered.x - margin - cardW; // flip left + left = Math.max(4, Math.min(left, pW - cardW - 4)); + + let top = hovered.y - margin; + if (top + cardH > pH) top = pH - cardH - 4; // shift up to stay in view + top = Math.max(4, top); + + setPos({ left, top }); + }, [hovered, meta, isLoading, traceSourceConfigured]); + + if (!hovered) return null; + const { exemplar } = hovered; + return ( +
+ + + + + Exemplar + + + {exemplar.traceId.slice(0, 16)}… + + + {!traceSourceConfigured ? ( + + Set an exemplar trace source in the chart editor to see trace + details. + + ) : isLoading ? ( + + Loading trace… + + ) : meta ? ( + + {meta.service && Service: {meta.service}} + {meta.spanName && Span: {meta.spanName}} + {meta.durationMs != null && ( + Duration: {meta.durationMs.toFixed(1)} ms + )} + {meta.statusCode && ( + Status: {meta.statusCode} + )} + + ) : ( + + Trace not found in source. + + )} + + + +
+ ); +} diff --git a/packages/app/src/components/Exemplars/index.ts b/packages/app/src/components/Exemplars/index.ts new file mode 100644 index 0000000000..49408d6a3b --- /dev/null +++ b/packages/app/src/components/Exemplars/index.ts @@ -0,0 +1,2 @@ +export { ExemplarDot } from './ExemplarDot'; +export { ExemplarHoverCard } from './ExemplarHoverCard'; From 29fd257c3fcf29771bc2fdb0b092ba2f1a3abf7e Mon Sep 17 00:00:00 2001 From: Jordan Simonovski Date: Fri, 24 Jul 2026 13:06:10 +1000 Subject: [PATCH 08/10] fix(exemplars): scope filters, gate behind a flag, and stop outliers crushing the axis Addresses review feedback (Greptile P1 + Deep Review P2s) and hardens the exemplar overlay while it's still being tested. Correctness / scoping: - Metric exemplar SQL now ANDs the metric-name predicate separately instead of appending it to the user filter group, so a chart with filtersLogicalOperator 'OR' can no longer let the exemplar scan match other metrics. - Prometheus normaliser drops the overlay when the query returns >1 series (exemplars are single-series only), and the PromQL result is capped client-side (no native limit param) to bound render/thinning work. - enableExemplars is cleared when a chart leaves single-series so a stale flag can't persist on a config that no longer supports it. Feature gate: - Whole feature (editor toggle, PromQL toggle, data fetch, team setting) gated behind NEXT_PUBLIC_ENABLE_EXEMPLARS; on in dev, off by default elsewhere. Rendering: - Exemplar outliers no longer stretch the y-axis: the domain follows the visible series and each marker is clamped to the series max (its hover card still shows the true value), so one slow trace can't flatten the p99 line. - The series hover tooltip is suppressed while hovering an exemplar so the two cards don't overlap; the chart resets this against the rendered points so a refetch/re-thinning that unmounts the hovered marker can't leave it stuck. - Higher-contrast marker outline for light mode. Maintainability / tests: - Exemplar bucketing extracted to a pure, unit-tested helper; Prometheus exemplar response type de-duplicated. Added SQL OR-filter and multi-series regression tests. --- docker-compose.yml | 2 + packages/app/.env.development | 1 + packages/app/src/HDXMultiSeriesTimeChart.tsx | 167 +++++++++--------- packages/app/src/api.ts | 2 +- .../ChartEditor/PromqlChartEditor.tsx | 49 ++--- .../ChartEditorControls.tsx | 18 +- packages/app/src/components/DBTimeChart.tsx | 5 + .../src/components/Exemplars/ExemplarDot.tsx | 5 +- .../__tests__/exemplarPoints.test.ts | 87 +++++++++ .../components/Exemplars/exemplarPoints.ts | 77 ++++++++ .../app/src/components/Exemplars/index.ts | 1 + .../TeamSettings/TeamQueryConfigSection.tsx | 43 +++-- packages/app/src/config.ts | 4 + .../src/hooks/__tests__/useExemplars.test.ts | 20 +++ packages/app/src/hooks/useExemplars.tsx | 37 ++-- .../src/__tests__/renderChartConfig.test.ts | 25 +++ .../src/core/renderChartConfig.ts | 16 +- 17 files changed, 410 insertions(+), 149 deletions(-) create mode 100644 packages/app/src/components/Exemplars/__tests__/exemplarPoints.test.ts create mode 100644 packages/app/src/components/Exemplars/exemplarPoints.ts diff --git a/docker-compose.yml b/docker-compose.yml index bdd7339e03..7a2ae855c5 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -74,6 +74,8 @@ services: # Uncomment the next two lines to enable PromQL (Prometheus-compatible metrics) # ENABLE_PROMQL: 'true' # NEXT_PUBLIC_ENABLE_PROMQL: 'true' + # Uncomment to enable exemplar overlays (trace markers on time charts) + # NEXT_PUBLIC_ENABLE_EXEMPLARS: 'true' DEFAULT_CONNECTIONS: '[{"name":"Local ClickHouse","host":"http://ch-server:8123","username":"default","password":""}]' diff --git a/packages/app/.env.development b/packages/app/.env.development index 7fd59cdfde..6c2de5a184 100644 --- a/packages/app/.env.development +++ b/packages/app/.env.development @@ -13,3 +13,4 @@ NEXT_PUBLIC_OTEL_EXPORTER_OTLP_ENDPOINT="http://localhost:${HDX_DEV_OTEL_HTTP_PO # NEXT_PUBLIC_OTEL_EXPORTER_OTLP_METRICS_ENDPOINT= # NEXT_PUBLIC_OTEL_EXPORTER_OTLP_LOGS_ENDPOINT= NEXT_PUBLIC_ENABLE_PROMQL=true +NEXT_PUBLIC_ENABLE_EXEMPLARS=true diff --git a/packages/app/src/HDXMultiSeriesTimeChart.tsx b/packages/app/src/HDXMultiSeriesTimeChart.tsx index bb50911545..41dc18f28f 100644 --- a/packages/app/src/HDXMultiSeriesTimeChart.tsx +++ b/packages/app/src/HDXMultiSeriesTimeChart.tsx @@ -46,7 +46,7 @@ import { toViewportPoint, useChartTooltipZIndex, } from './components/charts/ChartTooltip'; -import { ExemplarDot } from './components/Exemplars'; +import { computeExemplarPoints, ExemplarDot } from './components/Exemplars'; import { useChartSyncId } from './chartSync'; import { findNearestSeriesKey, @@ -595,10 +595,6 @@ function CaptureActiveDot({ ); } -function numOrNull(v: unknown): number | null { - return typeof v === 'number' && !isNaN(v) ? v : null; -} - /** * Compute the unique set of hexes referenced by `` defs * inside MemoChart. Exported so a unit test can pin the dedup-and-union @@ -796,6 +792,27 @@ export const MemoChart = memo(function MemoChart({ captureActivePointY, ]); + // Max value across the visible series. Exemplar markers are clamped to this so + // a single slow-trace outlier (which can be 100x the p99 line) can't stretch + // the y-axis and crush the series flat — the marker pins to the top of the + // series range while its hover card still shows the true duration. + const visibleSeriesMax = useMemo(() => { + const hasSelection = selectedSeriesNames && selectedSeriesNames.size > 0; + let max = -Infinity; + graphResults.forEach(dataPoint => { + lineData.forEach(ld => { + const seriesName = ld.displayName || ld.dataKey; + if (!hasSelection || selectedSeriesNames.has(seriesName)) { + const value = dataPoint[ld.dataKey]; + if (typeof value === 'number' && !isNaN(value)) { + max = Math.max(max, value); + } + } + }); + }); + return max; + }, [graphResults, lineData, selectedSeriesNames]); + const yAxisDomain: AxisDomain = useMemo(() => { const hasSelection = selectedSeriesNames && selectedSeriesNames.size > 0; @@ -805,39 +822,17 @@ export const MemoChart = memo(function MemoChart({ const shouldFitYAxis = fitYAxisToData && displayType !== DisplayType.StackedBar; - // Exemplars plot at the trace's own value, which can exceed the series - // (a slow request above the avg line), so they must influence the upper - // bound or they'd be clipped off-chart. - const exemplarValues = (exemplars ?? []) - .map(e => e.value) - .filter((v): v is number => typeof v === 'number' && !isNaN(v)); - const exemplarMax = exemplarValues.length - ? Math.max(...exemplarValues) - : -Infinity; - - // The data min/max is only needed to either zoom into a selection, fit the - // lower bound to the data, or make room for exemplar markers. When none - // apply, let Recharts auto-calculate the upper bound (lower pinned to zero). + // The domain follows the visible series only — exemplar markers are clamped + // to the series max at render, so they never need to widen the axis. When + // there's no selection or fit, let Recharts auto-scale (lower pinned to 0). if (!hasSelection && !shouldFitYAxis) { - if (exemplarMax === -Infinity) return [0, 'auto']; - // Need an explicit upper bound to include exemplars; derive it from the - // visible series max and the exemplar max. - let seriesMax = -Infinity; - graphResults.forEach(dataPoint => { - lineData.forEach(ld => { - const value = dataPoint[ld.dataKey]; - if (typeof value === 'number' && !isNaN(value)) { - seriesMax = Math.max(seriesMax, value); - } - }); - }); - return [0, Math.max(seriesMax, exemplarMax) * 1.05]; + return [0, 'auto']; } // Calculate domain based on visible series (all series when there's no // explicit selection). let minValue = Infinity; - let maxValue = exemplarMax; + let maxValue = -Infinity; graphResults.forEach(dataPoint => { lineData.forEach(ld => { @@ -870,7 +865,6 @@ export const MemoChart = memo(function MemoChart({ return ['auto', 'auto']; }, [ - exemplars, graphResults, lineData, selectedSeriesNames, @@ -973,6 +967,31 @@ export const MemoChart = memo(function MemoChart({ const [highlightEnd, setHighlightEnd] = useState(); const mouseDownPosRef = useRef(null); + // While the cursor is over an exemplar marker, the exemplar hover card owns + // the tooltip real estate — suppress the series hover tooltip so the two don't + // overlap. Wraps the parent's exemplar-hover callbacks to also track it here. + // Track the hovered marker by key (not just a boolean) so we can detect when a + // refetch/re-thinning unmounts it — React fires no mouseleave in that case, so + // the boolean would otherwise stick `true` and permanently suppress the series + // tooltip. The reset effect lives after `exemplarPoints` is computed. + const [hoveredExemplarKey, setHoveredExemplarKey] = useState( + null, + ); + const isExemplarHovered = hoveredExemplarKey != null; + const handleExemplarHoverStart = useCallback( + (exemplar: Exemplar, cx: number, cy: number) => { + setHoveredExemplarKey( + `exemplar-${exemplar.traceId}-${exemplar.timestamp}`, + ); + onExemplarHover?.(exemplar, cx, cy); + }, + [onExemplarHover], + ); + const handleExemplarHoverEnd = useCallback(() => { + setHoveredExemplarKey(null); + onExemplarHoverEnd?.(); + }, [onExemplarHoverEnd]); + // Tracks the time range that was displayed before the user brushed to zoom // in, so a "Reset zoom" control can restore it (mirrors Highcharts). It holds // the earliest pre-zoom range across consecutive zoom-ins so resetting jumps @@ -1039,53 +1058,29 @@ export const MemoChart = memo(function MemoChart({ // window. The window is coarser than the chart granularity so the count stays // readable even when every fine-grained bucket has an exemplar. // maxExemplars <= 0 means "unlimited" — show every exemplar (deduped). - const exemplarPoints = useMemo(() => { - type ExemplarPoint = { - x: number; - y: number; - exemplar: Exemplar; - key: string; - }; - if (!exemplars?.length) return [] as ExemplarPoint[]; - - const toPoint = (exemplar: Exemplar, value: number): ExemplarPoint => ({ - x: exemplar.timestamp / 1000, // ms -> seconds (chart x unit) - y: value, - exemplar, - key: `exemplar-${exemplar.traceId}-${exemplar.timestamp}`, - }); - - if (maxExemplars <= 0) { - const all = new Map(); - for (const exemplar of exemplars) { - const value = numOrNull(exemplar.value); - if (value == null) continue; - const p = toPoint(exemplar, value); - all.set(p.key, p); // dedupe identical trace+time - } - return Array.from(all.values()); - } - - const granMs = convertGranularityToSeconds(granularity) * 1000; - const rangeMs = dateRange[1].getTime() - dateRange[0].getTime(); - const bucketMs = Math.max( - granMs || 1, - rangeMs > 0 ? Math.floor(rangeMs / maxExemplars) : granMs || 1, - ); + const exemplarPoints = useMemo( + () => + computeExemplarPoints(exemplars, { + maxExemplars, + granularity, + dateRange, + }), + [exemplars, maxExemplars, granularity, dateRange], + ); - const bestPerBucket = new Map(); - for (const exemplar of exemplars) { - const value = numOrNull(exemplar.value); - if (value == null) continue; - const bucket = Math.floor(exemplar.timestamp / bucketMs); - const key = `${exemplar.groupKey ?? ''}@${bucket}`; - const existing = bestPerBucket.get(key); - if (!existing || value > existing.y) { - bestPerBucket.set(key, toPoint(exemplar, value)); - } + // If a refetch/re-thinning drops the hovered marker from the rendered set, its + // unmounts without a mouseleave. Reset the hover here (against the actual + // rendered points) so the series tooltip un-suppresses and the parent's hover + // card closes via onExemplarHoverEnd. + useEffect(() => { + if ( + hoveredExemplarKey != null && + !exemplarPoints.some(p => p.key === hoveredExemplarKey) + ) { + setHoveredExemplarKey(null); + onExemplarHoverEnd?.(); } - return Array.from(bestPerBucket.values()); - }, [exemplars, maxExemplars, granularity, dateRange]); + }, [exemplarPoints, hoveredExemplarKey, onExemplarHoverEnd]); const xAxisDomain: AxisDomain = useMemo(() => { let startTime = toStartOfInterval(dateRange[0], granularity); @@ -1367,7 +1362,7 @@ export const MemoChart = memo(function MemoChart({ Hidden once a point is clicked, where the pinned tooltip takes over. Portaled to body so HDXLineChartTooltip can self-position (see its docblock) and escape the chart's bounds near an edge. */} - {isClickActive == null && ( + {isClickActive == null && !isExemplarHovered && ( } /> diff --git a/packages/app/src/api.ts b/packages/app/src/api.ts index 0f656d2ad0..b8fafed44a 100644 --- a/packages/app/src/api.ts +++ b/packages/app/src/api.ts @@ -541,7 +541,7 @@ type PrometheusLabelValuesResponse = { // Native Prometheus /query_exemplars shape: one entry per series, each with its // own exemplar list. `labels` carries the trace/span id (label naming varies by // exporter, e.g. trace_id vs traceID — normalized downstream). -type PrometheusExemplarsResult = { +export type PrometheusExemplarsResult = { seriesLabels: PrometheusMetric; exemplars: { labels: PrometheusMetric; value: string; timestamp: number }[]; }; diff --git a/packages/app/src/components/ChartEditor/PromqlChartEditor.tsx b/packages/app/src/components/ChartEditor/PromqlChartEditor.tsx index c5c592a562..e35dcd5d8e 100644 --- a/packages/app/src/components/ChartEditor/PromqlChartEditor.tsx +++ b/packages/app/src/components/ChartEditor/PromqlChartEditor.tsx @@ -4,6 +4,7 @@ import { Box, Button, Flex, Stack, Switch, Text } from '@mantine/core'; import PromQLEditor from '@/components/PromQLEditor/PromQLEditor'; import { SourceSelectControlled } from '@/components/SourceSelect'; +import { IS_EXEMPLARS_ENABLED } from '@/config'; import { usePromqlMetricNames } from '@/hooks/usePromqlMetadata'; import { useSource } from '@/source'; @@ -63,29 +64,33 @@ export default function PromqlChartEditor({ - { - exemplarsField.onChange(exemplarsField.value !== true); - onSubmit(); - }} - /> - {exemplarsField.value === true && ( - - - Trace source - - + { + exemplarsField.onChange(exemplarsField.value !== true); + onSubmit(); + }} /> - + {exemplarsField.value === true && ( + + + Trace source + + + + )} + )}