diff --git a/.changeset/refactor-metric-ratios-cte.md b/.changeset/refactor-metric-ratios-cte.md new file mode 100644 index 0000000000..ee88c0bfec --- /dev/null +++ b/.changeset/refactor-metric-ratios-cte.md @@ -0,0 +1,5 @@ +--- +"@hyperdx/common-utils": patch +--- + +Refactor metric ratios to use native ClickHouse CTE diff --git a/packages/common-utils/src/__tests__/queryChartConfig.int.test.ts b/packages/common-utils/src/__tests__/queryChartConfig.int.test.ts index 5f76e73149..19a3eec441 100644 --- a/packages/common-utils/src/__tests__/queryChartConfig.int.test.ts +++ b/packages/common-utils/src/__tests__/queryChartConfig.int.test.ts @@ -247,6 +247,66 @@ describe('queryChartConfig Integration Tests', () => { } }); + it('computes ratio via native CTE when seriesReturnType is "ratio"', async () => { + const config: ChartConfigWithOptDateRange = { + displayType: DisplayType.Line, + connection: 'test-connection', + from: { databaseName: DATABASE, tableName: TABLE_NAME }, + metricTables: { [MetricsDataType.Gauge]: TABLE_NAME } as any, + seriesReturnType: 'ratio', + select: [ + { + aggFn: 'avg', + aggCondition: '', + aggConditionLanguage: 'sql', + valueExpression: 'Value', + metricName: 'metric.alpha', + metricType: MetricsDataType.Gauge, + alias: 'avg(metric.alpha)', + }, + { + aggFn: 'avg', + aggCondition: '', + aggConditionLanguage: 'sql', + valueExpression: 'Value', + metricName: 'metric.beta', + metricType: MetricsDataType.Gauge, + alias: 'avg(metric.beta)', + }, + ], + groupBy: [{ aggCondition: '', valueExpression: 'ServiceName' }], + where: '', + whereLanguage: 'sql', + timestampValueExpression: 'TimeUnix', + dateRange: [new Date('2025-04-14'), new Date('2025-04-16')], + granularity: '1 minute', + limit: { limit: 100 }, + }; + + const result = await hdxClient.queryChartConfig({ + config, + metadata, + querySettings: undefined, + }); + + const metaNames = result.meta?.map(m => m.name) ?? []; + + // Check that the ratio is the first column + expect(metaNames[0]).toBe('avg(metric.alpha)/avg(metric.beta)'); + expect(metaNames).toContain('__hdx_time_bucket'); + expect(metaNames).toContain('ServiceName'); + + const data = result.data as any[]; + expect(data.length).toBeGreaterThan(0); + for (const row of data) { + expect(row['avg(metric.alpha)/avg(metric.beta)']).toBeDefined(); + // It might be a number or string depending on ClickHouse formatting for JSON, usually number for Float64 + expect( + Number.isNaN(Number(row['avg(metric.alpha)/avg(metric.beta)'])), + ).toBe(false); + } + }); + // Regression: a comma-separated string group-by (with a Map access) must split // per-column (not emit toString(col1, col2)); empty-string groups are kept. it('handles a multi-column string group-by (with Map access) under seriesLimit', async () => { diff --git a/packages/common-utils/src/clickhouse/index.ts b/packages/common-utils/src/clickhouse/index.ts index 7e24cd5793..f0abef8a8e 100644 --- a/packages/common-utils/src/clickhouse/index.ts +++ b/packages/common-utils/src/clickhouse/index.ts @@ -834,9 +834,30 @@ export abstract class BaseClickhouseClient { }; querySettings: QuerySettings | undefined; }): Promise>> { - config = isBuilderChartConfig(config) - ? setChartSelectsAlias(config) - : config; + if (isBuilderChartConfig(config)) { + config = setChartSelectsAlias(config); + if (config.seriesReturnType === 'ratio' && config.ratioMode !== 'share_of_total') { + if (config.groupBy) { + if (typeof config.groupBy === 'string') { + config.groupBy = splitAndTrimWithBracket(config.groupBy).map(gb => ({ + type: 'string', + valueExpression: gb, + alias: gb, // Assign the raw expression as the alias so the CTE outputs exactly this column name + })); + } else if (Array.isArray(config.groupBy)) { + config.groupBy = config.groupBy.map(gb => { + if (typeof gb === 'string') { + return { type: 'string', valueExpression: gb, alias: gb }; + } + if (!gb.alias) { + return { ...gb, alias: gb.valueExpression }; + } + return gb; + }); + } + } + } + } const queries: ChSql[] = await Promise.all( splitChartConfigs(config).map(c => renderChartConfig(c, metadata, querySettings), @@ -845,6 +866,63 @@ export abstract class BaseClickhouseClient { const isTimeSeries = isTimeSeriesDisplayType(config.displayType); + if ( + isBuilderChartConfig(config) && + config.seriesReturnType === 'ratio' && + config.ratioMode !== 'share_of_total' && + queries.length === 2 && + Array.isArray(config.select) + ) { + const q0Alias = config.select[0].alias ?? 'q0_val'; + const originalQ1Alias = config.select[1].alias ?? 'q1_val'; + const ratioAlias = `${q0Alias}/${originalQ1Alias}`; + + const joinKeys: string[] = []; + if (isTimeSeries) { + joinKeys.push('__hdx_time_bucket'); + } + if (config.groupBy) { + for (const gb of config.groupBy) { + if (typeof gb === 'string') { + joinKeys.push(gb); + } else { + joinKeys.push(gb.alias || gb.valueExpression); + } + } + } + + // De-duplicate join keys just in case + const uniqueJoinKeys = Array.from(new Set(joinKeys)); + + let ratioSql: ChSql; + const selectCols = [ + chSql`(COALESCE(q0.${{ Identifier: q0Alias }}, 0) / q1.${{ Identifier: originalQ1Alias }}) AS ${{ Identifier: ratioAlias }}`, + ...uniqueJoinKeys.map(k => chSql`${{ Identifier: k }}`), + ]; + const selectClause = concatChSql(', ', selectCols); + + if (uniqueJoinKeys.length > 0) { + const joinKeysSql = uniqueJoinKeys.map( + k => chSql`${{ Identifier: k }}`, + ); + const usingClause = concatChSql(', ', joinKeysSql); + ratioSql = chSql`WITH q0 AS (${queries[0]}), q1 AS (${queries[1]}) SELECT ${selectClause} FROM q0 FULL OUTER JOIN q1 USING (${usingClause})`; + } else { + ratioSql = chSql`WITH q0 AS (${queries[0]}), q1 AS (${queries[1]}) SELECT ${selectClause} FROM q0 CROSS JOIN q1`; + } + + const resp = await this.query<'JSON'>({ + query: ratioSql.sql, + query_params: ratioSql.params, + format: 'JSON', + abort_signal: opts?.abort_signal, + connectionId: config.connection, + clickhouse_settings: opts?.clickhouse_settings, + }); + + return resp.json(); + } + const resultSets = await Promise.all( queries.map(async query => { const resp = await this.query<'JSON'>({