refactor(perf): offload metric ratio computation to native ClickHouse CTE

.changeset/refactor-metric-ratios-cte.md

@@ -0,0 +1,5 @@
+---
+"@hyperdx/common-utils": patch
+---
+
+Refactor metric ratios to use native ClickHouse CTE

packages/common-utils/src/__tests__/queryChartConfig.int.test.ts

@@ -247,6 +247,66 @@ describe('queryChartConfig Integration Tests', () => {
     }
   });
 
+  it('computes ratio via native CTE when seriesReturnType is "ratio"', async () => {
+    const config: ChartConfigWithOptDateRange = {
+      displayType: DisplayType.Line,
+      connection: 'test-connection',
+      from: { databaseName: DATABASE, tableName: TABLE_NAME },
+      metricTables: { [MetricsDataType.Gauge]: TABLE_NAME } as any,
+      seriesReturnType: 'ratio',
+      select: [
+        {
+          aggFn: 'avg',
+          aggCondition: '',
+          aggConditionLanguage: 'sql',
+          valueExpression: 'Value',
+          metricName: 'metric.alpha',
+          metricType: MetricsDataType.Gauge,
+          alias: 'avg(metric.alpha)',
+        },
+        {
+          aggFn: 'avg',
+          aggCondition: '',
+          aggConditionLanguage: 'sql',
+          valueExpression: 'Value',
+          metricName: 'metric.beta',
+          metricType: MetricsDataType.Gauge,
+          alias: 'avg(metric.beta)',
+        },
+      ],
+      groupBy: [{ aggCondition: '', valueExpression: 'ServiceName' }],
+      where: '',
+      whereLanguage: 'sql',
+      timestampValueExpression: 'TimeUnix',
+      dateRange: [new Date('2025-04-14'), new Date('2025-04-16')],
+      granularity: '1 minute',
+      limit: { limit: 100 },
+    };
+
+    const result = await hdxClient.queryChartConfig({
+      config,
+      metadata,
+      querySettings: undefined,
+    });
+
+    const metaNames = result.meta?.map(m => m.name) ?? [];
+
+    // Check that the ratio is the first column
+    expect(metaNames[0]).toBe('avg(metric.alpha)/avg(metric.beta)');
+    expect(metaNames).toContain('__hdx_time_bucket');
+    expect(metaNames).toContain('ServiceName');
+
+    const data = result.data as any[];
+    expect(data.length).toBeGreaterThan(0);
+    for (const row of data) {
+      expect(row['avg(metric.alpha)/avg(metric.beta)']).toBeDefined();
+      // It might be a number or string depending on ClickHouse formatting for JSON, usually number for Float64
+      expect(
+        Number.isNaN(Number(row['avg(metric.alpha)/avg(metric.beta)'])),
+      ).toBe(false);
+    }
+  });
+
   // Regression: a comma-separated string group-by (with a Map access) must split
   // per-column (not emit toString(col1, col2)); empty-string groups are kept.
   it('handles a multi-column string group-by (with Map access) under seriesLimit', async () => {

packages/common-utils/src/clickhouse/index.ts

@@ -834,9 +834,30 @@ export abstract class BaseClickhouseClient {
     };
     querySettings: QuerySettings | undefined;
   }): Promise<ResponseJSON<Record<string, string | number>>> {
-    config = isBuilderChartConfig(config)
-      ? setChartSelectsAlias(config)
-      : config;
+    if (isBuilderChartConfig(config)) {
+      config = setChartSelectsAlias(config);
+      if (config.seriesReturnType === 'ratio' && config.ratioMode !== 'share_of_total') {
+        if (config.groupBy) {
+          if (typeof config.groupBy === 'string') {
+            config.groupBy = splitAndTrimWithBracket(config.groupBy).map(gb => ({
+              type: 'string',
+              valueExpression: gb,
+              alias: gb, // Assign the raw expression as the alias so the CTE outputs exactly this column name
+            }));
+          } else if (Array.isArray(config.groupBy)) {
+            config.groupBy = config.groupBy.map(gb => {
+              if (typeof gb === 'string') {
+                return { type: 'string', valueExpression: gb, alias: gb };
+              }
+              if (!gb.alias) {
+                return { ...gb, alias: gb.valueExpression };
+              }
+              return gb;
+            });
+          }
+        }
+      }
+    }
     const queries: ChSql[] = await Promise.all(
       splitChartConfigs(config).map(c =>
         renderChartConfig(c, metadata, querySettings),
@@ -845,6 +866,63 @@ export abstract class BaseClickhouseClient {
 
     const isTimeSeries = isTimeSeriesDisplayType(config.displayType);
 
+    if (
+      isBuilderChartConfig(config) &&
+      config.seriesReturnType === 'ratio' &&
+      config.ratioMode !== 'share_of_total' &&
+      queries.length === 2 &&
+      Array.isArray(config.select)
+    ) {
+      const q0Alias = config.select[0].alias ?? 'q0_val';
+      const originalQ1Alias = config.select[1].alias ?? 'q1_val';
+      const ratioAlias = `${q0Alias}/${originalQ1Alias}`;
+
+      const joinKeys: string[] = [];
+      if (isTimeSeries) {
+        joinKeys.push('__hdx_time_bucket');
+      }
+      if (config.groupBy) {
+        for (const gb of config.groupBy) {
+          if (typeof gb === 'string') {
+            joinKeys.push(gb);
+          } else {
+            joinKeys.push(gb.alias || gb.valueExpression);
+          }
+        }
+      }
+
+      // De-duplicate join keys just in case
+      const uniqueJoinKeys = Array.from(new Set(joinKeys));
+
+      let ratioSql: ChSql;
+      const selectCols = [
+        chSql`(COALESCE(q0.${{ Identifier: q0Alias }}, 0) / q1.${{ Identifier: originalQ1Alias }}) AS ${{ Identifier: ratioAlias }}`,
+        ...uniqueJoinKeys.map(k => chSql`${{ Identifier: k }}`),
+      ];
+      const selectClause = concatChSql(', ', selectCols);
+
+      if (uniqueJoinKeys.length > 0) {
+        const joinKeysSql = uniqueJoinKeys.map(
+          k => chSql`${{ Identifier: k }}`,
+        );
+        const usingClause = concatChSql(', ', joinKeysSql);
+        ratioSql = chSql`WITH q0 AS (${queries[0]}), q1 AS (${queries[1]}) SELECT ${selectClause} FROM q0 FULL OUTER JOIN q1 USING (${usingClause})`;
+      } else {
+        ratioSql = chSql`WITH q0 AS (${queries[0]}), q1 AS (${queries[1]}) SELECT ${selectClause} FROM q0 CROSS JOIN q1`;
+      }
+
+      const resp = await this.query<'JSON'>({
+        query: ratioSql.sql,
+        query_params: ratioSql.params,
+        format: 'JSON',
+        abort_signal: opts?.abort_signal,
+        connectionId: config.connection,
+        clickhouse_settings: opts?.clickhouse_settings,
+      });
+
+      return resp.json<any>();
+    }
+
     const resultSets = await Promise.all(
       queries.map(async query => {
         const resp = await this.query<'JSON'>({