diff --git a/.changeset/sql-to-builder-conversion.md b/.changeset/sql-to-builder-conversion.md new file mode 100644 index 0000000000..ffa7b73130 --- /dev/null +++ b/.changeset/sql-to-builder-conversion.md @@ -0,0 +1,6 @@ +--- +'@hyperdx/common-utils': minor +'@hyperdx/app': minor +--- + +feat: Support conversion from Raw SQL to Builder charts \ No newline at end of file diff --git a/packages/app/src/components/ChartEditor/types.ts b/packages/app/src/components/ChartEditor/types.ts index 4e22b22d3c..0399a984f6 100644 --- a/packages/app/src/components/ChartEditor/types.ts +++ b/packages/app/src/components/ChartEditor/types.ts @@ -1,3 +1,4 @@ +import { FieldPath, FieldPathValue, SetValueConfig } from 'react-hook-form'; import { BuilderSavedChartConfig, PromqlSavedChartConfig, @@ -35,3 +36,30 @@ export type ChartEditorFormState = Partial & series: SavedChartConfigWithSelectArray['select']; configType?: 'sql' | 'builder' | 'promql'; }; + +/** + * Options accepted by {@link ChartFormSetValue}: react-hook-form's standard + * `setValue` config plus `isUserChange`, which records that the write came from + * a genuine user edit (see below). + */ +export type ChartFormSetValueOptions = SetValueConfig & { + isUserChange?: boolean; +}; + +/** + * A `setValue` wrapper for the chart editor form that also records whether the + * write came from a genuine user edit. Builder ⇄ SQL conversion uses this to + * tell user edits apart from the many programmatic `setValue` writes (defaults, + * resets, its own generated output). Pass `{ isUserChange: true }` only when the + * user directly changed the field via a control that isn't a registered + * react-hook-form input (toggles, pickers, table sorts, helper buttons, etc.). + * + * The flag lives in the options object (rather than a positional argument) so + * this stays assignable to `UseFormSetValue` and can be threaded through + * components that only forward `setValue` without any changes. + */ +export type ChartFormSetValue = >( + name: TName, + value: FieldPathValue, + options?: ChartFormSetValueOptions, +) => void; diff --git a/packages/app/src/components/DBEditTimeChartForm/ChartEditorControls.tsx b/packages/app/src/components/DBEditTimeChartForm/ChartEditorControls.tsx index 4ed805c6ff..613254e7b6 100644 --- a/packages/app/src/components/DBEditTimeChartForm/ChartEditorControls.tsx +++ b/packages/app/src/components/DBEditTimeChartForm/ChartEditorControls.tsx @@ -4,7 +4,6 @@ import { FieldArrayWithId, FieldErrors, UseFormClearErrors, - UseFormSetValue, useWatch, } from 'react-hook-form'; import { TableConnection } from '@hyperdx/common-utils/dist/core/metadata'; @@ -23,6 +22,7 @@ import { IconBell, IconCirclePlus } from '@tabler/icons-react'; import { ChartEditorFormState, + ChartFormSetValue, SavedChartConfigWithSelectArray, } from '@/components/ChartEditor/types'; import MVOptimizationIndicator from '@/components/MaterializedViews/MVOptimizationIndicator'; @@ -44,7 +44,7 @@ import { buildGroupByConnectionProps } from './utils'; type ChartEditorControlsProps = { control: Control; - setValue: UseFormSetValue; + setValue: ChartFormSetValue; clearErrors: UseFormClearErrors; errors: FieldErrors; fields: FieldArrayWithId[]; @@ -203,7 +203,7 @@ export function ChartEditorControls({ name="where" onSubmit={onSubmit} onLanguageChange={(lang: 'sql' | 'lucene') => - setValue('whereLanguage', lang) + setValue('whereLanguage', lang, { isUserChange: true }) } showLabel={false} /> @@ -335,6 +335,7 @@ export function ChartEditorControls({ setValue( 'seriesReturnType', seriesReturnType === 'ratio' ? 'column' : 'ratio', + { isUserChange: true }, ); onSubmit(); }} @@ -433,7 +434,7 @@ export function ChartEditorControls({ name="where" onSubmit={onSubmit} onLanguageChange={(lang: 'sql' | 'lucene') => - setValue('whereLanguage', lang) + setValue('whereLanguage', lang, { isUserChange: true }) } showLabel={false} /> diff --git a/packages/app/src/components/DBEditTimeChartForm/ChartPreviewPanel.tsx b/packages/app/src/components/DBEditTimeChartForm/ChartPreviewPanel.tsx index f7cacf916f..e802c759e0 100644 --- a/packages/app/src/components/DBEditTimeChartForm/ChartPreviewPanel.tsx +++ b/packages/app/src/components/DBEditTimeChartForm/ChartPreviewPanel.tsx @@ -16,7 +16,10 @@ import { SortingState } from '@tanstack/react-table'; import { buildTableRowSearchUrl } from '@/ChartUtils'; import { getAlertReferenceLines } from '@/components/Alerts'; -import { ChartEditorFormState } from '@/components/ChartEditor/types'; +import { + ChartEditorFormState, + ChartFormSetValueOptions, +} from '@/components/ChartEditor/types'; import ChartSQLPreview from '@/components/ChartSQLPreview'; import { DBBarChart } from '@/components/DBBarChart'; import DBHeatmapChart, { @@ -128,7 +131,11 @@ type ChartPreviewPanelProps = { showGeneratedSql: boolean; showSampleEvents: boolean; dbTimeChartConfig?: ChartConfigWithDateRange; - setValue: (name: 'orderBy', value: string) => void; + setValue: ( + name: 'orderBy', + value: string, + options?: ChartFormSetValueOptions, + ) => void; onSubmit: () => void; }; @@ -153,7 +160,9 @@ export function ChartPreviewPanel({ const onTableSortingChange = useCallback( (sortState: SortingState | null) => { - setValue('orderBy', sortingStateToOrderByString(sortState) ?? ''); + setValue('orderBy', sortingStateToOrderByString(sortState) ?? '', { + isUserChange: true, + }); onSubmit(); }, [setValue, onSubmit], diff --git a/packages/app/src/components/DBEditTimeChartForm/ChartSeriesEditor.tsx b/packages/app/src/components/DBEditTimeChartForm/ChartSeriesEditor.tsx index d0eab77d64..3f3c67bfb1 100644 --- a/packages/app/src/components/DBEditTimeChartForm/ChartSeriesEditor.tsx +++ b/packages/app/src/components/DBEditTimeChartForm/ChartSeriesEditor.tsx @@ -3,7 +3,6 @@ import { Control, FieldErrors, UseFormClearErrors, - UseFormSetValue, useWatch, } from 'react-hook-form'; import { @@ -33,6 +32,7 @@ import { AGG_FNS } from '@/ChartUtils'; import { AggFnSelectControlled } from '@/components/AggFnSelect'; import { ChartEditorFormState, + ChartFormSetValue, SavedChartConfigWithSelectArray, } from '@/components/ChartEditor/types'; import { @@ -68,7 +68,7 @@ type ChartSeriesEditorProps = { onSwapSeries: (from: number, to: number) => void; onDuplicateSeries: (index: number) => void; onSubmit: () => void; - setValue: UseFormSetValue; + setValue: ChartFormSetValue; showGroupBy: boolean; showHaving: boolean; showDuplicate: boolean; @@ -175,7 +175,7 @@ export function ChartSeriesEditor({ const currentValue = aggCondition || ''; const newValue = currentValue ? `${currentValue} AND ${clause}` : clause; - setValue(`${namePrefix}aggCondition`, newValue); + setValue(`${namePrefix}aggCondition`, newValue, { isUserChange: true }); onSubmit(); }, [aggCondition, namePrefix, setValue, onSubmit], @@ -185,7 +185,7 @@ export function ChartSeriesEditor({ (clause: string) => { const currentValue = groupBy || ''; const newValue = currentValue ? `${currentValue}, ${clause}` : clause; - setValue('groupBy', newValue); + setValue('groupBy', newValue, { isUserChange: true }); onSubmit(); }, [groupBy, setValue, onSubmit], @@ -316,11 +316,17 @@ export function ChartSeriesEditor({ metricName={metricName} metricType={metricType} setMetricName={value => { - setValue(`${namePrefix}metricName`, value); - setValue(`${namePrefix}valueExpression`, 'Value'); + setValue(`${namePrefix}metricName`, value, { + isUserChange: true, + }); + setValue(`${namePrefix}valueExpression`, 'Value', { + isUserChange: true, + }); }} setMetricType={value => - setValue(`${namePrefix}metricType`, value) + setValue(`${namePrefix}metricType`, value, { + isUserChange: true, + }) } metricSource={tableSource} data-testid="metric-name-selector" diff --git a/packages/app/src/components/DBEditTimeChartForm/EditTimeChartForm.tsx b/packages/app/src/components/DBEditTimeChartForm/EditTimeChartForm.tsx index 3af1e2f2a9..304253a12e 100644 --- a/packages/app/src/components/DBEditTimeChartForm/EditTimeChartForm.tsx +++ b/packages/app/src/components/DBEditTimeChartForm/EditTimeChartForm.tsx @@ -83,7 +83,7 @@ import { ChartActionBar } from './ChartActionBar'; import { ChartEditorControls } from './ChartEditorControls'; import { ChartPreviewPanel } from './ChartPreviewPanel'; import { ErrorNotificationMessage } from './ErrorNotificationMessage'; -import { useBuilderToSqlConversion } from './useBuilderToSqlConversion'; +import { useBuilderSqlConversion } from './useBuilderSqlConversion'; import { buildChartConfigForExplanations, computeDbTimeChartConfig, @@ -158,7 +158,7 @@ export default function EditTimeChartForm({ setValue, getValues, // The callback form of `watch` is used to subscribe to field changes - // (without re-rendering) in useBuilderToSqlConversion; useWatch can't do this. + // (without re-rendering) in useBuilderSqlConversion; useWatch can't do this. // eslint-disable-next-line react-hook-form/no-use-watch watch, handleSubmit, @@ -234,8 +234,13 @@ export default function EditTimeChartForm({ const databaseName = tableSource?.from.databaseName; const tableName = tableSource?.from.tableName; - // Carry the builder config over as a SQL template when switching to SQL mode - useBuilderToSqlConversion({ + // Carry the query over between builder and SQL modes when the config type is + // toggled (builder → SQL template, and SQL template → builder fields). + // `setValueWithEditTracking` wraps `setValue` and additionally records genuine + // user edits (passed with `isUserChange: true`) so builder ⇄ SQL conversion + // knows which representation is freshest. Hand it to the form controls in + // place of the raw `setValue`. + const { setValue: setValueWithEditTracking } = useBuilderSqlConversion({ control, getValues, setValue, @@ -637,15 +642,26 @@ export default function EditTimeChartForm({ if (numberFormat !== undefined) { setValue('numberFormat', numberFormat); } - setValue('alignDateRangeToGranularity', alignDateRangeToGranularity); - setValue('fillNulls', fillNulls); - setValue('compareToPreviousPeriod', compareToPreviousPeriod); + // These affect the generated query, so mark them as user edits. + setValueWithEditTracking( + 'alignDateRangeToGranularity', + alignDateRangeToGranularity, + { isUserChange: true }, + ); + setValueWithEditTracking('fillNulls', fillNulls, { isUserChange: true }); + setValueWithEditTracking( + 'compareToPreviousPeriod', + compareToPreviousPeriod, + { isUserChange: true }, + ); setValue('fitYAxisToData', fitYAxisToData); setValue('groupByColumnsOnLeft', groupByColumnsOnLeft); // Persist `null` (not undefined) when cleared so the disabled state // survives JSON round-tripping through the URL query state; otherwise // the dropped key lets RHF's `values` sync restore the stale value. - setValue('seriesLimit', seriesLimit ?? null); + setValueWithEditTracking('seriesLimit', seriesLimit ?? null, { + isUserChange: true, + }); setValue('color', color); setValue('colorRules', colorRules); setValue('backgroundChart', backgroundChart); @@ -657,21 +673,29 @@ export default function EditTimeChartForm({ } onSubmit(); }, - [setValue, onDirtyChange, onSubmit], + [setValue, setValueWithEditTracking, onDirtyChange, onSubmit], ); const handleUpdateHeatmapSettings = useCallback( (data: HeatmapSettingsValues) => { - setValue('series.0.valueExpression', data.value); - setValue('series.0.countExpression', data.count || 'count()'); - setValue('series.0.heatmapScaleType', data.scaleType); + setValueWithEditTracking('series.0.valueExpression', data.value, { + isUserChange: true, + }); + setValueWithEditTracking( + 'series.0.countExpression', + data.count || 'count()', + { isUserChange: true }, + ); + setValueWithEditTracking('series.0.heatmapScaleType', data.scaleType, { + isUserChange: true, + }); // Heatmap settings are applied outside RHF's change tracking. subFormDirty.current = true; onDirtyChange?.(true); onSubmit(); closeHeatmapSettings(); }, - [setValue, onDirtyChange, onSubmit, closeHeatmapSettings], + [setValueWithEditTracking, onDirtyChange, onSubmit, closeHeatmapSettings], ); const heatmapValueExpression = useWatch({ @@ -838,7 +862,7 @@ export default function EditTimeChartForm({ ) : isRawSqlInput ? ( setValue(name, value)} + setValue={setValueWithEditTracking} onSubmit={onSubmit} /> ; - setValue: UseFormSetValue; + setValue: ChartFormSetValue; tableSource?: TSource; onSubmit: () => void; onOpenDisplaySettings: () => void; @@ -32,7 +35,7 @@ export function HeatmapSeriesEditor({ name="where" onSubmit={onSubmit} onLanguageChange={(lang: 'sql' | 'lucene') => - setValue('whereLanguage', lang) + setValue('whereLanguage', lang, { isUserChange: true }) } showLabel={false} /> diff --git a/packages/app/src/components/DBEditTimeChartForm/__tests__/useBuilderToSqlTemplate.test.ts b/packages/app/src/components/DBEditTimeChartForm/__tests__/useBuilderSqlConversion.test.ts similarity index 98% rename from packages/app/src/components/DBEditTimeChartForm/__tests__/useBuilderToSqlTemplate.test.ts rename to packages/app/src/components/DBEditTimeChartForm/__tests__/useBuilderSqlConversion.test.ts index ae2037fb36..8eaf1e9342 100644 --- a/packages/app/src/components/DBEditTimeChartForm/__tests__/useBuilderToSqlTemplate.test.ts +++ b/packages/app/src/components/DBEditTimeChartForm/__tests__/useBuilderSqlConversion.test.ts @@ -1,4 +1,4 @@ -import { classifyFormEdit } from '@/components/DBEditTimeChartForm/useBuilderToSqlConversion'; +import { classifyFormEdit } from '@/components/DBEditTimeChartForm/useBuilderSqlConversion'; describe('classifyFormEdit', () => { it('classifies a SQL editor change in SQL mode as a SQL edit', () => { diff --git a/packages/app/src/components/DBEditTimeChartForm/useBuilderSqlConversion.ts b/packages/app/src/components/DBEditTimeChartForm/useBuilderSqlConversion.ts new file mode 100644 index 0000000000..9b06710a96 --- /dev/null +++ b/packages/app/src/components/DBEditTimeChartForm/useBuilderSqlConversion.ts @@ -0,0 +1,305 @@ +import { useCallback, useEffect, useRef } from 'react'; +import { + Control, + UseFormGetValues, + UseFormSetValue, + UseFormWatch, + useWatch, +} from 'react-hook-form'; +import { + renderBuilderConfigAsSqlTemplate, + RenderedSqlTemplate, +} from '@hyperdx/common-utils/dist/core/builderToRawSql'; +import { + convertRawSqlToBuilderConfig, + SqlToBuilderError, +} from '@hyperdx/common-utils/dist/core/rawSqlToBuilder'; +import { DisplayType, TSource } from '@hyperdx/common-utils/dist/types'; +import { usePrevious } from '@mantine/hooks'; +import { notifications } from '@mantine/notifications'; + +import { + ChartEditorFormState, + ChartFormSetValue, +} from '@/components/ChartEditor/types'; +import { convertFormStateToChartConfig } from '@/components/ChartEditor/utils'; +import { useMetadataWithSettings } from '@/hooks/useMetadata'; + +type ConfigType = ChartEditorFormState['configType']; + +/** The mode a chart edit belongs to for conversion purposes. */ +export type EditedMode = 'builder' | 'sql'; + +/** + * Fields that belong to neither the builder query nor the SQL editor (they're + * shared chart metadata), so editing them must not count as a "mode edit". + */ +const MODE_AGNOSTIC_FIELDS = new Set(['name', 'configType']); + +/** + * Classify a react-hook-form field change as a builder-mode edit, a SQL-mode + * edit, or neither. Only genuine user input (type='change') counts as an edit. + */ +export function classifyFormEdit({ + name, + type, + configType, +}: { + name?: string; + type?: string; + configType?: ConfigType; +}): EditedMode | null { + if (type !== 'change' || !name || MODE_AGNOSTIC_FIELDS.has(name)) { + return null; + } + // In SQL mode only the SQL editor is a query edit; in builder mode every + // query field except the SQL editor is (a denylist, so new builder fields + // are covered automatically). + if (configType === 'sql') { + return name === 'sqlTemplate' ? 'sql' : null; + } + if (configType === 'builder') { + return name === 'sqlTemplate' ? null : 'builder'; + } + return null; +} + +/** + * Two-way carry-over between the chart editor's builder config and its raw-SQL + * template. When the user toggles the config type, the query they were most + * recently editing is converted into the other representation: + * + * - **Builder → SQL**: switching to SQL mode generates a macro-based SQL + * template from the current builder config and populates the SQL editor. + * - **SQL → Builder**: switching to builder mode parses the current SQL + * template back into builder fields. + * + * Each direction only fires when the user edited that source mode more recently + * than the target mode, so a hand-edited query is never clobbered by a + * regeneration derived from the stale other side. Either direction failing + * surfaces a notification and leaves the target representation untouched, so an + * unconvertible query never destroys the config the user already had. + */ +export function useBuilderSqlConversion({ + control, + getValues, + setValue, + watch, + tableSource, +}: { + control: Control; + getValues: UseFormGetValues; + setValue: UseFormSetValue; + watch: UseFormWatch; + tableSource: TSource | undefined; +}) { + const metadata = useMetadataWithSettings(); + + const configType = useWatch({ control, name: 'configType' }); + const prevConfigType = usePrevious(configType); + + // Which mode has edits not yet synced to the other representation. `null` + // means the two are in sync (e.g. right after a successful conversion), so + // neither direction should re-convert until the user edits one side again. + const lastEditedModeRef = useRef( + getValues('configType') === 'sql' ? 'sql' : 'builder', + ); + + // A monotonically increasing request ID to ensure that only the latest + // generated SQL is written to the form. + const requestIdRef = useRef(0); + + // Track the most-recently-edited mode from genuine user input. + useEffect(() => { + const subscription = watch((_values, { name, type }) => { + const editMode = classifyFormEdit({ + name, + type, + configType: getValues('configType'), + }); + if (editMode) { + lastEditedModeRef.current = editMode; + } + }); + return () => subscription.unsubscribe(); + }, [watch, getValues]); + + // A `setValue` wrapper handed to the form controls. Controls that mutate a + // query field without going through a registered react-hook-form input (so the + // `watch` subscription above never sees a `type: 'change'` event) pass + // `isUserChange: true`, which records the edit here so the correct direction + // re-converts on the next mode toggle. Programmatic writes omit the flag and + // behave like a plain `setValue`. + const setValueWithEditTracking = useCallback( + (name, value, options) => { + const { isUserChange, ...setValueOptions } = options ?? {}; + setValue(name, value, setValueOptions); + if (!isUserChange) { + return; + } + const editMode = classifyFormEdit({ + name, + type: 'change', + configType: getValues('configType'), + }); + if (editMode) { + lastEditedModeRef.current = editMode; + } + }, + [setValue, getValues], + ); + + // Builder → SQL: regenerate the SQL template when switching to SQL mode. + useEffect(() => { + // Only generate a new SQL template when switching from builder mode to SQL mode. + if (prevConfigType !== 'builder' || configType !== 'sql') { + return; + } + + const showError = (message: string, err?: Error) => { + console.warn('Could not convert chart to SQL', { + form: getValues(), + err, + }); + notifications.show({ + id: 'builder-to-sql-error', + title: 'Could not auto-convert to SQL', + message, + color: 'red', + }); + }; + + // Only overwrite when the user edited in builder more recently than in SQL. + if (lastEditedModeRef.current !== 'builder') { + return; + } + + // A resolved source is required to build a ChartConfig from the form; + // every other reason a config can't be converted is reported by + // renderBuilderConfigAsSqlTemplate below. + if (!tableSource) { + showError('Auto-converting to SQL requires a source to be selected.'); + return; + } + + // Build a ChartConfig from the current form state + const form = getValues(); + const config = convertFormStateToChartConfig( + { ...form, configType: 'builder' }, + // dateRange is irrelevant here, since the SQL will contain date range macros + [new Date(0), new Date(0)], + tableSource, + ); + if (!config) return; + + const requestId = ++requestIdRef.current; + const applyResult = (result: RenderedSqlTemplate) => { + // Ignore results from generations that a newer switch has superseded. + if (requestId !== requestIdRef.current) { + return; + } + + if (result.isError) { + showError(result.error); + return; + } + + // Only write while still in SQL mode, and don't clobber a hand-edit the + // user made while generation was in flight. + if ( + getValues('configType') === 'sql' && + lastEditedModeRef.current === 'builder' + ) { + setValue('sqlTemplate', result.sql); + // The two representations are now in sync; neither has pending edits, so + // don't re-convert until the user edits one side again. + lastEditedModeRef.current = null; + notifications.show({ + title: 'Chart converted to SQL', + message: 'The existing chart configuration has been converted to SQL', + color: 'green', + }); + } + }; + + renderBuilderConfigAsSqlTemplate(config, metadata) + .then(applyResult) + .catch(e => showError('Chart could not be auto-converted to SQL', e)); + }, [configType, tableSource, metadata, getValues, setValue, prevConfigType]); + + // SQL → Builder: parse the SQL template back into builder fields when + // switching to builder mode (the inverse of the Builder → SQL effect above). + useEffect(() => { + // Only convert when switching from SQL mode to builder mode. + if (prevConfigType !== 'sql' || configType !== 'builder') { + return; + } + + // Only convert when the user edited in SQL more recently than in builder, + // so a builder config the user was working on is never clobbered by a + // conversion derived from stale (e.g. auto-generated) SQL. + if (lastEditedModeRef.current !== 'sql') { + return; + } + + const form = getValues(); + const displayType = form.displayType ?? DisplayType.Line; + const opts = { shouldDirty: true } as const; + + try { + const result = convertRawSqlToBuilderConfig({ + sqlTemplate: form.sqlTemplate ?? '', + displayType, + from: tableSource?.from, + timestampValueExpression: tableSource?.timestampValueExpression, + }); + + const series = result.select.map(s => ({ + ...s, + aggConditionLanguage: s.aggConditionLanguage ?? 'sql', + })); + setValue('select', series, opts); + setValue('series', series, opts); + setValue('where', result.where, opts); + setValue('whereLanguage', result.whereLanguage, opts); + setValue('groupBy', result.groupBy, opts); + setValue('granularity', result.granularity, opts); + setValue('having', result.having, opts); + setValue('havingLanguage', result.havingLanguage, opts); + setValue('orderBy', result.orderBy, opts); + setValue('limit', result.limit, opts); + setValue('seriesReturnType', result.seriesReturnType ?? 'column', opts); + + // The two representations are now in sync; neither has pending edits, so + // don't re-convert until the user edits one side again. + lastEditedModeRef.current = null; + + notifications.show({ + id: 'sql-to-builder', + title: 'Converted SQL to builder', + message: 'The SQL query was converted into a builder chart.', + color: 'green', + }); + } catch (e) { + const message = + e instanceof SqlToBuilderError + ? e.message + : 'The SQL query could not be converted to the builder.'; + // Logged so that failures can be monitored and fixed + console.warn('Could not convert SQL to builder', { + sqlTemplate: form.sqlTemplate ?? '', + displayType, + reason: message, + err: e, + }); + notifications.show({ + id: 'sql-to-builder', + title: 'Could not convert SQL to builder', + message, + color: 'yellow', + }); + } + }, [configType, prevConfigType, getValues, setValue, tableSource]); + + return { setValue: setValueWithEditTracking }; +} diff --git a/packages/app/src/components/DBEditTimeChartForm/useBuilderToSqlConversion.ts b/packages/app/src/components/DBEditTimeChartForm/useBuilderToSqlConversion.ts deleted file mode 100644 index 36eed01ea3..0000000000 --- a/packages/app/src/components/DBEditTimeChartForm/useBuilderToSqlConversion.ts +++ /dev/null @@ -1,184 +0,0 @@ -import { useEffect, useRef } from 'react'; -import { - Control, - UseFormGetValues, - UseFormSetValue, - UseFormWatch, - useWatch, -} from 'react-hook-form'; -import { - renderBuilderConfigAsSqlTemplate, - RenderedSqlTemplate, -} from '@hyperdx/common-utils/dist/core/builderToRawSql'; -import { TSource } from '@hyperdx/common-utils/dist/types'; -import { usePrevious } from '@mantine/hooks'; -import { notifications } from '@mantine/notifications'; - -import { ChartEditorFormState } from '@/components/ChartEditor/types'; -import { convertFormStateToChartConfig } from '@/components/ChartEditor/utils'; -import { useMetadataWithSettings } from '@/hooks/useMetadata'; - -type ConfigType = ChartEditorFormState['configType']; - -/** The mode a chart edit belongs to for conversion purposes. */ -export type EditedMode = 'builder' | 'sql'; - -/** - * Fields that belong to neither the builder query nor the SQL editor (they're - * shared chart metadata), so editing them must not count as a "mode edit". - */ -const MODE_AGNOSTIC_FIELDS = new Set(['name', 'configType']); - -/** - * Classify a react-hook-form field change as a builder-mode edit, a SQL-mode - * edit, or neither. Only genuine user input (type='change') counts as an edit. - */ -export function classifyFormEdit({ - name, - type, - configType, -}: { - name?: string; - type?: string; - configType?: ConfigType; -}): EditedMode | null { - if (type !== 'change' || !name || MODE_AGNOSTIC_FIELDS.has(name)) { - return null; - } - // In SQL mode only the SQL editor is a query edit; in builder mode every - // query field except the SQL editor is (a denylist, so new builder fields - // are covered automatically). - if (configType === 'sql') { - return name === 'sqlTemplate' ? 'sql' : null; - } - if (configType === 'builder') { - return name === 'sqlTemplate' ? null : 'builder'; - } - return null; -} - -/** - * One-way Builder → SQL carry-over: when the chart editor switches from builder - * mode to SQL mode, generates a macro-based SQL template from the current - * builder config and populates the SQL editor with it. - * - * The SQL is regenerated only when the user edited the chart in builder mode - * more recently than in SQL mode. Configs that can't be converted surface a - * notification and leave the SQL editor untouched. - */ -export function useBuilderToSqlConversion({ - control, - getValues, - setValue, - watch, - tableSource, -}: { - control: Control; - getValues: UseFormGetValues; - setValue: UseFormSetValue; - watch: UseFormWatch; - tableSource: TSource | undefined; -}) { - const metadata = useMetadataWithSettings(); - - const configType = useWatch({ control, name: 'configType' }); - const prevConfigType = usePrevious(configType); - - // Which mode the user most recently edited in - const lastEditedModeRef = useRef( - getValues('configType') === 'sql' ? 'sql' : 'builder', - ); - - // A monotonically increasing request ID to ensure that only the latest - // generated SQL is written to the form. - const requestIdRef = useRef(0); - - // Track the most-recently-edited mode from genuine user input. - useEffect(() => { - const subscription = watch((_values, { name, type }) => { - const editMode = classifyFormEdit({ - name, - type, - configType: getValues('configType'), - }); - if (editMode) { - lastEditedModeRef.current = editMode; - } - }); - return () => subscription.unsubscribe(); - }, [watch, getValues]); - - useEffect(() => { - // Only generate a new SQL template when switching from builder mode to SQL mode. - if (prevConfigType !== 'builder' || configType !== 'sql') { - return; - } - - const showError = (message: string, err?: Error) => { - console.warn('Could not convert chart to SQL', { - form: getValues(), - err, - }); - notifications.show({ - id: 'builder-to-sql-error', - title: 'Could not auto-convert to SQL', - message, - color: 'red', - }); - }; - - // Only overwrite when the user edited in builder more recently than in SQL. - if (lastEditedModeRef.current !== 'builder') { - return; - } - - // A resolved source is required to build a ChartConfig from the form; - // every other reason a config can't be converted is reported by - // renderBuilderConfigAsSqlTemplate below. - if (!tableSource) { - showError('Auto-converting to SQL requires a source to be selected.'); - return; - } - - // Build a ChartConfig from the current form state - const form = getValues(); - const config = convertFormStateToChartConfig( - { ...form, configType: 'builder' }, - // dateRange is irrelevant here, since the SQL will contain date range macros - [new Date(0), new Date(0)], - tableSource, - ); - if (!config) return; - - const requestId = ++requestIdRef.current; - const applyResult = (result: RenderedSqlTemplate) => { - // Ignore results from generations that a newer switch has superseded. - if (requestId !== requestIdRef.current) { - return; - } - - if (result.isError) { - showError(result.error); - return; - } - - // Only write while still in SQL mode, and don't clobber a hand-edit the - // user made while generation was in flight. - if ( - getValues('configType') === 'sql' && - lastEditedModeRef.current === 'builder' - ) { - setValue('sqlTemplate', result.sql); - notifications.show({ - title: 'Chart converted to SQL', - message: 'The existing chart configuration has been converted to SQL', - color: 'green', - }); - } - }; - - renderBuilderConfigAsSqlTemplate(config, metadata) - .then(applyResult) - .catch(e => showError('Chart could not be auto-converted to SQL', e)); - }, [configType, tableSource, metadata, getValues, setValue, prevConfigType]); -} diff --git a/packages/app/tests/e2e/features/chart-explorer.spec.ts b/packages/app/tests/e2e/features/chart-explorer.spec.ts index acb05b582c..2ce3323abf 100644 --- a/packages/app/tests/e2e/features/chart-explorer.spec.ts +++ b/packages/app/tests/e2e/features/chart-explorer.spec.ts @@ -359,4 +359,49 @@ test.describe('Chart Explorer Functionality', { tag: ['@charts'] }, () => { }); }, ); + + test( + 'should convert a macro-based SQL template back into the builder when switching to Builder mode', + { tag: '@full-stack' }, + async () => { + await test.step('Wait for the chart editor data to load', async () => { + await chartExplorerPage.chartEditor.waitForDataToLoad(); + }); + + await test.step('Select the E2E Logs source and add a group-by', async () => { + await chartExplorerPage.chartEditor.selectSource( + DEFAULT_LOGS_SOURCE_NAME, + ); + // Keep the default Line + count() series; add a GROUP BY so the + // round-trip has a non-trivial builder field to preserve. + await chartExplorerPage.chartEditor.setGroupBy('ServiceName'); + }); + + await test.step('Switch to SQL mode and get the auto-generated template', async () => { + await chartExplorerPage.chartEditor.switchToSqlMode(); + const sqlEditor = chartExplorerPage.chartEditor.sqlEditorContent(); + await expect(sqlEditor).toContainText('$__sourceTable'); + await expect(sqlEditor).toContainText('ServiceName'); + await expect(sqlEditor).toContainText('count()'); + }); + + await test.step('Switch back to Builder mode (SQL → Builder conversion)', async () => { + await chartExplorerPage.chartEditor.switchToBuilderMode(); + // The builder-only aggregation control confirms builder mode is active + // and the conversion did not fail into an unusable state. + await expect(chartExplorerPage.chartEditor.aggFn).toBeVisible(); + }); + + await test.step('Re-generate SQL and verify the converted builder config round-trips', async () => { + // The template was never hand-edited, so switching back regenerates it + // from the (converted) builder config — the count() series and + // ServiceName group-by must survive the SQL → Builder conversion. + await chartExplorerPage.chartEditor.switchToSqlMode(); + const sqlEditor = chartExplorerPage.chartEditor.sqlEditorContent(); + await expect(sqlEditor).toContainText('$__sourceTable'); + await expect(sqlEditor).toContainText('ServiceName'); + await expect(sqlEditor).toContainText('count()'); + }); + }, + ); }); diff --git a/packages/common-utils/package.json b/packages/common-utils/package.json index 92d7006597..e307129e9e 100644 --- a/packages/common-utils/package.json +++ b/packages/common-utils/package.json @@ -14,6 +14,7 @@ "@clickhouse/client": "1.23.0-head.fae5998.1", "@clickhouse/client-common": "1.23.0-head.fae5998.1", "@clickhouse/client-web": "1.23.0-head.fae5998.1", + "@clickhouse/parser": "0.3.0", "@hyperdx/lucene": "^3.1.1", "date-fns": "^2.28.0", "date-fns-tz": "^2.0.0", diff --git a/packages/common-utils/src/__tests__/rawSqlToBuilder.test.ts b/packages/common-utils/src/__tests__/rawSqlToBuilder.test.ts new file mode 100644 index 0000000000..fb441d5544 --- /dev/null +++ b/packages/common-utils/src/__tests__/rawSqlToBuilder.test.ts @@ -0,0 +1,1355 @@ +import { renderBuilderConfigAsSqlTemplate } from '@/core/builderToRawSql'; +import { Metadata } from '@/core/metadata'; +import { + convertRawSqlToBuilderConfig, + replaceMacrosWithSentinels, + SqlToBuilderError, +} from '@/core/rawSqlToBuilder'; +import { ChartConfigWithOptDateRange, DisplayType } from '@/types'; + +describe('convertRawSqlToBuilderConfig', () => { + let mockMetadata: jest.Mocked; + + beforeAll(() => { + jest.spyOn(console, 'warn').mockImplementation(() => {}); + jest.spyOn(console, 'error').mockImplementation(() => {}); + }); + afterAll(() => { + jest.restoreAllMocks(); + }); + + beforeEach(() => { + const columns = [ + { name: 'timestamp', type: 'DateTime' }, + { name: 'date', type: 'Date' }, + { name: 'Duration', type: 'Float64' }, + { name: 'ServiceName', type: 'String' }, + ]; + mockMetadata = { + getColumns: jest.fn().mockResolvedValue(columns), + getMaterializedColumnsLookupTable: jest.fn().mockResolvedValue(new Map()), + getColumn: jest + .fn() + .mockImplementation(async ({ column }) => + columns.find(col => col.name === column), + ), + getTableMetadata: jest + .fn() + .mockResolvedValue({ primary_key: 'timestamp' }), + getSkipIndices: jest.fn().mockResolvedValue([]), + getSetting: jest.fn().mockResolvedValue(undefined), + isClickHouseCloud: jest.fn().mockResolvedValue(false), + } as unknown as jest.Mocked; + }); + + const from = { databaseName: 'default', tableName: 'otel_logs' }; + + describe('replaceMacrosWithSentinels', () => { + it('rewrites macro heads while preserving arguments', () => { + expect( + replaceMacrosWithSentinels( + 'SELECT $__timeInterval(timestamp) FROM $__sourceTable WHERE $__filters AND x >= $__fromTime_ms', + ), + ).toBe( + 'SELECT hdx_macro_timeInterval(timestamp) FROM hdx_macro_sourceTable WHERE hdx_macro_filters AND x >= hdx_macro_fromTime_ms', + ); + }); + }); + + describe('source table validation', () => { + it('accepts the source-table macro', () => { + expect(() => + convertRawSqlToBuilderConfig({ + sqlTemplate: 'SELECT count() FROM $__sourceTable', + displayType: DisplayType.Number, + from, + }), + ).not.toThrow(); + }); + + it.each([ + 'SELECT count() FROM otel_logs', + 'SELECT count() FROM default.otel_logs', + 'SELECT count() FROM `default`.`otel_logs`', + ])('accepts a matching literal source table: %s', sqlTemplate => { + expect(() => + convertRawSqlToBuilderConfig({ + sqlTemplate, + displayType: DisplayType.Number, + from, + }), + ).not.toThrow(); + }); + + it.each([ + 'SELECT count() FROM other_table', + 'SELECT count() FROM other_database.otel_logs', + ])('rejects a different literal source table: %s', sqlTemplate => { + expect(() => + convertRawSqlToBuilderConfig({ + sqlTemplate, + displayType: DisplayType.Number, + from, + }), + ).toThrow(SqlToBuilderError); + }); + + it('rejects a source alias because the builder cannot preserve it', () => { + expect(() => + convertRawSqlToBuilderConfig({ + sqlTemplate: 'SELECT count(logs.Duration) FROM otel_logs AS logs', + displayType: DisplayType.Number, + from, + }), + ).toThrow(SqlToBuilderError); + }); + + it('rejects a parameterized source table', () => { + expect(() => + convertRawSqlToBuilderConfig({ + sqlTemplate: 'SELECT count() FROM {table:Identifier}', + displayType: DisplayType.Number, + from, + }), + ).toThrow(SqlToBuilderError); + }); + + it('rejects conversion when no builder source is selected', () => { + expect(() => + convertRawSqlToBuilderConfig({ + sqlTemplate: 'SELECT count() FROM $__sourceTable', + displayType: DisplayType.Number, + }), + ).toThrow(SqlToBuilderError); + }); + }); + + describe('round-trips builder-generated SQL', () => { + const baseLineConfig: ChartConfigWithOptDateRange = { + displayType: DisplayType.Line, + connection: 'test-connection', + from, + select: [ + { aggFn: 'count', aggCondition: '', valueExpression: '' }, + { aggFn: 'avg', aggCondition: '', valueExpression: 'Duration' }, + ], + groupBy: [{ aggCondition: '', valueExpression: 'ServiceName' }], + where: 'ServiceName:api', + whereLanguage: 'lucene', + timestampValueExpression: 'timestamp', + granularity: '1 minute', + }; + + it('recovers a line chart config with group-by and where', async () => { + const rendered = await renderBuilderConfigAsSqlTemplate( + baseLineConfig, + mockMetadata, + ); + if (rendered.isError) { + throw new Error(rendered.error); + } + + const result = convertRawSqlToBuilderConfig({ + sqlTemplate: rendered.sql, + displayType: DisplayType.Line, + from, + timestampValueExpression: baseLineConfig.timestampValueExpression, + }); + + // The top-level WHERE is broadcast into each series' aggCondition (these + // display types have no top-level WHERE input in the builder). + expect(result.select).toEqual([ + { + aggFn: 'count', + aggCondition: "ServiceName ILIKE '%api%'", + aggConditionLanguage: 'sql', + valueExpression: '', + }, + { + aggFn: 'avg', + aggCondition: "ServiceName ILIKE '%api%'", + aggConditionLanguage: 'sql', + valueExpression: 'Duration', + }, + ]); + expect(result.groupBy).toBe('ServiceName'); + expect(result.where).toBe(''); + expect(result.whereLanguage).toBe('sql'); + expect(result.granularity).toBe('auto'); + }); + + it('recovers a table chart with having, order by and limit', async () => { + const rendered = await renderBuilderConfigAsSqlTemplate( + { + ...baseLineConfig, + displayType: DisplayType.Table, + granularity: undefined, + having: 'count() > 5', + havingLanguage: 'sql', + orderBy: 'count() DESC', + limit: { limit: 100 }, + }, + mockMetadata, + ); + if (rendered.isError) { + throw new Error(rendered.error); + } + + const result = convertRawSqlToBuilderConfig({ + sqlTemplate: rendered.sql, + displayType: DisplayType.Table, + from, + timestampValueExpression: baseLineConfig.timestampValueExpression, + }); + + expect(result.groupBy).toBe('ServiceName'); + expect(result.having).toBe('count() > 5'); + expect(result.havingLanguage).toBe('sql'); + expect(result.orderBy).toBe('count() DESC'); + expect(result.limit).toEqual({ limit: 100, offset: undefined }); + expect(result.granularity).toBeUndefined(); + }); + + it('recovers a number chart with a single aggregation and no group by', async () => { + const rendered = await renderBuilderConfigAsSqlTemplate( + { + ...baseLineConfig, + displayType: DisplayType.Number, + select: [{ aggFn: 'count', aggCondition: '', valueExpression: '' }], + groupBy: undefined, + granularity: undefined, + }, + mockMetadata, + ); + if (rendered.isError) { + throw new Error(rendered.error); + } + + const result = convertRawSqlToBuilderConfig({ + sqlTemplate: rendered.sql, + displayType: DisplayType.Number, + from, + timestampValueExpression: baseLineConfig.timestampValueExpression, + }); + + expect(result.select).toHaveLength(1); + expect(result.select[0].aggFn).toBe('count'); + expect(result.groupBy).toBe(''); + expect(result.granularity).toBeUndefined(); + }); + }); + + describe('aggregation recognition', () => { + const parse = (select: string) => + convertRawSqlToBuilderConfig({ + sqlTemplate: `SELECT ${select} FROM $__sourceTable`, + displayType: DisplayType.Table, + from, + }).select; + + it('recognizes count and countIf', () => { + expect(parse('count()')[0]).toMatchObject({ + aggFn: 'count', + valueExpression: '', + }); + expect(parse("countIf(ServiceName = 'api')")[0]).toMatchObject({ + aggFn: 'count', + valueExpression: '', + aggCondition: "ServiceName = 'api'", + }); + }); + + it('recognizes count(*) and is case-insensitive', () => { + expect(parse('count(*)')[0]).toMatchObject({ + aggFn: 'count', + valueExpression: '', + }); + expect(parse('COUNT(*)')[0]).toMatchObject({ + aggFn: 'count', + valueExpression: '', + }); + }); + + it('recognizes count_distinct', () => { + expect(parse('count(DISTINCT UserId)')[0]).toMatchObject({ + aggFn: 'count_distinct', + valueExpression: 'UserId', + }); + }); + + it('recognizes uniqExact as count_distinct', () => { + expect(parse('uniqExact(ServiceName)')[0]).toMatchObject({ + aggFn: 'count_distinct', + valueExpression: 'ServiceName', + }); + }); + + it('recognizes numeric aggregations, unwrapping the numeric coercion', () => { + expect( + parse('avg(toFloat64OrDefault(toString(Duration)))')[0], + ).toMatchObject({ aggFn: 'avg', valueExpression: 'Duration' }); + expect( + parse('sum(toFloat64OrDefault(toString(Duration)))')[0], + ).toMatchObject({ aggFn: 'sum', valueExpression: 'Duration' }); + }); + + it('recognizes a numeric aggregation without a coercion wrapper', () => { + expect(parse('avg(Duration)')[0]).toMatchObject({ + aggFn: 'avg', + valueExpression: 'Duration', + }); + }); + + it('unwraps a plain toFloat64 coercion', () => { + expect(parse('sum(toFloat64(Duration))')[0]).toMatchObject({ + aggFn: 'sum', + valueExpression: 'Duration', + }); + }); + + it('unwraps toFloat64OrDefault without an inner toString', () => { + expect(parse('sum(toFloat64OrDefault(Duration))')[0]).toMatchObject({ + aggFn: 'sum', + valueExpression: 'Duration', + }); + }); + + it('unwraps toFloat64 with an inner toString', () => { + expect(parse('sum(toFloat64(toString(Duration)))')[0]).toMatchObject({ + aggFn: 'sum', + valueExpression: 'Duration', + }); + }); + + it('does not unwrap toFloat64OrDefault when it has an explicit default', () => { + expect( + parse('sum(toFloat64OrDefault(toString(Duration), 0))')[0], + ).toMatchObject({ + aggFn: 'sum', + valueExpression: 'toFloat64OrDefault(toString(Duration), 0)', + }); + }); + + it('recognizes min, max, any and last_value', () => { + expect(parse('min(Duration)')[0]).toMatchObject({ + aggFn: 'min', + valueExpression: 'Duration', + }); + expect(parse('max(Duration)')[0]).toMatchObject({ + aggFn: 'max', + valueExpression: 'Duration', + }); + expect(parse('any(ServiceName)')[0]).toMatchObject({ + aggFn: 'any', + valueExpression: 'ServiceName', + }); + expect(parse('last_value(Duration)')[0]).toMatchObject({ + aggFn: 'last_value', + valueExpression: 'Duration', + }); + }); + + it('recognizes an If aggregation and strips the null guard', () => { + const col = parse( + "avgIf(toFloat64OrDefault(toString(Duration)), (ServiceName = 'api') AND toFloat64OrDefault(toString(Duration)) IS NOT NULL)", + )[0]; + expect(col).toMatchObject({ + aggFn: 'avg', + valueExpression: 'Duration', + aggCondition: "ServiceName = 'api'", + }); + }); + + it('recognizes an If aggregation without a null guard', () => { + expect(parse('sumIf(Duration, StatusCode = 500)')[0]).toMatchObject({ + aggFn: 'sum', + valueExpression: 'Duration', + aggCondition: 'StatusCode = 500', + }); + }); + + it('recognizes quantile at builder-supported levels (p50/p90/p95/p99)', () => { + expect( + parse('quantile(0.95)(toFloat64OrDefault(toString(Duration)))')[0], + ).toMatchObject({ + aggFn: 'quantile', + level: 0.95, + valueExpression: 'Duration', + }); + for (const level of [0.5, 0.9, 0.95, 0.99]) { + expect(parse(`quantile(${level})(Duration)`)[0]).toMatchObject({ + aggFn: 'quantile', + level, + }); + } + }); + + it('recognizes median as quantile at level 0.5', () => { + expect(parse('median(Duration)')[0]).toMatchObject({ + aggFn: 'quantile', + level: 0.5, + valueExpression: 'Duration', + }); + }); + + it('recognizes quantileIf with a condition', () => { + expect( + parse('quantileIf(0.95)(Duration, StatusCode = 500)')[0], + ).toMatchObject({ + aggFn: 'quantile', + level: 0.95, + valueExpression: 'Duration', + aggCondition: 'StatusCode = 500', + }); + }); + + it('maps an unsupported quantile level to a custom (none) expression', () => { + expect(parse('quantile(0.98)(Duration)')[0]).toMatchObject({ + aggFn: 'none', + valueExpression: 'quantile(0.98)(Duration)', + }); + }); + + it('maps arbitrary aggregation functions to a custom (none) expression', () => { + expect(parse('quantileTDigest(0.98)(Duration)')[0]).toMatchObject({ + aggFn: 'none', + valueExpression: 'quantileTDigest(0.98)(Duration)', + }); + }); + + it('captures a series alias', () => { + expect(parse('count() AS total')[0]).toMatchObject({ + aggFn: 'count', + alias: 'total', + }); + }); + + it('falls back to none for a raw expression', () => { + expect(parse('Duration + 1')[0]).toMatchObject({ + aggFn: 'none', + valueExpression: 'Duration + 1', + }); + }); + + it('recognizes ratio (divide) selects', () => { + const result = convertRawSqlToBuilderConfig({ + sqlTemplate: + 'SELECT divide(count(), sum(toFloat64OrDefault(toString(Duration)))) FROM $__sourceTable', + displayType: DisplayType.Number, + from, + }); + expect(result.seriesReturnType).toBe('ratio'); + expect(result.select).toHaveLength(2); + expect(result.select[0].aggFn).toBe('count'); + expect(result.select[1].aggFn).toBe('sum'); + }); + + it('recognizes a ratio written with the / operator', () => { + const result = convertRawSqlToBuilderConfig({ + sqlTemplate: 'SELECT count() / sum(Duration) FROM $__sourceTable', + displayType: DisplayType.Number, + from, + }); + expect(result.seriesReturnType).toBe('ratio'); + expect(result.select).toHaveLength(2); + expect(result.select[0]).toMatchObject({ aggFn: 'count' }); + expect(result.select[1]).toMatchObject({ + aggFn: 'sum', + valueExpression: 'Duration', + }); + }); + + it('recovers per-leg conditions in a ratio', () => { + const result = convertRawSqlToBuilderConfig({ + sqlTemplate: + 'SELECT countIf(StatusCode = 500) / countIf(StatusCode < 500) FROM $__sourceTable', + displayType: DisplayType.Number, + from, + }); + expect(result.seriesReturnType).toBe('ratio'); + expect(result.select.map(s => s.aggCondition)).toEqual([ + 'StatusCode = 500', + 'StatusCode < 500', + ]); + }); + }); + + describe('shared WHERE broadcasting', () => { + it('broadcasts a top-level WHERE into every aggregation series and clears it', () => { + const result = convertRawSqlToBuilderConfig({ + sqlTemplate: + "SELECT count(), sum(toFloat64OrDefault(toString(Duration))) FROM $__sourceTable WHERE ServiceName = 'api'", + displayType: DisplayType.Table, + from, + }); + expect(result.where).toBe(''); + expect(result.select.map(s => s.aggCondition)).toEqual([ + "ServiceName = 'api'", + "ServiceName = 'api'", + ]); + }); + + it('broadcasts a WHERE ... IN (...) into the aggregation', () => { + const result = convertRawSqlToBuilderConfig({ + sqlTemplate: + "SELECT count() FROM $__sourceTable WHERE ServiceName IN ('api', 'web')", + displayType: DisplayType.Table, + from, + }); + expect(result.where).toBe(''); + expect(result.select[0].aggCondition).toBe( + "ServiceName IN ('api', 'web')", + ); + }); + + it('ANDs the broadcast WHERE with an existing -If condition', () => { + const result = convertRawSqlToBuilderConfig({ + sqlTemplate: + "SELECT countIf(StatusCode = 500) FROM $__sourceTable WHERE ServiceName = 'api'", + displayType: DisplayType.Number, + from, + }); + expect(result.where).toBe(''); + expect(result.select[0].aggCondition).toBe( + "(StatusCode = 500) AND (ServiceName = 'api')", + ); + }); + + it('broadcasts a compound WHERE (multiple conjuncts) into each series', () => { + const result = convertRawSqlToBuilderConfig({ + sqlTemplate: + "SELECT count() FROM $__sourceTable WHERE ServiceName = 'api' AND StatusCode >= 500", + displayType: DisplayType.Number, + from, + }); + expect(result.where).toBe(''); + expect(result.select[0].aggCondition).toBe( + "(ServiceName = 'api') AND (StatusCode >= 500)", + ); + }); + + it('keeps the WHERE at the top level when a raw (none) column is present', () => { + const result = convertRawSqlToBuilderConfig({ + sqlTemplate: + "SELECT count(), SpanId FROM $__sourceTable WHERE ServiceName = 'api'", + displayType: DisplayType.Table, + from, + }); + // SpanId is a raw `none` column that can't carry an aggCondition, so the + // shared WHERE is not broadcast. + expect(result.where).toBe("ServiceName = 'api'"); + expect(result.select.every(s => s.aggCondition === '')).toBe(true); + }); + + it('does not surface the builder aggCondition OR-group as an extra WHERE', async () => { + const orGroupConfig: ChartConfigWithOptDateRange = { + displayType: DisplayType.Table, + connection: 'test-connection', + from, + select: [ + { + aggFn: 'count', + aggCondition: "ServiceName = 'a'", + aggConditionLanguage: 'sql', + valueExpression: '', + }, + { + aggFn: 'sum', + aggCondition: "ServiceName = 'b'", + aggConditionLanguage: 'sql', + valueExpression: 'Duration', + }, + ], + where: '', + timestampValueExpression: 'timestamp', + }; + const rendered = await renderBuilderConfigAsSqlTemplate( + orGroupConfig, + mockMetadata, + ); + if (rendered.isError) { + throw new Error(rendered.error); + } + + const result = convertRawSqlToBuilderConfig({ + sqlTemplate: rendered.sql, + displayType: DisplayType.Table, + from, + timestampValueExpression: orGroupConfig.timestampValueExpression, + }); + // The `(ServiceName = 'a' OR ServiceName = 'b')` index hint is stripped, + // leaving each series' own aggCondition and no residual WHERE. + expect(result.where).toBe(''); + expect(result.select.map(s => s.aggCondition)).toEqual([ + "ServiceName = 'a'", + "ServiceName = 'b'", + ]); + }); + }); + + describe('query parameter recognition', () => { + it('strips a time-range WHERE written with query params directly', () => { + const result = convertRawSqlToBuilderConfig({ + sqlTemplate: `SELECT count() + FROM $__sourceTable + WHERE timestamp >= fromUnixTimestamp64Milli({startDateMilliseconds:Int64}) + AND timestamp < fromUnixTimestamp64Milli({endDateMilliseconds:Int64})`, + displayType: DisplayType.Number, + from, + timestampValueExpression: 'timestamp', + }); + // The time-range predicate is derived from the dashboard range, so it is + // dropped rather than surfaced as a user WHERE / aggCondition. + expect(result.where).toBe(''); + expect(result.select).toHaveLength(1); + expect(result.select[0]).toMatchObject({ aggFn: 'count' }); + expect(result.select[0].aggCondition).toBe(''); + }); + + it('keeps a genuine WHERE alongside a query-param time range', () => { + const result = convertRawSqlToBuilderConfig({ + sqlTemplate: `SELECT count() + FROM $__sourceTable + WHERE timestamp >= fromUnixTimestamp64Milli({startDateMilliseconds:Int64}) + AND timestamp < fromUnixTimestamp64Milli({endDateMilliseconds:Int64}) + AND ServiceName = 'api'`, + displayType: DisplayType.Number, + from, + timestampValueExpression: 'timestamp', + }); + // Only the user's predicate survives; it broadcasts into the aggregation. + expect(result.where).toBe(''); + expect(result.select[0].aggCondition).toBe("ServiceName = 'api'"); + }); + + it('rejects an arbitrary use of a time-range query parameter', () => { + expect(() => + convertRawSqlToBuilderConfig({ + sqlTemplate: `SELECT count() + FROM $__sourceTable + WHERE price > {startDateMilliseconds:Int64}`, + displayType: DisplayType.Number, + from, + timestampValueExpression: 'timestamp', + }), + ).toThrow(SqlToBuilderError); + }); + + it('rejects a time bound wrapped differently from the timestamp expression', () => { + expect(() => + convertRawSqlToBuilderConfig({ + sqlTemplate: `SELECT count() + FROM $__sourceTable + WHERE timestamp >= toStartOfHour($__fromTime_ms) + AND timestamp <= toStartOfHour($__toTime_ms)`, + displayType: DisplayType.Number, + from, + timestampValueExpression: 'timestamp', + }), + ).toThrow(SqlToBuilderError); + }); + + it('rejects wrappers that the renderer does not use for time bounds', () => { + expect(() => + convertRawSqlToBuilderConfig({ + sqlTemplate: `SELECT count() + FROM $__sourceTable + WHERE timestamp >= toDateTime($__fromTime_ms) + AND timestamp <= toDateTime($__toTime_ms)`, + displayType: DisplayType.Number, + from, + timestampValueExpression: 'timestamp', + }), + ).toThrow(SqlToBuilderError); + }); + + it('rejects a malformed included-data-interval expansion', () => { + expect(() => + convertRawSqlToBuilderConfig({ + sqlTemplate: `SELECT count() + FROM $__sourceTable + WHERE timestamp >= toStartOfInterval($__fromTime_ms, INTERVAL $__interval_s second) - INTERVAL 1 minute + AND timestamp <= toStartOfInterval($__toTime_ms, INTERVAL $__interval_s second) + INTERVAL 1 minute`, + displayType: DisplayType.Number, + from, + timestampValueExpression: 'timestamp', + }), + ).toThrow(SqlToBuilderError); + }); + + it('rejects a one-sided time range', () => { + expect(() => + convertRawSqlToBuilderConfig({ + sqlTemplate: + 'SELECT count() FROM $__sourceTable WHERE timestamp >= $__fromTime_ms', + displayType: DisplayType.Number, + from, + timestampValueExpression: 'timestamp', + }), + ).toThrow(SqlToBuilderError); + }); + + it('rejects a complete range on a different timestamp expression', () => { + expect(() => + convertRawSqlToBuilderConfig({ + sqlTemplate: `SELECT count() + FROM $__sourceTable + WHERE other_timestamp >= $__fromTime_ms + AND other_timestamp <= $__toTime_ms`, + displayType: DisplayType.Number, + from, + timestampValueExpression: 'timestamp', + }), + ).toThrow( + new SqlToBuilderError( + 'The SQL time filter uses timestamp expression "other_timestamp", but the selected source uses "timestamp". Update the SQL time filter or select a matching source.', + ), + ); + }); + + it('explains a timestamp mismatch in a time-filter macro', () => { + expect(() => + convertRawSqlToBuilderConfig({ + sqlTemplate: + 'SELECT count() FROM $__sourceTable WHERE $__timeFilter(other_timestamp)', + displayType: DisplayType.Number, + from, + timestampValueExpression: 'timestamp', + }), + ).toThrow( + new SqlToBuilderError( + 'The SQL time filter uses timestamp expression "other_timestamp", but the selected source uses "timestamp". Update the SQL time filter or select a matching source.', + ), + ); + }); + + it('consumes every pair in a multi-column timestamp range', () => { + const result = convertRawSqlToBuilderConfig({ + sqlTemplate: `SELECT count() + FROM $__sourceTable + WHERE date >= toDate($__fromTime_ms) + AND date <= toDate($__toTime_ms) + AND timestamp >= $__fromTime_ms + AND timestamp <= $__toTime_ms + AND ServiceName = 'api'`, + displayType: DisplayType.Number, + from, + timestampValueExpression: 'date, timestamp', + }); + + expect(result.where).toBe(''); + expect(result.select[0].aggCondition).toBe("ServiceName = 'api'"); + }); + + it('splits multi-column timestamp expressions without splitting nested commas', () => { + const result = convertRawSqlToBuilderConfig({ + sqlTemplate: `SELECT count() + FROM $__sourceTable + WHERE ResourceAttributes['date,time'] >= $__fromTime_ms + AND ResourceAttributes['date,time'] <= $__toTime_ms + AND toDateTime64(timestamp, 3) >= $__fromTime_ms + AND toDateTime64(timestamp, 3) <= $__toTime_ms`, + displayType: DisplayType.Number, + from, + timestampValueExpression: + "ResourceAttributes['date,time'], toDateTime64(timestamp, 3)", + }); + expect(result.where).toBe(''); + }); + + it('recognizes renderer-expanded included-data-interval bounds', () => { + const result = convertRawSqlToBuilderConfig({ + sqlTemplate: `SELECT count() + FROM $__sourceTable + WHERE timestamp >= toStartOfInterval($__fromTime_ms, INTERVAL $__interval_s second) - INTERVAL $__interval_s second + AND timestamp <= toStartOfInterval($__toTime_ms, INTERVAL $__interval_s second) + INTERVAL $__interval_s second`, + displayType: DisplayType.Number, + from, + timestampValueExpression: 'timestamp', + }); + expect(result.where).toBe(''); + }); + + it('rejects partial coverage of a multi-column timestamp expression', () => { + expect(() => + convertRawSqlToBuilderConfig({ + sqlTemplate: `SELECT count() + FROM $__sourceTable + WHERE timestamp >= $__fromTime_ms + AND timestamp <= $__toTime_ms`, + displayType: DisplayType.Number, + from, + timestampValueExpression: 'date, timestamp', + }), + ).toThrow(SqlToBuilderError); + }); + + it('rejects time-range parameters nested in OR', () => { + expect(() => + convertRawSqlToBuilderConfig({ + sqlTemplate: `SELECT count() + FROM $__sourceTable + WHERE timestamp >= $__fromTime_ms OR ServiceName = 'api'`, + displayType: DisplayType.Number, + from, + timestampValueExpression: 'timestamp', + }), + ).toThrow(SqlToBuilderError); + }); + + it('accepts a complete time-filter macro for the configured timestamp', () => { + const result = convertRawSqlToBuilderConfig({ + sqlTemplate: + 'SELECT count() FROM $__sourceTable WHERE $__timeFilter(timestamp)', + displayType: DisplayType.Number, + from, + timestampValueExpression: 'timestamp', + }); + expect(result.where).toBe(''); + expect(result.select[0].aggCondition).toBe(''); + }); + + it('accepts a complete two-column time-filter macro', () => { + const result = convertRawSqlToBuilderConfig({ + sqlTemplate: + 'SELECT count() FROM $__sourceTable WHERE $__dateTimeFilter(date, timestamp)', + displayType: DisplayType.Number, + from, + timestampValueExpression: 'date, timestamp', + }); + expect(result.where).toBe(''); + expect(result.select[0].aggCondition).toBe(''); + }); + + it('rejects a time range when timestamp context is unavailable', () => { + expect(() => + convertRawSqlToBuilderConfig({ + sqlTemplate: `SELECT count() + FROM $__sourceTable + WHERE timestamp >= $__fromTime_ms + AND timestamp <= $__toTime_ms`, + displayType: DisplayType.Number, + from, + }), + ).toThrow(SqlToBuilderError); + }); + + it('round-trips renderer-added primary-key time filters', async () => { + mockMetadata.getTableMetadata.mockResolvedValue({ + primary_key: 'toStartOfHour(timestamp), timestamp', + } as any); + const config: ChartConfigWithOptDateRange = { + displayType: DisplayType.Line, + connection: 'test-connection', + from, + select: [{ aggFn: 'count', aggCondition: '', valueExpression: '' }], + where: '', + whereLanguage: 'sql', + timestampValueExpression: 'timestamp', + granularity: '1 minute', + }; + const rendered = await renderBuilderConfigAsSqlTemplate( + config, + mockMetadata, + ); + if (rendered.isError) throw new Error(rendered.error); + + expect(() => + convertRawSqlToBuilderConfig({ + sqlTemplate: rendered.sql, + displayType: DisplayType.Line, + from, + timestampValueExpression: 'timestamp', + }), + ).not.toThrow(); + }); + + it('round-trips a renderer-generated multi-column timestamp range', async () => { + const config: ChartConfigWithOptDateRange = { + displayType: DisplayType.Line, + connection: 'test-connection', + from, + select: [{ aggFn: 'count', aggCondition: '', valueExpression: '' }], + where: '', + whereLanguage: 'sql', + timestampValueExpression: 'date, timestamp', + granularity: '1 minute', + }; + const rendered = await renderBuilderConfigAsSqlTemplate( + config, + mockMetadata, + ); + if (rendered.isError) throw new Error(rendered.error); + + const result = convertRawSqlToBuilderConfig({ + sqlTemplate: rendered.sql, + displayType: DisplayType.Line, + from, + timestampValueExpression: config.timestampValueExpression, + }); + + expect(result.granularity).toBe('auto'); + expect(result).not.toHaveProperty('timestampValueExpression'); + }); + + it('recognizes a time bucket written as toStartOfInterval over intervalSeconds', () => { + const result = convertRawSqlToBuilderConfig({ + sqlTemplate: `SELECT + toStartOfInterval(timestamp, INTERVAL {intervalSeconds:Int64} second) AS ts, + ServiceName, + count() + FROM $__sourceTable + WHERE timestamp >= fromUnixTimestamp64Milli({startDateMilliseconds:Int64}) + AND timestamp < fromUnixTimestamp64Milli({endDateMilliseconds:Int64}) + GROUP BY ServiceName, ts`, + displayType: DisplayType.Line, + from, + timestampValueExpression: 'timestamp', + }); + // The bucket (referenced by its `ts` alias in GROUP BY) is dropped and + // maps to auto granularity, leaving ServiceName as the only grouping. + expect(result.granularity).toBe('auto'); + expect(result.groupBy).toBe('ServiceName'); + expect(result.select).toHaveLength(1); + expect(result.select[0]).toMatchObject({ aggFn: 'count' }); + }); + + it('recognizes a millisecond time bucket over intervalMilliseconds', () => { + const result = convertRawSqlToBuilderConfig({ + sqlTemplate: `SELECT + toStartOfInterval(toDateTime64(timestamp, 3), INTERVAL {intervalMilliseconds:Int64} millisecond) AS ts, + count() + FROM $__sourceTable + GROUP BY ts`, + displayType: DisplayType.Line, + from, + timestampValueExpression: 'timestamp', + }); + expect(result.granularity).toBe('auto'); + expect(result.groupBy).toBe(''); + expect(result.select).toHaveLength(1); + }); + + it('drops ORDER BY that references the query-param time bucket alias', () => { + const result = convertRawSqlToBuilderConfig({ + sqlTemplate: `SELECT + toStartOfInterval(timestamp, INTERVAL {intervalSeconds:Int64} second) AS ts, + count() + FROM $__sourceTable + GROUP BY ts + ORDER BY ts ASC`, + displayType: DisplayType.Line, + from, + timestampValueExpression: 'timestamp', + }); + // Line charts reject ORDER BY; dropping the implicit bucket ordering keeps + // the conversion valid. + expect(result.orderBy).toBeUndefined(); + expect(result.granularity).toBe('auto'); + }); + }); + + describe('table charts', () => { + it('recovers a single grouping column', () => { + const result = convertRawSqlToBuilderConfig({ + sqlTemplate: + 'SELECT count(), ServiceName FROM $__sourceTable GROUP BY ServiceName', + displayType: DisplayType.Table, + from, + }); + expect(result.groupBy).toBe('ServiceName'); + expect(result.select).toHaveLength(1); + expect(result.select[0]).toMatchObject({ aggFn: 'count' }); + }); + + it('resolves positional GROUP BY references to the selected columns', () => { + const result = convertRawSqlToBuilderConfig({ + sqlTemplate: + 'SELECT ServiceName, StatusCode, count() FROM $__sourceTable GROUP BY 1, 2', + displayType: DisplayType.Table, + from, + }); + expect(result.groupBy).toBe('ServiceName, StatusCode'); + expect(result.select).toHaveLength(1); + expect(result.select[0]).toMatchObject({ aggFn: 'count' }); + }); + + it('resolves a GROUP BY that references a select alias', () => { + const result = convertRawSqlToBuilderConfig({ + sqlTemplate: + 'SELECT ServiceName AS svc, count() FROM $__sourceTable GROUP BY svc', + displayType: DisplayType.Table, + from, + }); + expect(result.groupBy).toBe('ServiceName'); + expect(result.select).toHaveLength(1); + expect(result.select[0]).toMatchObject({ aggFn: 'count' }); + }); + + it('recovers HAVING', () => { + const result = convertRawSqlToBuilderConfig({ + sqlTemplate: + 'SELECT count(), ServiceName FROM $__sourceTable GROUP BY ServiceName HAVING count(*) > 5', + displayType: DisplayType.Table, + from, + }); + expect(result.having).toBe('count(*) > 5'); + expect(result.havingLanguage).toBe('sql'); + }); + + it('recovers a multi-column ORDER BY', () => { + const result = convertRawSqlToBuilderConfig({ + sqlTemplate: + 'SELECT count(), ServiceName FROM $__sourceTable GROUP BY ServiceName ORDER BY count() DESC, ServiceName ASC', + displayType: DisplayType.Table, + from, + }); + expect(result.orderBy).toBe('count() DESC, ServiceName ASC'); + }); + + it('recovers the comma form of LIMIT (offset, count)', () => { + const result = convertRawSqlToBuilderConfig({ + sqlTemplate: + 'SELECT count(), ServiceName FROM $__sourceTable GROUP BY ServiceName LIMIT 5, 10', + displayType: DisplayType.Table, + from, + }); + expect(result.limit).toEqual({ limit: 10, offset: 5 }); + }); + }); + + describe('time series charts', () => { + it('recovers auto granularity with multiple grouping columns', () => { + const result = convertRawSqlToBuilderConfig({ + sqlTemplate: `SELECT + toStartOfInterval(timestamp, INTERVAL {intervalSeconds:Int64} second) AS ts, + ServiceName, + StatusCode, + count() + FROM $__sourceTable + GROUP BY ServiceName, StatusCode, ts`, + displayType: DisplayType.Line, + from, + timestampValueExpression: 'timestamp', + }); + expect(result.granularity).toBe('auto'); + expect(result.groupBy).toBe('ServiceName, StatusCode'); + expect(result.select).toHaveLength(1); + }); + + it('recovers a stacked bar chart time bucket', () => { + const result = convertRawSqlToBuilderConfig({ + sqlTemplate: `SELECT + toStartOfInterval(timestamp, INTERVAL {intervalSeconds:Int64} second) AS ts, + ServiceName, + count() + FROM $__sourceTable + GROUP BY ServiceName, ts`, + displayType: DisplayType.StackedBar, + from, + timestampValueExpression: 'timestamp', + }); + expect(result.granularity).toBe('auto'); + expect(result.groupBy).toBe('ServiceName'); + expect(result.select).toHaveLength(1); + }); + + it('maps toStartOfMinute to a 1 minute granularity', () => { + const result = convertRawSqlToBuilderConfig({ + sqlTemplate: + 'SELECT toStartOfMinute(timestamp) AS ts, count() FROM $__sourceTable GROUP BY ts', + displayType: DisplayType.Line, + from, + timestampValueExpression: 'timestamp', + }); + expect(result.granularity).toBe('1 minute'); + expect(result.groupBy).toBe(''); + expect(result.select).toHaveLength(1); + expect(result.select[0]).toMatchObject({ aggFn: 'count' }); + }); + + it('maps a literal INTERVAL time bucket to a granularity', () => { + const result = convertRawSqlToBuilderConfig({ + sqlTemplate: + 'SELECT toStartOfInterval(timestamp, INTERVAL 1 MINUTE) AS ts, count() FROM $__sourceTable GROUP BY ts', + displayType: DisplayType.Line, + from, + timestampValueExpression: 'timestamp', + }); + expect(result.granularity).toBe('1 minute'); + expect(result.groupBy).toBe(''); + expect(result.select).toHaveLength(1); + }); + + it('maps toStartOfHour to a 1 hour granularity', () => { + const result = convertRawSqlToBuilderConfig({ + sqlTemplate: + 'SELECT toStartOfHour(timestamp) AS ts, count() FROM $__sourceTable GROUP BY ts', + displayType: DisplayType.Line, + from, + timestampValueExpression: 'timestamp', + }); + expect(result.granularity).toBe('1 hour'); + expect(result.groupBy).toBe(''); + expect(result.select).toHaveLength(1); + }); + + it('rejects a bucket on a timestamp not configured by the source', () => { + expect(() => + convertRawSqlToBuilderConfig({ + sqlTemplate: + 'SELECT toStartOfMinute(created_at) AS ts, count() FROM $__sourceTable GROUP BY ts', + displayType: DisplayType.Line, + from, + timestampValueExpression: 'timestamp', + }), + ).toThrow(SqlToBuilderError); + }); + + it('rejects a bucket without source timestamp context', () => { + expect(() => + convertRawSqlToBuilderConfig({ + sqlTemplate: + 'SELECT toStartOfMinute(timestamp) AS ts, count() FROM $__sourceTable GROUP BY ts', + displayType: DisplayType.Line, + from, + }), + ).toThrow(SqlToBuilderError); + }); + + it('rejects a bucket expression even when the source time filters match', () => { + expect(() => + convertRawSqlToBuilderConfig({ + sqlTemplate: `SELECT toStartOfMinute(created_at) AS ts, count() + FROM $__sourceTable + WHERE timestamp >= $__fromTime_ms + AND timestamp <= $__toTime_ms + GROUP BY ts`, + displayType: DisplayType.Line, + from, + timestampValueExpression: 'timestamp', + }), + ).toThrow(SqlToBuilderError); + }); + + it('does not treat the fixed bucket alias as proof of time bucketing', () => { + const result = convertRawSqlToBuilderConfig({ + sqlTemplate: `SELECT ServiceName AS __hdx_time_bucket, count() + FROM $__sourceTable + GROUP BY __hdx_time_bucket`, + displayType: DisplayType.Line, + from, + }); + + expect(result.granularity).toBeUndefined(); + expect(result.groupBy).toBe('ServiceName'); + expect(result.select).toHaveLength(1); + }); + }); + + describe('pie and bar charts', () => { + it('recovers a single pie series with group by and order by', () => { + const result = convertRawSqlToBuilderConfig({ + sqlTemplate: + 'SELECT count(), ServiceName FROM $__sourceTable GROUP BY ServiceName ORDER BY count() DESC', + displayType: DisplayType.Pie, + from, + }); + expect(result.groupBy).toBe('ServiceName'); + expect(result.orderBy).toBe('count() DESC'); + expect(result.select).toHaveLength(1); + }); + + it('maps a uniqExact pie series to count_distinct', () => { + const result = convertRawSqlToBuilderConfig({ + sqlTemplate: + 'SELECT uniqExact(ServiceName) AS uv, StatusCode FROM $__sourceTable GROUP BY StatusCode', + displayType: DisplayType.Pie, + from, + }); + expect(result.groupBy).toBe('StatusCode'); + expect(result.select).toHaveLength(1); + expect(result.select[0]).toMatchObject({ + aggFn: 'count_distinct', + valueExpression: 'ServiceName', + alias: 'uv', + }); + }); + + it('recovers a single bar series with group by and order by', () => { + const result = convertRawSqlToBuilderConfig({ + sqlTemplate: + 'SELECT avg(Duration), ServiceName FROM $__sourceTable GROUP BY ServiceName ORDER BY avg(Duration) DESC', + displayType: DisplayType.Bar, + from, + }); + expect(result.groupBy).toBe('ServiceName'); + expect(result.orderBy).toBe('avg(Duration) DESC'); + expect(result.select).toHaveLength(1); + expect(result.select[0]).toMatchObject({ + aggFn: 'avg', + valueExpression: 'Duration', + }); + }); + + it('maps a median bar series to quantile 0.5', () => { + const result = convertRawSqlToBuilderConfig({ + sqlTemplate: + 'SELECT median(Duration), ServiceName FROM $__sourceTable GROUP BY ServiceName', + displayType: DisplayType.Bar, + from, + }); + expect(result.groupBy).toBe('ServiceName'); + expect(result.select).toHaveLength(1); + expect(result.select[0]).toMatchObject({ + aggFn: 'quantile', + level: 0.5, + valueExpression: 'Duration', + }); + }); + }); + + describe('rejects leaked macros', () => { + const expectMacroRejected = ( + sqlTemplate: string, + displayType: DisplayType = DisplayType.Table, + ) => + expect(() => + convertRawSqlToBuilderConfig({ sqlTemplate, displayType, from }), + ).toThrow(SqlToBuilderError); + + it('rejects a macro in a raw (none) select column', () => { + expectMacroRejected('SELECT $__fromTime FROM $__sourceTable'); + }); + + it('rejects a macro nested inside a raw select expression', () => { + expectMacroRejected('SELECT toDateTime($__fromTime) FROM $__sourceTable'); + }); + + it('rejects a macro in an aggregation value expression', () => { + expectMacroRejected('SELECT max($__fromTime) FROM $__sourceTable'); + }); + + it('rejects a macro in GROUP BY', () => { + expectMacroRejected( + 'SELECT count(), $__fromTime FROM $__sourceTable GROUP BY $__fromTime', + ); + }); + + it('rejects a macro in ORDER BY', () => { + expectMacroRejected( + 'SELECT SpanId FROM $__sourceTable ORDER BY $__fromTime', + ); + }); + + it('rejects a leaked macro even on time series charts', () => { + expectMacroRejected( + 'SELECT $__timeInterval(timestamp) + 1 FROM $__sourceTable', + DisplayType.Line, + ); + }); + + it('names the offending macro in the error message', () => { + expect(() => + convertRawSqlToBuilderConfig({ + sqlTemplate: 'SELECT toDateTime($__fromTime) FROM $__sourceTable', + displayType: DisplayType.Table, + from, + }), + ).toThrow('$__fromTime'); + }); + + it('converts a query whose macros are all recognized and consumed', () => { + // $__sourceTable (FROM), $__timeInterval (time bucket) and $__filters are + // all consumed during conversion, so nothing leaks. + const result = convertRawSqlToBuilderConfig({ + sqlTemplate: `SELECT $__timeInterval(timestamp) AS ts, count() + FROM $__sourceTable + WHERE $__filters + GROUP BY ts`, + displayType: DisplayType.Line, + from, + timestampValueExpression: 'timestamp', + }); + expect(result.granularity).toBe('auto'); + expect(result.select).toHaveLength(1); + expect(result.select[0]).toMatchObject({ aggFn: 'count' }); + }); + }); + + describe('interval query parameter on non-time-series charts', () => { + const intervalSql = + 'SELECT toStartOfInterval(timestamp, INTERVAL {intervalSeconds:Int64} second) + 1 AS x FROM $__sourceTable'; + + it('rejects an interval query parameter in a raw column', () => { + expect(() => + convertRawSqlToBuilderConfig({ + sqlTemplate: intervalSql, + displayType: DisplayType.Table, + from, + }), + ).toThrow(SqlToBuilderError); + }); + }); + + describe('unsupported patterns', () => { + const expectError = ( + sqlTemplate: string, + displayType: DisplayType = DisplayType.Table, + ) => + expect(() => + convertRawSqlToBuilderConfig({ sqlTemplate, displayType, from }), + ).toThrow(SqlToBuilderError); + + it('rejects empty SQL', () => { + expectError(' '); + }); + + it('rejects unparseable SQL', () => { + expectError('SELECT ??? FROM'); + }); + + it('rejects UNION queries', () => { + expectError( + 'SELECT count() FROM $__sourceTable UNION ALL SELECT count() FROM $__sourceTable', + ); + }); + + it('rejects joins / multiple tables', () => { + expectError('SELECT count() FROM a JOIN b ON a.id = b.id'); + }); + + it('rejects subquery FROM', () => { + expectError('SELECT count() FROM (SELECT 1)'); + }); + + it('rejects user CTEs', () => { + expectError('WITH x AS (SELECT 1) SELECT count() FROM $__sourceTable'); + }); + + it('rejects SELECT DISTINCT', () => { + expectError('SELECT DISTINCT ServiceName FROM $__sourceTable'); + }); + + it('rejects GROUP BY on a number chart', () => { + expectError( + 'SELECT count(), ServiceName FROM $__sourceTable GROUP BY ServiceName', + DisplayType.Number, + ); + }); + + it('rejects a time bucket on a table chart', () => { + expectError( + 'SELECT count(), $__timeInterval(timestamp) AS `__hdx_time_bucket` FROM $__sourceTable GROUP BY $__timeInterval(timestamp) AS `__hdx_time_bucket`', + DisplayType.Table, + ); + }); + + it('rejects multiple series on a pie chart', () => { + expectError( + 'SELECT count(), sum(toFloat64OrDefault(toString(Duration))), ServiceName FROM $__sourceTable GROUP BY ServiceName', + DisplayType.Pie, + ); + }); + }); +}); diff --git a/packages/common-utils/src/core/materializedViews.ts b/packages/common-utils/src/core/materializedViews.ts index 810dd79063..83818411d7 100644 --- a/packages/common-utils/src/core/materializedViews.ts +++ b/packages/common-utils/src/core/materializedViews.ts @@ -27,7 +27,7 @@ import { } from './utils'; // ClickHouse named time-bucketing functions and their granularity equivalents. -const NAMED_BUCKET_FUNCTIONS: Record = { +export const NAMED_BUCKET_FUNCTIONS: Record = { toStartOfSecond: '1 second', toStartOfMinute: '1 minute', toStartOfFiveMinutes: '5 minute', diff --git a/packages/common-utils/src/core/rawSqlToBuilder.ts b/packages/common-utils/src/core/rawSqlToBuilder.ts new file mode 100644 index 0000000000..d469d2cc82 --- /dev/null +++ b/packages/common-utils/src/core/rawSqlToBuilder.ts @@ -0,0 +1,1476 @@ +import { + type ASTNode, + type Expression, + formatNode, + type FunctionNode, + isExpressionList, + isFunction, + isIdentifier, + isQueryParameter, + isTableIdentifier, + isWithElement, + type OrderByElementNode, + parse, + ParseError, + type SelectQueryNode, + type Statement, +} from '@clickhouse/parser'; + +import { + FILTERS_MACRO_NAME, + INTERVAL_MACROS, + TIME_RANGE_MACROS, +} from '@/macros'; +import { RawSqlQueryParam } from '@/rawSqlParams'; +import { + type AggregateFunction, + AggregateFunctionSchema, + DerivedColumn, + DISPLAY_TYPE_LABELS, + DisplayType, + type Limit, + type SQLInterval, +} from '@/types'; + +import { NAMED_BUCKET_FUNCTIONS } from './materializedViews'; +import { isNumericAggFn } from './renderChartConfig'; +import { splitAndTrimWithBracket } from './utils'; + +export class SqlToBuilderError extends Error { + constructor(message: string) { + super(message); + this.name = 'SqlToBuilderError'; + } +} + +/** Prefix a macro is rewritten to so the SQL parses. */ +const MACRO_SENTINEL_PREFIX = 'hdx_macro_'; + +/** Macro name that resolves to the builder's configured source table. */ +const SOURCE_TABLE_MACRO_NAME = 'sourceTable'; + +type BuilderSource = { + databaseName: string; + tableName: string; +}; + +/** Macro base names that expand to a time-range predicate on the WHERE clause. */ +const TIME_RANGE_MACRO_NAMES: Set = new Set(TIME_RANGE_MACROS); + +const START_TIME_MACROS: Set = new Set(['fromTime', 'fromTime_ms']); +const END_TIME_MACROS: Set = new Set(['toTime', 'toTime_ms']); + +type TimeBoundKind = 'start' | 'end'; + +/** Macro base names that expand to a time-bucketing expression. */ +const INTERVAL_MACRO_NAMES: Set = new Set(INTERVAL_MACROS); + +/** + * Query-parameter names the time-range macros expand to. Users can bind a query + * to the dashboard time range either through a macro (`$__timeFilter(col)`) or + * by referencing these parameters directly (the form the macros expand to, + * e.g. `col >= fromUnixTimestamp64Milli({startDateMilliseconds:Int64})`), so + * both are recognized identically. + */ +const TIME_RANGE_QUERY_PARAMS: Set = new Set([ + RawSqlQueryParam.startDateMilliseconds, + RawSqlQueryParam.endDateMilliseconds, +]); + +/** + * Query-parameter names the interval macros expand to. A time bucket can be + * written as a macro (`$__timeInterval(col)`) or as its expansion referencing + * these parameters directly (`toStartOfInterval(col, INTERVAL + * {intervalSeconds:Int64} second)`). + */ +const INTERVAL_QUERY_PARAMS: Set = new Set([ + RawSqlQueryParam.intervalSeconds, + RawSqlQueryParam.intervalMilliseconds, +]); + +/** + * ClickHouse named time-bucket functions (`toStartOfMinute`, `toStartOfHour`, etc) + * keyed by lower-cased name, mapped to the fixed granularity they represent. + */ +const NAMED_BUCKET_FNS_BY_LOWER: Map = new Map( + Object.entries(NAMED_BUCKET_FUNCTIONS).map(([fn, granularity]) => [ + fn.toLowerCase(), + granularity, + ]), +); + +/** + * `INTERVAL ` parses to a `toInterval(n)` function node; maps the + * lower-cased function name to the `SQLInterval` unit the builder understands. + * Units outside the builder's supported set (week/month/…) are intentionally + * absent, so those intervals aren't recognized as a builder granularity. + */ +const INTERVAL_FN_UNITS: Map = new Map([ + ['tointervalsecond', 'second'], + ['tointervalminute', 'minute'], + ['tointervalhour', 'hour'], + ['tointervalday', 'day'], +]); + +/** Builder-supported, single-arg functions (`fn[If](expr)`) */ +const SIMPLE_AGG_FNS: Set = new Set( + AggregateFunctionSchema.options.filter( + fn => + !['count', 'count_distinct', 'quantile', 'none', 'increase'].includes(fn), + ), +); + +/** Narrows a raw SQL function name to a simple builder aggFn. */ +function isSimpleAggFn(fn: string): fn is AggregateFunction { + return SIMPLE_AGG_FNS.has(fn); +} + +function isSupportedQuantileLevel(level: number): boolean { + return [0.5, 0.9, 0.95, 0.99].some(l => Math.abs(l - level) < 1e-9); +} + +/** + * The `count(DISTINCT …)` family, keyed by (lower-cased) function name. Approximate + * `uniq` is intentionally absent (it falls through to a raw `none` column). + */ +const COUNT_DISTINCT_FORMS: Map = new Map([ + ['countdistinct', { conditional: false }], + ['countifdistinct', { conditional: true }], + ['uniqexact', { conditional: false }], + ['uniqexactif', { conditional: true }], +]); + +type AggSpec = { + aggFn: AggregateFunction; + /** Fixed quantile level (e.g. `median` → 0.5). */ + level?: number; + /** Read the level from the call's parametric args (`quantile(0.95)(x)`). */ + levelFromParameters?: boolean; +}; + +/** Aggregate functions that map to quantile, keyed by their lower-cased base name. */ +const AGG_SPECS: Map = new Map([ + ['median', { aggFn: 'quantile', level: 0.5 }], + ['quantile', { aggFn: 'quantile', levelFromParameters: true }], +]); + +const macroName = (nodeName: string): string | undefined => + nodeName.startsWith(MACRO_SENTINEL_PREFIX) + ? nodeName.slice(MACRO_SENTINEL_PREFIX.length) + : undefined; + +/** Rewrites `$__macro`/`$__macro(args)` tokens to `hdx_macro_macro`(args) */ +export function replaceMacrosWithSentinels(sql: string): string { + return sql.replace(/\$__(\w+)/g, `${MACRO_SENTINEL_PREFIX}$1`); +} + +/** + * Collects the macro tokens (`$__name`) left in emitted builder SQL fields. + * Recognized macros are consumed during conversion (time bucket → granularity, + * time filter / `$__filters` dropped, `$__sourceTable` → FROM); anything that + * remains references a macro the builder has no field for. The parse step + * rewrote `$__name` to a `hdx_macro_name` sentinel, so that is what survives in + * the emitted strings — this reports it back in its original `$__name` form. + */ +function collectLeakedMacros(fields: string[]): string[] { + // Literal form of `${MACRO_SENTINEL_PREFIX}()` — kept literal so it isn't + // flagged as a dynamic RegExp. + const sentinelRe = /hdx_macro_(\w+)/g; + const macros = new Set(); + for (const field of fields) { + for (const match of field.matchAll(sentinelRe)) { + macros.add(`$__${match[1]}`); + } + } + return [...macros].sort(); +} + +/** Depth-first walk over every nested AST node reachable from `node`. */ +function walk(node: unknown, visit: (n: ASTNode) => void) { + if (!node) return; + + if (Array.isArray(node)) { + for (const item of node) { + walk(item, visit); + } + return; + } + + if (typeof node === 'object') { + if ('type' in node && typeof node.type === 'string') { + // eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion -- a string `type` marks a parser AST node + visit(node as ASTNode); + } + + // Recurse into every child node + for (const value of Object.values(node)) { + walk(value, visit); + } + } +} + +/** True when any node in the subtree rooted at `node` satisfies `predicate`. */ +function subtreeSome( + node: ASTNode, + predicate: (n: ASTNode) => boolean, +): boolean { + let found = false; + walk(node, n => { + if (!found && predicate(n)) { + found = true; + } + }); + return found; +} + +/** True when any node in `node`'s subtree is a `hdx_macro_` reference for + * a macro base name accepted by `predicate`. */ +function subtreeReferencesMacro( + node: Expression, + predicate: (name: string) => boolean, +): boolean { + // Macros are rewritten to sentinel identifiers (`$__fromTime`) or functions + // (`$__timeInterval(col)`), so only those two node kinds carry a macro name. + return subtreeSome(node, n => { + if (!isIdentifier(n) && !isFunction(n)) return false; + const base = macroName(n.name); + return base != null && predicate(base); + }); +} + +/** True when any `{name:Type}` query parameter in `node`'s subtree has a name + * accepted by `predicate`. */ +function subtreeReferencesQueryParam( + node: ASTNode, + predicate: (name: string) => boolean, +): boolean { + return subtreeSome(node, n => isQueryParameter(n) && predicate(n.name)); +} + +/** True when a subtree contains a dashboard time-range macro or query param. */ +function subtreeReferencesTimeRange(node: ASTNode): boolean { + return ( + subtreeSome(node, n => { + if (!isIdentifier(n) && !isFunction(n)) return false; + const base = macroName(n.name); + return base != null && TIME_RANGE_MACRO_NAMES.has(base); + }) || + subtreeReferencesQueryParam(node, name => TIME_RANGE_QUERY_PARAMS.has(name)) + ); +} + +/** True when any identifier/table in `node`'s subtree has the given name. */ +function subtreeReferencesName(node: Expression, name: string): boolean { + return subtreeSome( + node, + n => + (isIdentifier(n) || isFunction(n) || isTableIdentifier(n)) && + n.name === name, + ); +} + +/** Formats an AST expression back to a single-line SQL string. */ +function toSql(node: Expression): string { + return formatNode(node) + .replace(/\s*\n\s*/g, ' ') + .trim(); +} + +/** Formats an expression, stripping a trailing `AS alias` if present. */ +function toSqlWithoutAlias(node: Expression): string { + if ('alias' in node && node.alias) { + return toSql({ ...node, alias: undefined }); + } + return toSql(node); +} + +/** Parses and formats a standalone expression for stable AST comparisons. */ +function normalizeExpressionSql(expression: string): string | undefined { + try { + const statements = parse(`SELECT ${expression}`); + if (statements.length !== 1) return undefined; + const statement = statements[0]; + if ( + statement.type !== 'SelectWithUnionQuery' || + statement.selects.length !== 1 || + statement.selects[0].type !== 'SelectQuery' || + statement.selects[0].select.length !== 1 + ) { + return undefined; + } + return toSqlWithoutAlias(statement.selects[0].select[0]); + } catch { + return undefined; + } +} + +/** Timestamp expressions the current builder source will filter. */ +function configuredTimestampExpressions( + timestampValueExpression: string | undefined, +): Set { + if (!timestampValueExpression?.trim()) return new Set(); + return new Set( + splitAndTrimWithBracket(timestampValueExpression) + .map(normalizeExpressionSql) + .filter((expression): expression is string => expression != null), + ); +} + +/** Recognizes the renderer's unwrapped start/end bound. */ +function rawTimeBoundKind(node: Expression): TimeBoundKind | undefined { + if (isIdentifier(node)) { + const base = macroName(node.name); + if (base != null && START_TIME_MACROS.has(base)) return 'start'; + if (base != null && END_TIME_MACROS.has(base)) return 'end'; + return undefined; + } + if (!isFunction(node)) return undefined; + + const lower = node.name.toLowerCase(); + if (lower === 'fromunixtimestamp64milli' && node.arguments.length === 1) { + const parameter = node.arguments[0]; + if (!isQueryParameter(parameter) || parameter.param_type !== 'Int64') { + return undefined; + } + if (parameter.name === RawSqlQueryParam.startDateMilliseconds) { + return 'start'; + } + if (parameter.name === RawSqlQueryParam.endDateMilliseconds) return 'end'; + } + return undefined; +} + +/** Recognizes the renderer's exact included-data-interval expansion. */ +function includedIntervalBoundKind( + node: Expression, +): TimeBoundKind | undefined { + if (!isFunction(node) || node.arguments.length !== 2) return undefined; + const operation = node.name.toLowerCase(); + const intervalStart = node.arguments[0]; + if ( + (operation !== 'minus' && operation !== 'plus') || + !isFunction(intervalStart) || + intervalStart.name.toLowerCase() !== 'tostartofinterval' || + intervalStart.arguments.length !== 2 || + subtreeReferencesTimeRange(node.arguments[1]) || + toSql(intervalStart.arguments[1]) !== toSql(node.arguments[1]) + ) { + return undefined; + } + + const kind = rawTimeBoundKind(intervalStart.arguments[0]); + if (operation === 'minus' && kind === 'start') return kind; + if (operation === 'plus' && kind === 'end') return kind; + return undefined; +} + +function unwrappedTimeBoundKind(node: Expression): TimeBoundKind | undefined { + return rawTimeBoundKind(node) ?? includedIntervalBoundKind(node); +} + +/** + * Recognizes the complete RHS shape emitted for `lhs`. Renderer-added wrappers + * must match the timestamp expression instead of being accepted recursively. + */ +function rendererTimeBoundKind( + lhs: Expression, + rhs: Expression, +): TimeBoundKind | undefined { + if (isFunction(lhs)) { + const lhsName = lhs.name.toLowerCase(); + if (lhsName === 'todate') { + return isFunction(rhs) && + rhs.name.toLowerCase() === lhsName && + rhs.arguments.length === 1 + ? unwrappedTimeBoundKind(rhs.arguments[0]) + : undefined; + } + if (lhsName.startsWith('tostartof')) { + if ( + !isFunction(rhs) || + rhs.name.toLowerCase() !== lhsName || + rhs.arguments.length !== lhs.arguments.length || + rhs.arguments + .slice(1) + .some( + (argument, index) => + toSql(argument) !== toSql(lhs.arguments[index + 1]), + ) + ) { + return undefined; + } + return unwrappedTimeBoundKind(rhs.arguments[0]); + } + } + + const direct = unwrappedTimeBoundKind(rhs); + if (direct != null) return direct; + + // A bare Date column is only distinguishable through metadata during render, + // so accept the renderer's single toDate wrapper for configured expressions. + return isFunction(rhs) && + rhs.name.toLowerCase() === 'todate' && + rhs.arguments.length === 1 + ? unwrappedTimeBoundKind(rhs.arguments[0]) + : undefined; +} + +type TimestampMatch = { + baseExpression: string; + expression: string; +}; + +/** Matches a configured timestamp or one renderer-added primary-key bucket. */ +function matchTimestampExpression( + node: Expression, + configured: Set, +): TimestampMatch | undefined { + const expression = toSql(node); + if (configured.has(expression)) { + return { baseExpression: expression, expression }; + } + if (!isFunction(node) || !node.name.toLowerCase().startsWith('tostartof')) { + return undefined; + } + const firstArgument = node.arguments[0]; + if (firstArgument == null) return undefined; + const baseExpression = toSql(firstArgument); + return configured.has(baseExpression) + ? { baseExpression, expression } + : undefined; +} + +function unsupportedTimeRangeError(): SqlToBuilderError { + return new SqlToBuilderError( + 'This query uses dashboard time-range macros or parameters in a WHERE expression that cannot be represented by the chart builder.', + ); +} + +function timestampExpressionMismatchError( + sqlExpressions: string[], + configuredExpressions: Set, +): SqlToBuilderError { + const quotedSqlExpressions = sqlExpressions + .map(expression => `"${expression}"`) + .join(', '); + const quotedConfiguredExpressions = [...configuredExpressions] + .map(expression => `"${expression}"`) + .join(', '); + const sqlExpressionLabel = + sqlExpressions.length === 1 + ? 'timestamp expression' + : 'timestamp expressions'; + const sourceExpressionLabel = + configuredExpressions.size === 1 ? 'uses' : 'uses timestamp expressions'; + + return new SqlToBuilderError( + `The SQL time filter uses ${sqlExpressionLabel} ${quotedSqlExpressions}, but the selected source ${sourceExpressionLabel} ${quotedConfiguredExpressions}. Update the SQL time filter or select a matching source.`, + ); +} + +type RecognizedTimePredicate = TimestampMatch & { + bounds: readonly TimeBoundKind[]; +}; + +/** Maps a timestamp comparison function to the dashboard bound it represents. */ +function comparisonTimeBoundKind( + comparison: string, +): TimeBoundKind | undefined { + const lower = comparison.toLowerCase(); + if (lower === 'greater' || lower === 'greaterorequals') return 'start'; + if (lower === 'less' || lower === 'lessorequals') return 'end'; + return undefined; +} + +/** Positively recognizes one complete renderer-supported time conjunct. */ +function recognizeTimeConjunct( + conjunct: Expression, + configured: Set, +): RecognizedTimePredicate[] | undefined { + if (!isFunction(conjunct)) return undefined; + + const macro = macroName(conjunct.name); + if (macro != null && TIME_RANGE_MACRO_NAMES.has(macro)) { + const expressions = conjunct.arguments.map(toSql); + const expectedCount = macro === 'dateTimeFilter' || macro === 'dt' ? 2 : 1; + return expressions.length === expectedCount && + new Set(expressions).size === expectedCount && + expressions.every(expression => configured.has(expression)) + ? expressions.map(expression => ({ + baseExpression: expression, + bounds: ['start', 'end'], + expression, + })) + : undefined; + } + + if (conjunct.arguments.length !== 2) return undefined; + const expectedBound = comparisonTimeBoundKind(conjunct.name); + if (expectedBound == null) return undefined; + + const timestamp = matchTimestampExpression(conjunct.arguments[0], configured); + const bound = rendererTimeBoundKind( + conjunct.arguments[0], + conjunct.arguments[1], + ); + return timestamp != null && bound === expectedBound + ? [{ ...timestamp, bounds: [bound] }] + : undefined; +} + +/** + * Returns timestamp expressions from an otherwise valid time predicate that + * do not match the selected source. Malformed time predicates return an empty + * array so they retain the more general unsupported-time-range error. + */ +function mismatchedTimestampExpressions( + conjunct: Expression, + configured: Set, +): string[] { + if (!isFunction(conjunct)) return []; + + const macro = macroName(conjunct.name); + if (macro != null && TIME_RANGE_MACRO_NAMES.has(macro)) { + const expressions = conjunct.arguments.map(toSql); + const expectedCount = macro === 'dateTimeFilter' || macro === 'dt' ? 2 : 1; + if ( + expressions.length !== expectedCount || + new Set(expressions).size !== expectedCount + ) { + return []; + } + return expressions.filter(expression => !configured.has(expression)); + } + + if (conjunct.arguments.length !== 2) return []; + const expectedBound = comparisonTimeBoundKind(conjunct.name); + if (expectedBound == null) return []; + + const [lhs, rhs] = conjunct.arguments; + const bound = rendererTimeBoundKind(lhs, rhs); + if (bound !== expectedBound || matchTimestampExpression(lhs, configured)) { + return []; + } + return [toSql(lhs)]; +} + +type TimeRangePair = { + baseExpression: string; + bounds: Set; +}; + +/** + * Returns the indices of canonical time-range conjuncts that the builder will + * reproduce. Throws when any time reference is partial, targets another + * expression, or is nested in a shape the builder cannot represent. + */ +function canonicalTimeRangeConjuncts( + conjuncts: Expression[], + timestampValueExpression: string | undefined, +): Set { + const configured = configuredTimestampExpressions(timestampValueExpression); + const consumed = new Set(); + const pairs = new Map(); + + for (const [index, conjunct] of conjuncts.entries()) { + if (!subtreeReferencesTimeRange(conjunct)) continue; + const recognized = recognizeTimeConjunct(conjunct, configured); + if (recognized == null) { + const mismatchedExpressions = mismatchedTimestampExpressions( + conjunct, + configured, + ); + if (configured.size > 0 && mismatchedExpressions.length > 0) { + throw timestampExpressionMismatchError( + mismatchedExpressions, + configured, + ); + } + throw unsupportedTimeRangeError(); + } + consumed.add(index); + + for (const predicate of recognized) { + const pair = pairs.get(predicate.expression) ?? { + baseExpression: predicate.baseExpression, + bounds: new Set(), + }; + predicate.bounds.forEach(bound => pair.bounds.add(bound)); + pairs.set(predicate.expression, pair); + } + } + + if (pairs.size === 0) return consumed; + + for (const [expression, pair] of pairs) { + if (pair.bounds.size !== 2) throw unsupportedTimeRangeError(); + if ( + !configured.has(expression) && + pairs.get(pair.baseExpression)?.bounds.size !== 2 + ) { + throw unsupportedTimeRangeError(); + } + } + if ( + [...configured].some(expression => pairs.get(expression)?.bounds.size !== 2) + ) { + throw unsupportedTimeRangeError(); + } + return consumed; +} + +/** + * Flattens a left/variadic `and(...)` tree into its top-level conjuncts. + * `renderChartConfig` joins the time filter, WHERE, filters placeholder, and + * series-limit predicate with ` AND `, so the parser yields a single (often + * variadic) `and` node whose arguments are exactly those pieces. + */ +function flattenAnd(node: Expression): Expression[] { + if (isFunction(node) && node.name.toLowerCase() === 'and') { + return node.arguments.flatMap(flattenAnd); + } + return [node]; +} + +/** Flattens a variadic `or(...)` tree into its disjuncts. */ +function flattenOr(node: Expression): Expression[] { + if (isFunction(node) && node.name.toLowerCase() === 'or') { + return node.arguments.flatMap(flattenOr); + } + return [node]; +} + +/** ANDs two SQL condition strings, parenthesizing when both are present. */ +function andConditions(existing: string, addition: string): string { + const a = existing.trim(); + const b = addition.trim(); + if (!a) return b; + if (!b) return a; + return `(${a}) AND (${b})`; +} + +/** + * Unwraps the `toFloat64OrDefault(toString(expr))` or partial hand-written variants. + */ +function unwrapNumericCoercion(node: Expression): Expression { + const lower = isFunction(node) ? node.name.toLowerCase() : ''; + if ( + !isFunction(node) || + (lower !== 'tofloat64' && lower !== 'tofloat64ordefault') || + // Intentionally not unwrapping `toFloat64OrDefault(expr, )` + // because the builder doesn't represent the default value. + node.arguments.length !== 1 + ) { + return node; + } + + const inner = node.arguments[0]; + if ( + isFunction(inner) && + inner.name.toLowerCase() === 'tostring' && + inner.arguments.length === 1 + ) { + return inner.arguments[0]; + } + return inner; +} + +/** + * Extracts the user-authored aggregation condition from an `...If(...)` filter + * argument, dropping the trailing `AND IS NOT NULL` guard the builder + * appends for numeric aggregations. + */ +function extractAggCondition( + condition: Expression, + valueNode: Expression, +): string { + if (isFunction(condition) && condition.name.toLowerCase() === 'and') { + const conjuncts = flattenAnd(condition); + const valueSql = toSql(valueNode); + const kept = conjuncts.filter(conjunct => { + if ( + isFunction(conjunct) && + conjunct.name.toLowerCase() === 'isnotnull' && + conjunct.arguments.length === 1 && + toSql(conjunct.arguments[0]) === valueSql + ) { + return false; + } + return true; + }); + if (kept.length === 1) { + return toSql(kept[0]); + } + if (kept.length > 1) { + return kept.map(c => `(${toSql(c)})`).join(' AND '); + } + return ''; + } + return toSql(condition); +} + +/** Base fields shared by every parsed series column. */ +function makeColumn( + fields: Partial & { valueExpression: string }, +): DerivedColumn { + return { + aggCondition: '', + aggConditionLanguage: 'sql', + ...fields, + } as DerivedColumn; +} + +/** + * Maps a single SELECT expression to a builder series column. Recognizes the + * aggregation shapes `renderChartConfig` emits (count/countIf, count(DISTINCT), + * quantile(level), avg/sum/min/max/any/last_value and their `...If` variants); + * anything else becomes a raw `none` column carrying the verbatim expression. + */ +function parseSeriesColumn(node: Expression): DerivedColumn { + const alias = 'alias' in node && node.alias ? node.alias : undefined; + + const none = () => + makeColumn({ + aggFn: 'none', + valueExpression: toSqlWithoutAlias(node), + alias, + }); + + if (!isFunction(node)) return none(); + + // ClickHouse aggregate function names are case-insensitive, and the SQL + // formatter uppercases some of them (e.g. `AVG`), so match on lower case. + const { arguments: args } = node; + const lower = node.name.toLowerCase(); + + // count() / count(*) / countIf(cond). `count` is special: it carries no value + // expression, and countIf's condition is its first argument. `count(*)` parses + // to a single Asterisk argument, equivalent to the no-arg `count()`. + if ( + lower === 'count' && + (args.length === 0 || (args.length === 1 && args[0].type === 'Asterisk')) + ) { + return makeColumn({ aggFn: 'count', valueExpression: '', alias }); + } + if (lower === 'countif' && args.length >= 1) { + return makeColumn({ + aggFn: 'count', + valueExpression: '', + aggCondition: toSql(args[0]), + alias, + }); + } + + // count(DISTINCT expr) / uniqExact(expr) and their conditional forms all map + // to count_distinct, carrying the distinct expression in args[0]. + const distinctForm = COUNT_DISTINCT_FORMS.get(lower); + if (distinctForm) { + const expectedArgs = distinctForm.conditional ? 2 : 1; + if (args.length === expectedArgs) { + return makeColumn({ + aggFn: 'count_distinct', + valueExpression: toSql(args[0]), + aggCondition: distinctForm.conditional + ? extractAggCondition(args[1], args[0]) + : '', + alias, + }); + } + } + + // Every remaining supported aggregation carries its value in args[0] and, for + // the `...If` variant, its condition in args[1]. A trailing `If` selects the + // conditional form; the base name maps to a builder aggFn via AGG_SPECS or, + // for avg/sum/min/max/any/last_value, the simple-aggFn fallback. + const ifMatch = lower.match(/^(.+?)if$/); + const baseFn = ifMatch ? ifMatch[1] : lower; + const spec: AggSpec | undefined = + AGG_SPECS.get(baseFn) ?? + (isSimpleAggFn(baseFn) ? { aggFn: baseFn } : undefined); + if (spec && args.length >= 1) { + const column = buildAggColumn(spec, node, ifMatch != null, alias); + if (column) return column; + } + + // Unrecognized expression (or unsupported quantile level) → raw passthrough. + return none(); +} + +/** + * Builds a series column for an aggregate function call. The value + * expression is `args[0]` and, for an `...If` variant, the condition is + * `args[1]`. Returns `undefined` when the call doesn't match its spec. + */ +function buildAggColumn( + spec: AggSpec, + node: FunctionNode, + isConditional: boolean, + alias: string | undefined, +): DerivedColumn | undefined { + const { arguments: args, parameters } = node; + + let level = spec.level; + if (spec.levelFromParameters) { + if (!parameters || parameters.length !== 1) return undefined; + const levelNode = parameters[0]; + const parsed = levelNode.type === 'Literal' ? Number(levelNode.value) : NaN; + if (!isSupportedQuantileLevel(parsed)) return undefined; + level = parsed; + } + + const valueNode = isNumericAggFn(spec.aggFn) + ? unwrapNumericCoercion(args[0]) + : args[0]; + + return makeColumn({ + aggFn: spec.aggFn, + ...(level != null ? { level } : {}), + valueExpression: toSql(valueNode), + aggCondition: + isConditional && args.length >= 2 + ? extractAggCondition(args[1], args[0]) + : '', + alias, + } as Partial & { valueExpression: string }); +} + +/** + * A recognized time bucket and the timestamp expression needed to reproduce it + * in the builder. + */ +type TimeBucket = { + granularity: SQLInterval | 'auto'; + timestampValueExpression: string; +}; + +/** Removes the conversion wrapper added by the interval macro/renderer. */ +function unwrapBucketTimestamp(node: Expression): Expression { + if (!isFunction(node)) return node; + const lower = node.name.toLowerCase(); + if ( + (lower === 'todatetime' || lower === 'todatetime64') && + node.arguments.length >= 1 + ) { + return node.arguments[0]; + } + return node; +} + +/** + * Recognizes a complete time-bucket expression. The alias is deliberately + * ignored: only the expression itself proves that it is a bucket. + */ +function parseTimeBucket(node: Expression): TimeBucket | undefined { + if (!isFunction(node)) return undefined; + + const base = macroName(node.name); + if ( + base != null && + INTERVAL_MACRO_NAMES.has(base) && + node.arguments.length === 1 + ) { + return { + granularity: 'auto', + timestampValueExpression: toSql(node.arguments[0]), + }; + } + + const lower = node.name.toLowerCase(); + + // Named fixed buckets, e.g. `toStartOfMinute(ts)` / `toStartOfHour(ts)`. + const named = NAMED_BUCKET_FNS_BY_LOWER.get(lower); + if (named != null && node.arguments.length >= 1) { + return { + granularity: named, + timestampValueExpression: toSql(node.arguments[0]), + }; + } + + // Hand-written queries may spell out the interval macro's expansion directly, + // e.g. `toStartOfInterval(TimestampTime, INTERVAL {intervalSeconds:Int64} second)`. + if (lower !== 'tostartofinterval' || node.arguments.length < 2) + return undefined; + + const interval = node.arguments[1]; + const granularity = subtreeReferencesQueryParam(interval, name => + INTERVAL_QUERY_PARAMS.has(name), + ) + ? 'auto' + : fixedIntervalFromToStartOfInterval(node); + if (granularity == null) return undefined; + + return { + granularity, + timestampValueExpression: toSql(unwrapBucketTimestamp(node.arguments[0])), + }; +} + +/** Ensures the builder will reproduce the bucket against the same timestamp. */ +function validateTimeBucketTimestamp( + bucket: TimeBucket, + configuredExpression: string | undefined, +): void { + if ( + !configuredTimestampExpressions(configuredExpression).has( + bucket.timestampValueExpression, + ) + ) { + throw new SqlToBuilderError( + 'The query time bucket must use a timestamp expression configured by the selected source.', + ); + } +} + +/** + * Reads the fixed `SQLInterval` from a `toStartOfInterval(col, INTERVAL + * )` node. `INTERVAL ` parses to a `toInterval(n)` + * function argument; units outside the builder's set yield `undefined`. + */ +function fixedIntervalFromToStartOfInterval( + node: Expression, +): SQLInterval | undefined { + if (!isFunction(node) || node.arguments.length < 2) return undefined; + const intervalArg = node.arguments[1]; + if (!isFunction(intervalArg) || intervalArg.arguments.length !== 1) { + return undefined; + } + const unit = INTERVAL_FN_UNITS.get(intervalArg.name.toLowerCase()); + if (unit == null) return undefined; + const amount = intervalArg.arguments[0]; + if (amount.type !== 'Literal') return undefined; + const num = Number(amount.value); + if (!Number.isInteger(num) || num <= 0) return undefined; + return `${num} ${unit}` as SQLInterval; +} + +/** True when a SELECT/GROUP BY item is a time-bucket expression. */ +function isTimeBucket(node: Expression): boolean { + return parseTimeBucket(node) !== undefined; +} + +/** + * Aliases assigned to time-bucket SELECT items (e.g. `... AS ts`), so a GROUP BY + * or ORDER BY that references the bucket by that alias (`GROUP BY ts`) — the + * shape hand-written queries use — is recognized as the bucket too. + */ +function timeBucketAliases(query: SelectQueryNode): Set { + return new Set( + query.select + .filter(isTimeBucket) + .map(item => ('alias' in item ? item.alias : undefined)) + .filter((alias): alias is string => alias != null), + ); +} + +/** True when `node` is the time bucket itself or a reference to it by alias. */ +function isTimeBucketOrAlias(node: Expression, aliases: Set): boolean { + return isTimeBucket(node) || (isIdentifier(node) && aliases.has(node.name)); +} + +export type SqlToBuilderResult = { + select: DerivedColumn[]; + seriesReturnType?: 'ratio' | 'column'; + where: string; + whereLanguage: 'sql'; + groupBy: string; + granularity?: SQLInterval | 'auto'; + having?: string; + havingLanguage?: 'sql'; + orderBy?: string; + limit?: Limit; +}; + +/** + * Parses the (macro-substituted) SQL and validates it is a shape the builder + * can represent, returning the single SELECT query node. Throws a + * `SqlToBuilderError` for anything the builder has no representation for. + */ +function parseSelectQuery( + sqlTemplate: string, + source: BuilderSource | undefined, +): SelectQueryNode { + if (!sqlTemplate.trim()) { + throw new SqlToBuilderError('The SQL query is empty.'); + } + + let statements: Statement[]; + try { + statements = parse(replaceMacrosWithSentinels(sqlTemplate)); + } catch (e) { + const detail = e instanceof ParseError ? `: ${e.message}` : ''; + throw new SqlToBuilderError(`The SQL query could not be parsed${detail}.`); + } + + if (statements.length !== 1) { + throw new SqlToBuilderError( + 'Only a single SELECT statement can be converted to the builder.', + ); + } + + const [statement] = statements; + if (statement.type !== 'SelectWithUnionQuery') { + throw new SqlToBuilderError( + 'Only SELECT queries can be converted to the builder.', + ); + } + if (statement.selects.length !== 1) { + throw new SqlToBuilderError( + 'UNION queries cannot be converted to the builder.', + ); + } + const query = statement.selects[0]; + if (query.type !== 'SelectQuery') { + throw new SqlToBuilderError( + 'This query shape cannot be converted to the builder.', + ); + } + + // Reject constructs the builder has no representation for. + if (query.distinct) { + throw new SqlToBuilderError( + 'SELECT DISTINCT cannot be converted to the builder.', + ); + } + if (query.prewhere || query.qualify || query.window || query.limit_by) { + throw new SqlToBuilderError( + 'PREWHERE, QUALIFY, WINDOW, and LIMIT BY clauses are not supported by the builder.', + ); + } + if ((query.with ?? []).length > 0) { + throw new SqlToBuilderError('CTEs cannot be converted to the builder.'); + } + + // FROM must be a single table (the source table macro or a plain table); + // joins / subqueries can't be expressed in the builder. + const fromChildren = query.from?.children ?? []; + if (fromChildren.length !== 1) { + throw new SqlToBuilderError( + 'Only queries selecting from a single source table can be converted to the builder.', + ); + } + const tableExpr = fromChildren[0].table_expression; + if (!tableExpr || !tableExpr.database_and_table_name) { + throw new SqlToBuilderError( + 'Subqueries and table functions in FROM are not supported by the builder.', + ); + } + + const table = tableExpr.database_and_table_name; + if (table.alias) { + throw new SqlToBuilderError( + 'Source table aliases cannot be converted to the builder.', + ); + } + if (!source?.databaseName || !source.tableName) { + throw new SqlToBuilderError( + 'Select a builder source before converting SQL to the builder.', + ); + } + if (typeof table.name !== 'string') { + throw new SqlToBuilderError( + 'Parameterized source table names cannot be converted to the builder.', + ); + } + + const isSourceTableMacro = + table.database == null && macroName(table.name) === SOURCE_TABLE_MACRO_NAME; + const isMatchingLiteralTable = + table.name === source.tableName && + (table.database == null || table.database === source.databaseName); + if (!isSourceTableMacro && !isMatchingLiteralTable) { + const parsedSource = table.database + ? `${table.database}.${table.name}` + : table.name; + throw new SqlToBuilderError( + `The SQL source (${parsedSource}) does not match the builder source (${source.databaseName}.${source.tableName}).`, + ); + } + + return query; +} + +/** + * Resolves a GROUP BY reference to the SELECT expression it points at: a + * positional reference (`GROUP BY 1`) to the 1-based SELECT item, and an alias + * reference (`GROUP BY svc` for `... AS svc`) to the aliased item. Anything else + * passes through unchanged. + */ +function resolveGroupByReference( + item: Expression, + query: SelectQueryNode, +): Expression { + // Positional reference: a 1-based integer literal → that SELECT item. + if (item.type === 'Literal') { + const pos = Number(item.value); + if (Number.isInteger(pos) && pos >= 1 && pos <= query.select.length) { + return query.select[pos - 1]; + } + } + // Alias reference: an identifier matching a SELECT item's alias. + if (isIdentifier(item)) { + const aliased = query.select.find( + s => 'alias' in s && s.alias === item.name, + ); + if (aliased) return aliased; + } + return item; +} + +/** + * Splits GROUP BY into plain grouping columns and detects the builder's time + * bucket. Positional/alias references are first resolved to their SELECT + * expressions. Grouping sets / ROLLUP wrap columns in an ExpressionList; the + * builder has no representation for those, so only plain group-by expressions + * are kept. + */ +function parseGroupBy( + query: SelectQueryNode, + bucketAliases: Set, +): { + groupByColumns: Expression[]; + hasTimeBucket: boolean; +} { + const groupByItems = (query.group_by ?? []) + .filter((item): item is Expression => !isExpressionList(item)) + .map(item => resolveGroupByReference(item, query)); + const hasTimeBucket = + query.select.some(isTimeBucket) || + groupByItems.some(item => isTimeBucketOrAlias(item, bucketAliases)); + const groupByColumns = groupByItems.filter( + item => !isTimeBucketOrAlias(item, bucketAliases), + ); + return { groupByColumns, hasTimeBucket }; +} + +/** + * Finds the granularity of the query's time bucket (searching SELECT then + * GROUP BY, resolving positional/alias GROUP BY references), defaulting to + * `'auto'` when a bucket exists but its granularity can't be pinned down. + * Returns `undefined` when there is no time bucket. + */ +function detectTimeBucket(query: SelectQueryNode): TimeBucket | undefined { + for (const item of query.select) { + const bucket = parseTimeBucket(item); + if (bucket != null) return bucket; + } + for (const item of query.group_by ?? []) { + if (isExpressionList(item)) continue; + const bucket = parseTimeBucket(resolveGroupByReference(item, query)); + if (bucket != null) return bucket; + } + return undefined; +} + +/** + * Maps the SELECT list to builder series columns, dropping the time bucket and + * any columns repeated from GROUP BY. Ratio charts render as + * `divide(seriesA, seriesB)` and yield the two arguments as separate series. + */ +function parseSelect( + query: SelectQueryNode, + groupByColumns: Expression[], +): { select: DerivedColumn[]; seriesReturnType?: 'ratio' } { + const groupByStrings = new Set(groupByColumns.map(toSqlWithoutAlias)); + + const seriesNodes: Expression[] = []; + for (const item of query.select) { + if (isTimeBucket(item)) continue; + if (groupByStrings.has(toSqlWithoutAlias(item))) continue; + seriesNodes.push(item); + } + + if (seriesNodes.length === 0) { + throw new SqlToBuilderError( + 'No selectable columns were found in the query.', + ); + } + + if ( + seriesNodes.length === 1 && + isFunction(seriesNodes[0]) && + seriesNodes[0].name.toLowerCase() === 'divide' && + seriesNodes[0].arguments.length === 2 + ) { + const legs = seriesNodes[0].arguments.map(parseSeriesColumn); + // Only a `divide` of two aggregations is a builder ratio; a division of raw + // columns has no ratio representation, so fall through to a single raw + // column carrying the whole `divide(...)` expression. + if (legs.every(leg => leg.aggFn != null && leg.aggFn !== 'none')) { + return { seriesReturnType: 'ratio', select: legs }; + } + } + return { select: seriesNodes.map(parseSeriesColumn) }; +} + +/** + * Extracts the user-authored WHERE from the query's conjuncts. + * + * The builder emits the WHERE clause as ` AND `-joined pieces: the time filter, + * the top-level `where`, the per-series aggCondition OR-group (only when every + * series has one; see `renderWhere`), the `$__filters` placeholder, and the + * series-limit predicate. This strips the macro-derived and builder-internal + * pieces so only a genuine user WHERE remains. + * + * The aggregation display types this converter handles have no top-level WHERE + * input — only per-series conditions. When every series is an aggregation the + * surviving WHERE is broadcast into each series' `aggCondition` (mutating + * `select`, AND-ed with any existing `...If` condition) so it stays visible and + * editable in the builder, and `''` is returned. Otherwise — e.g. a raw + * (`none`) column that can't carry a condition — it is returned as the + * top-level `where` string (still applied at render, just not shown in the UI). + */ +function parseWhere( + query: SelectQueryNode, + select: DerivedColumn[], + timestampValueExpression: string | undefined, +): string { + const aggConditions = select.map(c => c.aggCondition ?? ''); + const everySeriesHasAggCondition = + select.length > 0 && aggConditions.every(c => c.trim() !== ''); + const aggConditionSet = new Set( + aggConditions.filter(c => c.trim() !== '').map(c => c.trim()), + ); + + const whereConjuncts = query.where ? flattenAnd(query.where) : []; + const timeRangeConjuncts = canonicalTimeRangeConjuncts( + whereConjuncts, + timestampValueExpression, + ); + const userWhere = whereConjuncts.filter((conjunct, index) => { + if (timeRangeConjuncts.has(index)) return false; + if (subtreeReferencesMacro(conjunct, name => name === FILTERS_MACRO_NAME)) { + return false; + } + // The aggCondition OR-group (`(cond1 OR cond2 …)`) the builder adds as an + // index hint duplicates the per-series conditions, so drop it rather than + // surface it as an extra WHERE. + if (everySeriesHasAggCondition) { + const disjuncts = flattenOr(conjunct).map(toSql); + if ( + disjuncts.length === aggConditionSet.size && + disjuncts.every(d => aggConditionSet.has(d)) + ) { + return false; + } + } + return true; + }); + const additionalWhere = + userWhere.length === 0 + ? '' + : userWhere.length === 1 + ? toSql(userWhere[0]) + : userWhere.map(c => `(${toSql(c)})`).join(' AND '); + + const canBroadcastWhere = + additionalWhere !== '' && + select.length > 0 && + select.every(c => c.aggFn != null && c.aggFn !== 'none'); + + if (canBroadcastWhere) { + for (const col of select) { + col.aggCondition = andConditions(col.aggCondition ?? '', additionalWhere); + } + return ''; + } + return additionalWhere; +} + +/** Renders ORDER BY, dropping the implicit time-bucket ordering. */ +function parseOrderBy( + query: SelectQueryNode, + bucketAliases: Set, +): string | undefined { + const orderByElements = (query.order_by ?? []).filter( + (el: OrderByElementNode) => + !isTimeBucketOrAlias(el.expression, bucketAliases), + ); + return orderByElements.length > 0 + ? orderByElements + .map(el => `${toSqlWithoutAlias(el.expression)} ${el.direction}`) + .join(', ') + : undefined; +} + +/** Reads a literal LIMIT (and optional OFFSET) into the builder's limit shape. + * A non-integer or negative literal is ignored rather than passed through. */ +function parseLimit(query: SelectQueryNode): Limit | undefined { + const limit = literalCount(query.limit); + if (limit == null) return undefined; + return { limit, offset: literalCount(query.offset) }; +} + +/** Reads a non-negative integer from a `Literal` node, or `undefined`. */ +function literalCount(node: Expression | undefined): number | undefined { + if (!node || node.type !== 'Literal') return undefined; + const n = Number(node.value); + return Number.isInteger(n) && n >= 0 ? n : undefined; +} + +/** Rejects clause combinations the given display type cannot represent. */ +function validateDisplayType({ + displayType, + typeLabel, + isTimeSeries, + hasTimeBucket, + groupByColumns, + having, + orderBy, + select, + seriesReturnType, +}: { + displayType: DisplayType; + typeLabel: string; + isTimeSeries: boolean; + hasTimeBucket: boolean; + groupByColumns: Expression[]; + having: string | undefined; + orderBy: string | undefined; + select: DerivedColumn[]; + seriesReturnType: 'ratio' | undefined; +}): void { + const supportsGroupBy = displayType !== DisplayType.Number; + const supportsOrderBy = + displayType === DisplayType.Table || + displayType === DisplayType.Bar || + displayType === DisplayType.Pie; + + if (hasTimeBucket && !isTimeSeries) { + throw new SqlToBuilderError( + `Time bucketing is only supported for Time Series and Bar charts, not ${typeLabel} charts.`, + ); + } + if (groupByColumns.length > 0 && !supportsGroupBy) { + throw new SqlToBuilderError( + `GROUP BY is not supported for ${typeLabel} charts.`, + ); + } + if (having != null && displayType !== DisplayType.Table) { + throw new SqlToBuilderError( + `HAVING is only supported for Table charts, not ${typeLabel} charts.`, + ); + } + if (orderBy != null && !supportsOrderBy) { + throw new SqlToBuilderError( + `ORDER BY is only supported for Table, Bar, and Pie charts, not ${typeLabel} charts.`, + ); + } + if ( + (displayType === DisplayType.Pie || displayType === DisplayType.Bar) && + select.length > 1 + ) { + throw new SqlToBuilderError( + `${typeLabel} charts support only a single series.`, + ); + } + if ( + displayType === DisplayType.Number && + seriesReturnType !== 'ratio' && + select.length > 1 + ) { + throw new SqlToBuilderError( + 'Number charts support a single series unless ratio mode is used.', + ); + } +} + +/** + * SQL → Builder conversion. + * + * This does not handle every possible SQL query - many cannot be represented as + * a builder chart config, and some (metrics!) are exceedingly complicated to + * convert, so this is a best-effort conversion that will return a user-facing + * error if the SQL cannot be converted. + * + * Macros are dynamic template tokens (`$__fromTime_ms`, `$__timeInterval(col)`, etc) + * that are not valid ClickHouse SQL, so they are first swapped for parseable sentinels + * (`hdx_macro_`), parsed, and then recognized in the AST when building the config. + */ +export function convertRawSqlToBuilderConfig({ + sqlTemplate, + displayType, + from, + timestampValueExpression, +}: { + sqlTemplate: string; + displayType: DisplayType; + from?: BuilderSource; + timestampValueExpression?: string; +}): SqlToBuilderResult { + // eslint-disable-next-line security/detect-object-injection + const typeLabel = DISPLAY_TYPE_LABELS[displayType] ?? 'this'; + + const query = parseSelectQuery(sqlTemplate, from); + + const bucketAliases = timeBucketAliases(query); + const { groupByColumns, hasTimeBucket } = parseGroupBy(query, bucketAliases); + const timeBucket = detectTimeBucket(query); + if (timeBucket != null) { + validateTimeBucketTimestamp(timeBucket, timestampValueExpression); + } + const { select, seriesReturnType } = parseSelect(query, groupByColumns); + // parseWhere may broadcast the WHERE into each series' aggCondition (mutating + // `select`), so it must run after the series columns are built. + const where = parseWhere(query, select, timestampValueExpression); + const orderBy = parseOrderBy(query, bucketAliases); + const having = query.having ? toSql(query.having) : undefined; + const limit = parseLimit(query); + const granularity = hasTimeBucket ? timeBucket?.granularity : undefined; + + const isTimeSeries = + displayType === DisplayType.Line || displayType === DisplayType.StackedBar; + + validateDisplayType({ + displayType, + typeLabel, + isTimeSeries, + hasTimeBucket, + groupByColumns, + having, + orderBy, + select, + seriesReturnType, + }); + + const groupBy = groupByColumns.map(toSqlWithoutAlias).join(', '); + + // The builder has no representation for macros. Recognized ones are consumed + // during conversion; any that survived into an emitted field (a raw column, + // WHERE, GROUP BY, ORDER BY, HAVING) can't be represented, so fail. + const leakedMacros = collectLeakedMacros([ + ...select.flatMap(c => [c.valueExpression, c.aggCondition ?? '']), + where, + groupBy, + orderBy ?? '', + having ?? '', + ]); + if (leakedMacros.length > 0) { + throw new SqlToBuilderError( + `This query uses macros in locations that cannot be represented in the chart builder: ${leakedMacros.join(', ')}.`, + ); + } + + // A recognized time bucket on a non-time-series chart is already rejected + // above; this catches an interval query parameter that survived into a raw + // column, WHERE, etc. (it only binds on time-series charts). + if ( + !isTimeSeries && + subtreeReferencesQueryParam(query, name => INTERVAL_QUERY_PARAMS.has(name)) + ) { + throw new SqlToBuilderError( + `The interval query parameter ({intervalSeconds}/{intervalMilliseconds}) only resolves on Time Series charts, not ${typeLabel} charts.`, + ); + } + + return { + select, + ...(seriesReturnType ? { seriesReturnType } : {}), + where, + whereLanguage: 'sql', + groupBy, + ...(granularity != null && isTimeSeries ? { granularity } : {}), + ...(having != null ? { having, havingLanguage: 'sql' as const } : {}), + ...(orderBy != null ? { orderBy } : {}), + ...(limit != null ? { limit } : {}), + }; +} diff --git a/packages/common-utils/src/core/renderChartConfig.ts b/packages/common-utils/src/core/renderChartConfig.ts index faa05f0003..2d057f9cb9 100644 --- a/packages/common-utils/src/core/renderChartConfig.ts +++ b/packages/common-utils/src/core/renderChartConfig.ts @@ -500,6 +500,18 @@ export const rewriteSqlFilterWithKvItems = ( } }; +/** + * Whether the aggregate function operates on numeric values + * and is therefore wrapped with `toFloat64OrDefault(toString(...))` + **/ +export const isNumericAggFn = ( + fn: AggregateFunction | AggregateFunctionWithCombinators, +) => { + const isAny = fn === 'any'; + const isNone = fn === 'none'; + return !(isAny || isNone); +}; + const aggFnExpr = ({ fn, expr, @@ -513,14 +525,14 @@ const aggFnExpr = ({ where?: string; sampleWeightExpression?: string; }) => { - const isAny = fn === 'any'; - const isNone = fn === 'none'; + const isNumeric = isNumericAggFn(fn); const isCount = fn.startsWith('count'); const isWhereUsed = isNonEmptyWhereExpr(where); // Cast to float64 because the expr might not be a number const unsafeExpr = { - UNSAFE_RAW_SQL: - isAny || isNone ? `${expr}` : `toFloat64OrDefault(toString(${expr}))`, + UNSAFE_RAW_SQL: !isNumeric + ? `${expr}` + : `toFloat64OrDefault(toString(${expr}))`, }; const whereWithExtraNullCheck = `${where} AND ${unsafeExpr.UNSAFE_RAW_SQL} IS NOT NULL`; diff --git a/packages/common-utils/src/macros.ts b/packages/common-utils/src/macros.ts index c6517cd4b1..73af36e82f 100644 --- a/packages/common-utils/src/macros.ts +++ b/packages/common-utils/src/macros.ts @@ -149,10 +149,12 @@ const MACROS: Macro[] = [ }, ]; +export const FILTERS_MACRO_NAME = 'filters'; + /** Macro metadata for autocomplete suggestions */ export const MACRO_SUGGESTIONS = [ ...MACROS.map(({ name, minArgs, maxArgs }) => ({ name, minArgs, maxArgs })), - { name: 'filters', minArgs: 0, maxArgs: 0 }, + { name: FILTERS_MACRO_NAME, minArgs: 0, maxArgs: 0 }, { name: 'sourceTable', minArgs: 0, maxArgs: 1 }, ...Object.values(MetricsDataType).map(type => ({ name: `sourceTable(${type})`, @@ -172,7 +174,10 @@ export const MACRO_SUGGESTIONS = [ * Every other macro only needs the dashboard time range / interval and takes * its column as an argument, so it does not require a source. */ -export const SOURCE_DEPENDENT_MACROS = ['filters', 'sourceTable'] as const; +export const SOURCE_DEPENDENT_MACROS = [ + FILTERS_MACRO_NAME, + 'sourceTable', +] as const; /** * Time-range macros. Any single one binds a raw @@ -275,7 +280,7 @@ export function replaceMacros( const allMacros: Macro[] = [ ...MACROS, { - name: 'filters', + name: FILTERS_MACRO_NAME, minArgs: 0, maxArgs: 0, replace: () => filtersSQL || NO_FILTERS, diff --git a/packages/common-utils/src/types.ts b/packages/common-utils/src/types.ts index fd830d07ea..b70ee56a00 100644 --- a/packages/common-utils/src/types.ts +++ b/packages/common-utils/src/types.ts @@ -43,6 +43,20 @@ export enum DisplayType { EventPatterns = 'event_patterns', } +/** User-facing display-type labels */ +export const DISPLAY_TYPE_LABELS: Record = { + [DisplayType.Line]: 'Time Series', + [DisplayType.StackedBar]: 'Bar', + [DisplayType.Table]: 'Table', + [DisplayType.Pie]: 'Pie', + [DisplayType.Bar]: 'Bar', + [DisplayType.Number]: 'Number', + [DisplayType.Search]: 'Search', + [DisplayType.Heatmap]: 'Heatmap', + [DisplayType.Markdown]: 'Markdown', + [DisplayType.EventPatterns]: 'Event Patterns', +}; + export type KeyValue = { key: Key; value: Value }; export const MetricTableSchema = z @@ -309,7 +323,7 @@ export type SelectList = z.infer; export type SortSpecificationList = z.infer; -type Limit = { limit?: number; offset?: number }; +export type Limit = { limit?: number; offset?: number }; export type SelectSQLStatement = { select: SelectList; diff --git a/yarn.lock b/yarn.lock index 23f5a9fede..8d6876bd4e 100644 --- a/yarn.lock +++ b/yarn.lock @@ -3139,6 +3139,15 @@ __metadata: languageName: node linkType: hard +"@clickhouse/parser@npm:0.3.0": + version: 0.3.0 + resolution: "@clickhouse/parser@npm:0.3.0" + dependencies: + zod: "npm:^4.3.6" + checksum: 10c0/2f65cb64586922262a25c9ca202c38a8f67daf8ddc5c3e1fabbcee60314f1c3b9da9e102212f2bba795fde866aab2cfe32d3a33d7361f544036521a506fc632d + languageName: node + linkType: hard + "@codemirror/autocomplete@npm:^6.0.0": version: 6.13.0 resolution: "@codemirror/autocomplete@npm:6.13.0" @@ -4693,6 +4702,7 @@ __metadata: "@clickhouse/client": "npm:1.23.0-head.fae5998.1" "@clickhouse/client-common": "npm:1.23.0-head.fae5998.1" "@clickhouse/client-web": "npm:1.23.0-head.fae5998.1" + "@clickhouse/parser": "npm:0.3.0" "@hyperdx/lucene": "npm:^3.1.1" "@types/hyperdx__lucene": "npm:@types/lucene@*" "@types/jest": "npm:^29.5.14"