fix: stringify JSON search sidebar filters

.changeset/tame-json-sidebar-filters.md

@@ -0,0 +1,6 @@
+---
+'@hyperdx/app': patch
+'@hyperdx/common-utils': patch
+---
+
+Fix JSON-backed search sidebar filters and metadata value queries to serialize resource attributes as ClickHouse string expressions, prioritize selected and pinned fields during facet loading, refresh loaded facet values when the active filter context changes, and surface load-more actions for empty facets that can fetch additional values.

packages/app/src/DBSearchPage.tsx

@@ -115,7 +115,10 @@ import {
   useSavedSearch,
   useUpdateSavedSearch,
 } from '@/savedSearch';
-import { useSearchPageFilterState } from '@/searchFilters';
+import {
+  canonicalizeFilterQuery,
+  useSearchPageFilterState,
+} from '@/searchFilters';
 import { getEventBody, useSource, useSources } from '@/source';
 import { useAppTheme, useBrandDisplayName } from '@/theme/ThemeProvider';
 import {
@@ -144,6 +147,7 @@ import {
 } from './components/TimePicker/utils';
 import {
   useColumns,
+  useJsonColumns,
   useResolvedDateTimeColumns,
   useTableMetadata,
 } from './hooks/useMetadata';
@@ -1069,6 +1073,7 @@ export function DBSearchPage() {
       },
       resolver: zodResolver(SearchConfigSchema),
     });
+  const canonicalizedFiltersSyncRef = useRef<Filter[] | null>(null);
 
   const inputSource = useWatch({ name: 'source', control });
 
@@ -1103,6 +1108,10 @@ export function DBSearchPage() {
   // const { data: inputSourceObj } = useSource({ id: inputSource });
   const { data: inputSourceObjs } = useSources();
   const inputSourceObj = inputSourceObjs?.find(s => s.id === inputSource);
+  const inputSourceTableConnection = useMemo(
+    () => tcFromSource(inputSourceObj),
+    [inputSourceObj],
+  );
 
   const [displayedTimeInputValue, setDisplayedTimeInputValue] =
     useState('Live Tail');
@@ -1122,6 +1131,15 @@ export function DBSearchPage() {
   const prevSearched = usePrevious(searchedConfig);
   useEffect(() => {
     if (JSON.stringify(prevSearched) !== JSON.stringify(searchedConfig)) {
+      if (
+        canonicalizedFiltersSyncRef.current != null &&
+        JSON.stringify(canonicalizedFiltersSyncRef.current) ===
+          JSON.stringify(searchedConfig.filters ?? [])
+      ) {
+        canonicalizedFiltersSyncRef.current = null;
+        return;
+      }
+
       reset({
         select: searchedConfig?.select ?? '',
         where: searchedConfig?.where ?? '',
@@ -1255,6 +1273,10 @@ export function DBSearchPage() {
         : new Set<string>(),
     [inputSourceColumns],
   );
+  const { data: inputSourceJsonColumns = [] } = useJsonColumns(
+    inputSourceTableConnection,
+    { enabled: !!inputSourceObj },
+  );
 
   const watchedSource = useWatch({
     control,
@@ -1285,11 +1307,43 @@ export function DBSearchPage() {
     useResolvedDateTimeColumns(inputSourceColumns);
 
   const filters = useWatch({ name: 'filters', control });
+  const canonicalizedFilters = useMemo(() => {
+    if (!filters?.length || !inputSourceColumns) {
+      return null;
+    }
+
+    const nextFilters = canonicalizeFilterQuery(
+      filters,
+      knownColumns,
+      inputSourceJsonColumns,
+      dateTimeColumns,
+    );
+
+    return JSON.stringify(nextFilters) === JSON.stringify(filters)
+      ? null
+      : nextFilters;
+  }, [
+    dateTimeColumns,
+    filters,
+    inputSourceColumns,
+    inputSourceJsonColumns,
+    knownColumns,
+  ]);
+  useEffect(() => {
+    if (!canonicalizedFilters) {
+      return;
+    }
+
+    setValue('filters', canonicalizedFilters);
+    canonicalizedFiltersSyncRef.current = canonicalizedFilters;
+    setSearchedConfig({ filters: canonicalizedFilters });
+  }, [canonicalizedFilters, setSearchedConfig, setValue]);
   const searchFilters = useSearchPageFilterState({
     searchQuery: filters ?? undefined,
     onFilterChange: handleSetFilters,
     dateTimeColumns,
     knownColumns,
+    jsonColumns: inputSourceJsonColumns,
   });
 
   useEffect(() => {
@@ -1878,11 +1932,6 @@ export function DBSearchPage() {
     ],
   );
 
-  const inputSourceTableConnection = useMemo(
-    () => tcFromSource(inputSourceObj),
-    [inputSourceObj],
-  );
-
   const [isSourceSchemaPreviewOpen, setIsSourceSchemaPreviewOpen] =
     useState(false);
 

packages/app/src/__tests__/DBSearchPage.directTrace.test.tsx

@@ -11,10 +11,15 @@ const mockSetAnalysisMode = jest.fn();
 const mockSetIsLive = jest.fn();
 const mockOnSearch = jest.fn();
 const mockOnTimeRangeSelect = jest.fn();
+const mockCanonicalizeFilterQuery = jest.fn();
+const mockFormReset = jest.fn();
 
 let mockDirectTraceId: string | null = null;
 let mockSearchedConfig: Record<string, any> = {};
+let mockApplySearchedConfigUpdates = false;
 let mockSources: any[] = [];
+let mockColumns: any[] | undefined;
+let mockJsonColumns: string[] = [];
 let latestDirectTracePanelProps: Record<string, any> | null = null;
 
 jest.mock('@/layout', () => ({
@@ -69,9 +74,42 @@ jest.mock('nuqs', () => ({
         return [null, jest.fn()];
     }
   },
-  useQueryStates: () => [mockSearchedConfig, mockSetSearchedConfig],
+  useQueryStates: () => {
+    const [, forceRender] = React.useReducer(value => value + 1, 0);
+    const setSearchedConfig = React.useCallback(
+      (value: Record<string, any>) => {
+        mockSetSearchedConfig(value);
+        if (mockApplySearchedConfigUpdates) {
+          mockSearchedConfig = { ...mockSearchedConfig, ...value };
+          forceRender();
+        }
+      },
+      [],
+    );
+    return [mockSearchedConfig, setSearchedConfig];
+  },
 }));
 
+jest.mock('react-hook-form', () => {
+  const actual = jest.requireActual('react-hook-form');
+
+  return {
+    ...actual,
+    useForm: (options: unknown) => {
+      const form = actual.useForm(options);
+      const originalReset = form.reset;
+      const reset = React.useCallback(
+        (...args: Parameters<typeof originalReset>) => {
+          mockFormReset(...args);
+          return originalReset(...args);
+        },
+        [originalReset],
+      );
+      return { ...form, reset };
+    },
+  };
+});
+
 jest.mock('@/source', () => ({
   getEventBody: () => 'Body',
   getFirstTimestampValueExpression: () => 'Timestamp',
@@ -106,6 +144,8 @@ jest.mock('@/savedSearch', () => ({
 }));
 
 jest.mock('@/searchFilters', () => ({
+  canonicalizeFilterQuery: (...args: unknown[]) =>
+    mockCanonicalizeFilterQuery(...args),
   useSearchPageFilterState: () => ({
     filters: [],
     whereSuggestions: [],
@@ -134,7 +174,11 @@ jest.mock('../hooks/useMetadata', () => ({
     isLoading: false,
   }),
   useColumns: () => ({
-    data: undefined,
+    data: mockColumns,
+    isLoading: false,
+  }),
+  useJsonColumns: () => ({
+    data: mockJsonColumns,
     isLoading: false,
   }),
 }));
@@ -245,7 +289,14 @@ jest.mock('@/utils', () => ({
     initialValue,
     jest.fn(),
   ],
-  usePrevious: (value: unknown) => value,
+  usePrevious: (value: unknown) => {
+    const ref = React.useRef(value);
+    const previous = ref.current;
+    React.useEffect(() => {
+      ref.current = value;
+    }, [value]);
+    return previous;
+  },
 }));
 
 jest.mock('@tanstack/react-query', () => ({
@@ -257,6 +308,10 @@ describe('DBSearchPage direct trace flow', () => {
     jest.clearAllMocks();
     latestDirectTracePanelProps = null;
     mockDirectTraceId = 'trace-123';
+    mockApplySearchedConfigUpdates = false;
+    mockColumns = undefined;
+    mockJsonColumns = [];
+    mockCanonicalizeFilterQuery.mockImplementation(filters => filters);
     mockSearchedConfig = {
       source: undefined,
       where: '',
@@ -380,4 +435,52 @@ describe('DBSearchPage direct trace flow', () => {
 
     expect(mockSetDirectTraceId).toHaveBeenCalledWith(null);
   });
+
+  it('canonicalizes only the applied filters without resetting form drafts', async () => {
+    mockDirectTraceId = null;
+    mockApplySearchedConfigUpdates = true;
+    const staleFilters = [
+      {
+        type: 'sql',
+        condition: "ResourceAttributes['k8s.namespace.name'] IN ('backend')",
+      },
+    ];
+    const canonicalFilters = [
+      {
+        type: 'sql',
+        condition:
+          "toString(ResourceAttributes.`k8s`.`namespace`.`name`) IN ('backend')",
+      },
+    ];
+    mockSearchedConfig = {
+      source: 'log-source',
+      where: 'ServiceName:api',
+      select: 'Timestamp,Body',
+      whereLanguage: 'lucene',
+      filters: staleFilters,
+      orderBy: 'Timestamp DESC',
+    };
+    mockColumns = [
+      {
+        name: 'ResourceAttributes',
+        type: 'JSON(max_dynamic_types=8, max_dynamic_paths=64)',
+      },
+    ];
+    mockJsonColumns = ['ResourceAttributes'];
+    mockCanonicalizeFilterQuery.mockReturnValue(canonicalFilters);
+
+    renderWithMantine(<DBSearchPage />);
+
+    await waitFor(() => {
+      expect(mockSetSearchedConfig).toHaveBeenCalledWith({
+        filters: canonicalFilters,
+      });
+    });
+    await waitFor(() => {
+      expect(mockSearchedConfig.filters).toEqual(canonicalFilters);
+    });
+
+    expect(mockSetSearchedConfig).toHaveBeenCalledTimes(1);
+    expect(mockFormReset).not.toHaveBeenCalled();
+  });
 });