Skip to content

Commit 418f70c

Browse files
wrn14897wrn14897
authored
[HDX-3964] Add event pattern mining to CLI (Shift+P) (#2106)
## Summary Adds a pattern mining feature to the CLI, accessible via `Shift+P`. This mirrors the web app's Pattern Table functionality but runs entirely in TypeScript — no Pyodide/Python WASM needed. **Linear:** https://linear.app/hyperdx/issue/HDX-3964 ## What changed ### 1. Drain library in common-utils (`packages/common-utils/src/drain/`) Ported the [browser-drain](https://github.com/DeploySentinel/browser-drain) TypeScript library into `@hyperdx/common-utils`. This is a pure TypeScript implementation of the Drain3 log template mining algorithm, including: - `TemplateMiner` / `TemplateMinerConfig` — main API - `Drain` — core algorithm with prefix tree and LRU cluster cache - `LogMasker` — regex-based token masking (IPs, numbers, etc.) - `LruCache` — custom LRU cache matching Python Drain3's eviction semantics - 11 Jest tests ported from the original `node:test` suite ### 2. CLI pattern view (`packages/cli/src/components/EventViewer/`) **Keybinding:** `Shift+P` toggles pattern view (pauses follow mode, restores on exit) **Data flow (mirrors web app's `useGroupedPatterns`):** - Issues `SELECT ... ORDER BY rand() LIMIT 100000` to randomly sample up to 100K events - Issues parallel `SELECT count()` to get true total event count - Feeds sampled log bodies through the TypeScript `TemplateMiner` - Estimates pattern counts via `sampleMultiplier = totalCount / sampledRowCount` - Computes time-bucketed trend data per pattern **UI:** - Pattern list with columns: Est. Count (with `~` prefix), Pattern - `l`/`Enter` expands a pattern to show its sample events (full table columns) - `h`/`Esc` returns to pattern list - `j/k/G/g/Ctrl+D/Ctrl+U` navigation throughout - Loading spinner while sampling query runs **Alias fix:** Pattern and count queries compute `WITH` clauses from the source's `defaultTableSelectExpression` so Lucene searches using aliases (e.g. `level:error` where `level` is an alias for `SeverityText`) resolve correctly. ### New files - `packages/common-utils/src/drain/` — 7 source files + barrel index - `packages/common-utils/src/__tests__/drain.test.ts` - `packages/cli/src/components/EventViewer/usePatternData.ts` - `packages/cli/src/components/EventViewer/PatternView.tsx` - `packages/cli/src/components/EventViewer/PatternSamplesView.tsx` ### Modified files - `packages/cli/src/api/eventQuery.ts` — added `buildPatternSampleQuery`, `buildTotalCountQuery`, `buildAliasWithClauses` - `packages/cli/src/components/EventViewer/EventViewer.tsx` — wired in pattern state + rendering - `packages/cli/src/components/EventViewer/useKeybindings.ts` — added P, l, h keybindings + pattern/sample navigation - `packages/cli/src/components/EventViewer/SubComponents.tsx` — added P to help screen ### Demo https://github.com/user-attachments/assets/50a2edfc-8891-43ae-ab86-b96fca778c66
1 parent 7a49c74 commit 418f70c

21 files changed

Lines changed: 2119 additions & 5 deletions

‎.changeset/add-cli-pattern-mining.md‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
---
2+
"@hyperdx/cli": patch
3+
---
4+
5+
Add event pattern mining view (Shift+P) with sampled estimation and drill-down

‎.changeset/add-drain-library.md‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
---
2+
"@hyperdx/common-utils": patch
3+
---
4+
5+
Add Drain log template mining library (ported from browser-drain)

‎package.json‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -42,6 +42,7 @@
4242
"run:clickhouse": "nx run @hyperdx/app:run:clickhouse",
4343
"dev": "sh -c '. ./scripts/dev-env.sh && yarn build:common-utils && dotenvx run --convention=nextjs -- docker compose -p \"$HDX_DEV_PROJECT\" -f docker-compose.dev.yml up -d && yarn app:dev; dotenvx run --convention=nextjs -- docker compose -p \"$HDX_DEV_PROJECT\" -f docker-compose.dev.yml down'",
4444
"dev:local": "IS_LOCAL_APP_MODE='DANGEROUSLY_is_local_app_mode💀' yarn dev",
45+
"cli:dev": "yarn workspace @hyperdx/cli dev",
4546
"dev:down": "sh -c '. ./scripts/dev-env.sh && docker compose -p \"$HDX_DEV_PROJECT\" -f docker-compose.dev.yml down && sh ./scripts/dev-kill-ports.sh'",
4647
"dev:compose": "sh -c '. ./scripts/dev-env.sh && docker compose -p \"$HDX_DEV_PROJECT\" -f docker-compose.dev.yml'",
4748
"knip": "knip",

‎packages/cli/src/api/eventQuery.ts‎

Lines changed: 139 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,7 @@ import type {
1010
import { chSqlToAliasMap } from '@hyperdx/common-utils/dist/clickhouse';
1111
import { renderChartConfig } from '@hyperdx/common-utils/dist/core/renderChartConfig';
1212
import type { Metadata } from '@hyperdx/common-utils/dist/core/metadata';
13+
import { aliasMapToWithClauses } from '@hyperdx/common-utils/dist/core/utils';
1314
import { DisplayType } from '@hyperdx/common-utils/dist/types';
1415
import type {
1516
BuilderChartConfigWithDateRange,
@@ -106,6 +107,144 @@ export async function buildEventSearchQuery(
106107
return renderChartConfig(config, metadata, source.querySettings);
107108
}
108109

110+
// ---- Alias WITH clauses from source select --------------------------
111+
112+
/**
113+
* Compute WITH clauses from the source's default select expression.
114+
* When a source defines `SeverityText as level`, searches for `level:error`
115+
* need `WITH SeverityText AS level` so the alias is available in WHERE.
116+
*/
117+
async function buildAliasWithClauses(
118+
source: SourceResponse,
119+
metadata: Metadata,
120+
): Promise<BuilderChartConfigWithDateRange['with']> {
121+
const selectExpr = source.defaultTableSelectExpression;
122+
if (!selectExpr) return undefined;
123+
124+
const tsExpr = source.timestampValueExpression ?? 'TimestampTime';
125+
126+
// Render a dummy query with the source's select to extract aliases
127+
const dummyConfig: BuilderChartConfigWithDateRange = {
128+
displayType: DisplayType.Search,
129+
select: selectExpr,
130+
from: source.from,
131+
where: '',
132+
connection: source.connection,
133+
timestampValueExpression: tsExpr,
134+
implicitColumnExpression: source.implicitColumnExpression,
135+
limit: { limit: 0 },
136+
dateRange: [new Date(), new Date()],
137+
};
138+
139+
const dummySql = await renderChartConfig(
140+
dummyConfig,
141+
metadata,
142+
source.querySettings,
143+
);
144+
const aliasMap = chSqlToAliasMap(dummySql);
145+
return aliasMapToWithClauses(aliasMap);
146+
}
147+
148+
// ---- Pattern sampling query -----------------------------------------
149+
150+
export interface PatternSampleQueryOptions {
151+
source: SourceResponse;
152+
searchQuery?: string;
153+
startTime: Date;
154+
endTime: Date;
155+
/** Number of random rows to sample (default 100_000) */
156+
sampleLimit?: number;
157+
}
158+
159+
/**
160+
* Build a query that randomly samples events for pattern mining.
161+
* Selects the body column and timestamp, ordered by rand().
162+
*/
163+
export async function buildPatternSampleQuery(
164+
opts: PatternSampleQueryOptions,
165+
metadata: Metadata,
166+
): Promise<ChSql> {
167+
const {
168+
source,
169+
searchQuery = '',
170+
startTime,
171+
endTime,
172+
sampleLimit = 100_000,
173+
} = opts;
174+
175+
const tsExpr = source.timestampValueExpression ?? 'TimestampTime';
176+
177+
// Use the same select as the main event table so sample rows have all columns
178+
let selectExpr = source.defaultTableSelectExpression ?? '';
179+
if (!selectExpr && source.kind === 'trace') {
180+
selectExpr = buildTraceSelectExpression(source);
181+
}
182+
183+
// Compute alias WITH clauses so search aliases (e.g. `level`) resolve in WHERE
184+
const aliasWith = searchQuery
185+
? await buildAliasWithClauses(source, metadata)
186+
: undefined;
187+
188+
const config: BuilderChartConfigWithDateRange = {
189+
displayType: DisplayType.Search,
190+
select: selectExpr,
191+
from: source.from,
192+
where: searchQuery,
193+
whereLanguage: searchQuery ? 'lucene' : 'sql',
194+
connection: source.connection,
195+
timestampValueExpression: tsExpr,
196+
implicitColumnExpression: source.implicitColumnExpression,
197+
orderBy: 'rand()',
198+
limit: { limit: sampleLimit },
199+
dateRange: [startTime, endTime],
200+
...(aliasWith ? { with: aliasWith } : {}),
201+
};
202+
203+
return renderChartConfig(config, metadata, source.querySettings);
204+
}
205+
206+
// ---- Total count query ----------------------------------------------
207+
208+
export interface TotalCountQueryOptions {
209+
source: SourceResponse;
210+
searchQuery?: string;
211+
startTime: Date;
212+
endTime: Date;
213+
}
214+
215+
/**
216+
* Build a query to get the total count of events matching the search.
217+
*/
218+
export async function buildTotalCountQuery(
219+
opts: TotalCountQueryOptions,
220+
metadata: Metadata,
221+
): Promise<ChSql> {
222+
const { source, searchQuery = '', startTime, endTime } = opts;
223+
224+
const tsExpr = source.timestampValueExpression ?? 'TimestampTime';
225+
226+
// Compute alias WITH clauses so search aliases (e.g. `level`) resolve in WHERE
227+
const aliasWith = searchQuery
228+
? await buildAliasWithClauses(source, metadata)
229+
: undefined;
230+
231+
const config: BuilderChartConfigWithDateRange = {
232+
displayType: DisplayType.Table,
233+
select: 'count() as total',
234+
from: source.from,
235+
where: searchQuery,
236+
whereLanguage: searchQuery ? 'lucene' : 'sql',
237+
connection: source.connection,
238+
timestampValueExpression: tsExpr,
239+
implicitColumnExpression: source.implicitColumnExpression,
240+
limit: { limit: 1 },
241+
dateRange: [startTime, endTime],
242+
...(aliasWith ? { with: aliasWith } : {}),
243+
};
244+
245+
return renderChartConfig(config, metadata, source.querySettings);
246+
}
247+
109248
// ---- Full row fetch (SELECT *) -------------------------------------
110249

111250
// ---- Trace waterfall queries ----------------------------------------

‎packages/cli/src/components/EventViewer/EventViewer.tsx‎

Lines changed: 89 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -15,8 +15,11 @@ import {
1515
SqlPreviewScreen,
1616
} from './SubComponents';
1717
import { TableView } from './TableView';
18+
import { PatternView } from './PatternView';
19+
import { PatternSamplesView } from './PatternSamplesView';
1820
import { DetailPanel } from './DetailPanel';
1921
import { useEventData } from './useEventData';
22+
import { usePatternData } from './usePatternData';
2023
import { useKeybindings } from './useKeybindings';
2124

2225
export default function EventViewer({
@@ -76,6 +79,12 @@ export default function EventViewer({
7679
const [traceSelectedNode, setTraceSelectedNode] = useState<SpanNode | null>(
7780
null,
7881
);
82+
const [showPatterns, setShowPatterns] = useState(false);
83+
const [patternSelectedRow, setPatternSelectedRow] = useState(0);
84+
const [patternScrollOffset, setPatternScrollOffset] = useState(0);
85+
const [expandedPattern, setExpandedPattern] = useState<number | null>(null);
86+
const [sampleSelectedRow, setSampleSelectedRow] = useState(0);
87+
const [sampleScrollOffset, setSampleScrollOffset] = useState(0);
7988
const [timeRange, setTimeRange] = useState<TimeRange>(() => {
8089
const now = new Date();
8190
return { start: new Date(now.getTime() - 60 * 60 * 1000), end: now };
@@ -111,6 +120,23 @@ export default function EventViewer({
111120
expandedRow,
112121
});
113122

123+
// ---- Pattern mining -----------------------------------------------
124+
125+
const {
126+
patterns,
127+
loading: patternsLoading,
128+
error: patternsError,
129+
totalCount: patternsTotalCount,
130+
} = usePatternData({
131+
clickhouseClient,
132+
metadata,
133+
source,
134+
submittedQuery,
135+
startTime: timeRange.start,
136+
endTime: timeRange.end,
137+
enabled: showPatterns,
138+
});
139+
114140
// ---- Derived values ----------------------------------------------
115141

116142
const columns = useMemo(
@@ -171,11 +197,22 @@ export default function EventViewer({
171197
focusDetailSearch,
172198
showHelp,
173199
showSql,
200+
showPatterns,
174201
expandedRow,
175202
detailTab,
176203
traceDetailExpanded,
177204
selectedRow,
178205
scrollOffset,
206+
patternSelectedRow,
207+
patternScrollOffset,
208+
patternCount: patterns.length,
209+
expandedPattern,
210+
sampleSelectedRow,
211+
sampleScrollOffset,
212+
sampleCount:
213+
expandedPattern !== null
214+
? (patterns[expandedPattern]?.samples.length ?? 0)
215+
: 0,
179216
isFollowing,
180217
hasMore,
181218
events,
@@ -198,7 +235,13 @@ export default function EventViewer({
198235
setFocusDetailSearch,
199236
setShowHelp,
200237
setShowSql,
238+
setShowPatterns,
201239
setSqlScrollOffset,
240+
setPatternSelectedRow,
241+
setPatternScrollOffset,
242+
setExpandedPattern,
243+
setSampleSelectedRow,
244+
setSampleScrollOffset,
202245
setSelectedRow,
203246
setScrollOffset,
204247
setExpandedRow,
@@ -313,6 +356,27 @@ export default function EventViewer({
313356
onTraceChSqlChange={setTraceChSql}
314357
onTraceSelectedNodeChange={setTraceSelectedNode}
315358
/>
359+
) : showPatterns &&
360+
expandedPattern !== null &&
361+
patterns[expandedPattern] ? (
362+
<PatternSamplesView
363+
pattern={patterns[expandedPattern]}
364+
columns={columns}
365+
selectedRow={sampleSelectedRow}
366+
scrollOffset={sampleScrollOffset}
367+
maxRows={maxRows}
368+
wrapLines={wrapLines}
369+
/>
370+
) : showPatterns ? (
371+
<PatternView
372+
patterns={patterns}
373+
selectedRow={patternSelectedRow}
374+
scrollOffset={patternScrollOffset}
375+
maxRows={maxRows}
376+
loading={patternsLoading}
377+
error={patternsError}
378+
wrapLines={wrapLines}
379+
/>
316380
) : (
317381
<TableView
318382
columns={columns}
@@ -328,13 +392,35 @@ export default function EventViewer({
328392
)}
329393

330394
<Footer
331-
rowCount={events.length}
332-
cursorPos={scrollOffset + selectedRow + 1}
395+
rowCount={
396+
showPatterns && expandedPattern !== null
397+
? (patterns[expandedPattern]?.samples.length ?? 0)
398+
: showPatterns
399+
? patterns.length
400+
: events.length
401+
}
402+
cursorPos={
403+
showPatterns && expandedPattern !== null
404+
? sampleScrollOffset + sampleSelectedRow + 1
405+
: showPatterns
406+
? patternScrollOffset + patternSelectedRow + 1
407+
: scrollOffset + selectedRow + 1
408+
}
333409
wrapLines={wrapLines}
334410
isFollowing={isFollowing}
335411
loadingMore={loadingMore}
336412
paginationError={paginationError}
337-
scrollInfo={expandedRow !== null ? `Ctrl+D/U to scroll` : undefined}
413+
scrollInfo={
414+
expandedRow !== null
415+
? 'Ctrl+D/U to scroll'
416+
: showPatterns && expandedPattern !== null
417+
? '[SAMPLES] h to go back'
418+
: showPatterns
419+
? patternsLoading
420+
? '[PATTERNS] Sampling…'
421+
: `[PATTERNS] ${patterns.length} patterns${patternsTotalCount != null ? ` from ~${patternsTotalCount.toLocaleString()} events` : ''} — l to expand, P/Esc to close`
422+
: undefined
423+
}
338424
/>
339425
</Box>
340426
</Box>

0 commit comments

Comments
 (0)