Skip to content

Commit 4c24c45

Browse files
ZeynelKocaZeynelKoca
committed
Add file-based dashboard provisioner
1 parent 6936ef8 commit 4c24c45

5 files changed

Lines changed: 407 additions & 0 deletions

File tree

‎.changeset/dashboard-provisioner.md‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
---
2+
"@hyperdx/api": minor
3+
---
4+
5+
feat: add file-based dashboard provisioner that watches a directory for JSON files and upserts dashboards into MongoDB

‎packages/api/src/__tests__/dashboardProvisioner.test.ts‎

Lines changed: 192 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,192 @@
1+
import fs from 'fs';
2+
import os from 'os';
3+
import path from 'path';
4+
5+
import { createTeam } from '@/controllers/team';
6+
import { readDashboardFiles, syncDashboards } from '@/dashboardProvisioner';
7+
import { clearDBCollections, closeDB, connectDB, makeTile } from '@/fixtures';
8+
import Dashboard from '@/models/dashboard';
9+
10+
describe('dashboardProvisioner', () => {
11+
let tmpDir: string;
12+
13+
beforeAll(async () => {
14+
await connectDB();
15+
});
16+
17+
beforeEach(() => {
18+
tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'hdx-dash-test-'));
19+
});
20+
21+
afterEach(async () => {
22+
fs.rmSync(tmpDir, { recursive: true, force: true });
23+
await clearDBCollections();
24+
});
25+
26+
afterAll(async () => {
27+
await closeDB();
28+
});
29+
30+
describe('readDashboardFiles', () => {
31+
it('returns empty array for non-existent directory', () => {
32+
const result = readDashboardFiles('/non/existent/path');
33+
expect(result).toEqual([]);
34+
});
35+
36+
it('returns empty array for directory with no json files', () => {
37+
fs.writeFileSync(path.join(tmpDir, 'readme.txt'), 'not a dashboard');
38+
const result = readDashboardFiles(tmpDir);
39+
expect(result).toEqual([]);
40+
});
41+
42+
it('skips invalid JSON files', () => {
43+
fs.writeFileSync(path.join(tmpDir, 'bad.json'), '{invalid json');
44+
const result = readDashboardFiles(tmpDir);
45+
expect(result).toEqual([]);
46+
});
47+
48+
it('skips files that fail schema validation', () => {
49+
fs.writeFileSync(
50+
path.join(tmpDir, 'no-name.json'),
51+
JSON.stringify({ tiles: [] }),
52+
);
53+
const result = readDashboardFiles(tmpDir);
54+
expect(result).toEqual([]);
55+
});
56+
57+
it('parses valid dashboard files', () => {
58+
fs.writeFileSync(
59+
path.join(tmpDir, 'test.json'),
60+
JSON.stringify({ name: 'Test', tiles: [makeTile()], tags: [] }),
61+
);
62+
const result = readDashboardFiles(tmpDir);
63+
expect(result).toHaveLength(1);
64+
expect(result[0].name).toBe('Test');
65+
});
66+
});
67+
68+
describe('syncDashboards', () => {
69+
it('creates a new dashboard', async () => {
70+
const team = await createTeam({ name: 'My Team' });
71+
fs.writeFileSync(
72+
path.join(tmpDir, 'test.json'),
73+
JSON.stringify({
74+
name: 'New Dashboard',
75+
tiles: [makeTile()],
76+
tags: [],
77+
}),
78+
);
79+
80+
await syncDashboards(team._id.toString(), tmpDir);
81+
82+
const count = await Dashboard.countDocuments({ team: team._id });
83+
expect(count).toBe(1);
84+
});
85+
86+
it('updates an existing dashboard by name', async () => {
87+
const team = await createTeam({ name: 'My Team' });
88+
const tile = makeTile();
89+
await new Dashboard({
90+
name: 'Existing',
91+
tiles: [tile],
92+
tags: [],
93+
team: team._id,
94+
provisioned: true,
95+
}).save();
96+
97+
const newTile = makeTile();
98+
fs.writeFileSync(
99+
path.join(tmpDir, 'existing.json'),
100+
JSON.stringify({
101+
name: 'Existing',
102+
tiles: [newTile],
103+
tags: ['updated'],
104+
}),
105+
);
106+
107+
await syncDashboards(team._id.toString(), tmpDir);
108+
109+
const dashboard = (await Dashboard.findOne({
110+
name: 'Existing',
111+
team: team._id,
112+
})) as any;
113+
expect(dashboard.tiles[0].id).toBe(newTile.id);
114+
expect(dashboard.tags).toEqual(['updated']);
115+
});
116+
117+
it('does not create duplicates on repeated sync', async () => {
118+
const team = await createTeam({ name: 'My Team' });
119+
fs.writeFileSync(
120+
path.join(tmpDir, 'test.json'),
121+
JSON.stringify({ name: 'Dashboard', tiles: [makeTile()], tags: [] }),
122+
);
123+
124+
await syncDashboards(team._id.toString(), tmpDir);
125+
await syncDashboards(team._id.toString(), tmpDir);
126+
await syncDashboards(team._id.toString(), tmpDir);
127+
128+
const count = await Dashboard.countDocuments({ team: team._id });
129+
expect(count).toBe(1);
130+
});
131+
132+
it('provisions to multiple teams', async () => {
133+
const teamA = await createTeam({ name: 'Team A' });
134+
const teamB = await createTeam({ name: 'Team B' });
135+
fs.writeFileSync(
136+
path.join(tmpDir, 'shared.json'),
137+
JSON.stringify({
138+
name: 'Shared Dashboard',
139+
tiles: [makeTile()],
140+
tags: [],
141+
}),
142+
);
143+
144+
await syncDashboards(teamA._id.toString(), tmpDir);
145+
await syncDashboards(teamB._id.toString(), tmpDir);
146+
147+
expect(await Dashboard.countDocuments({ team: teamA._id })).toBe(1);
148+
expect(await Dashboard.countDocuments({ team: teamB._id })).toBe(1);
149+
});
150+
151+
it('does not overwrite user-created dashboards', async () => {
152+
const team = await createTeam({ name: 'My Team' });
153+
const userTile = makeTile();
154+
await new Dashboard({
155+
name: 'My Dashboard',
156+
tiles: [userTile],
157+
tags: ['user-tag'],
158+
team: team._id,
159+
// provisioned defaults to false
160+
}).save();
161+
162+
fs.writeFileSync(
163+
path.join(tmpDir, 'my-dashboard.json'),
164+
JSON.stringify({
165+
name: 'My Dashboard',
166+
tiles: [makeTile()],
167+
tags: ['provisioned-tag'],
168+
}),
169+
);
170+
171+
await syncDashboards(team._id.toString(), tmpDir);
172+
173+
// User-created dashboard is untouched
174+
const userDashboard = (await Dashboard.findOne({
175+
name: 'My Dashboard',
176+
team: team._id,
177+
provisioned: { $ne: true },
178+
})) as any;
179+
expect(userDashboard).toBeTruthy();
180+
expect(userDashboard.tiles[0].id).toBe(userTile.id);
181+
expect(userDashboard.tags).toEqual(['user-tag']);
182+
183+
// Provisioned copy was created alongside it
184+
const provisionedDashboard = await Dashboard.findOne({
185+
name: 'My Dashboard',
186+
team: team._id,
187+
provisioned: true,
188+
});
189+
expect(provisionedDashboard).toBeTruthy();
190+
});
191+
});
192+
});

‎packages/api/src/dashboardProvisioner.ts‎

Lines changed: 190 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,190 @@
1+
import fs from 'fs';
2+
import path from 'path';
3+
4+
import {
5+
DashboardWithoutId,
6+
DashboardWithoutIdSchema,
7+
} from '@hyperdx/common-utils/dist/types';
8+
9+
import Dashboard from '@/models/dashboard';
10+
import Team from '@/models/team';
11+
import logger from '@/utils/logger';
12+
13+
let intervalHandle: ReturnType<typeof setInterval> | null = null;
14+
let isSyncing = false;
15+
16+
export function readDashboardFiles(dir: string): DashboardWithoutId[] {
17+
let files: string[];
18+
try {
19+
files = fs.readdirSync(dir).filter(f => f.endsWith('.json'));
20+
} catch (err) {
21+
logger.error({ err, dir }, 'Failed to read dashboard directory');
22+
return [];
23+
}
24+
25+
const dashboards: DashboardWithoutId[] = [];
26+
for (const file of files) {
27+
try {
28+
const raw = JSON.parse(
29+
fs.readFileSync(path.join(dir, file), 'utf8'),
30+
);
31+
const parsed = DashboardWithoutIdSchema.safeParse(raw);
32+
if (!parsed.success) {
33+
logger.warn(
34+
{ file, errors: parsed.error.issues },
35+
'Skipping invalid dashboard file',
36+
);
37+
continue;
38+
}
39+
dashboards.push(parsed.data);
40+
} catch (err) {
41+
logger.error({ err, file }, 'Failed to parse dashboard file');
42+
}
43+
}
44+
return dashboards;
45+
}
46+
47+
export async function syncDashboards(teamId: string, dir: string) {
48+
const dashboards = readDashboardFiles(dir);
49+
if (dashboards.length === 0) return;
50+
51+
for (const dashboard of dashboards) {
52+
try {
53+
// Warn if a user-created dashboard with the same name exists
54+
const userDashboard = await Dashboard.exists({
55+
name: dashboard.name,
56+
team: teamId,
57+
provisioned: { $ne: true },
58+
});
59+
if (userDashboard) {
60+
logger.warn(
61+
{ name: dashboard.name },
62+
'A user-created dashboard with this name already exists, provisioned copy will coexist',
63+
);
64+
}
65+
66+
// Only match provisioned dashboards to avoid overwriting user-created ones
67+
const result = await Dashboard.findOneAndUpdate(
68+
{ name: dashboard.name, team: teamId, provisioned: true },
69+
{
70+
$set: {
71+
tiles: dashboard.tiles || [],
72+
tags: dashboard.tags || [],
73+
filters: dashboard.filters || [],
74+
savedQuery: dashboard.savedQuery || null,
75+
savedQueryLanguage: dashboard.savedQueryLanguage || null,
76+
savedFilterValues: dashboard.savedFilterValues || [],
77+
containers: dashboard.containers || [],
78+
},
79+
$setOnInsert: {
80+
name: dashboard.name,
81+
team: teamId,
82+
provisioned: true,
83+
},
84+
},
85+
{ upsert: true, new: false },
86+
);
87+
88+
if (result === null) {
89+
logger.info({ name: dashboard.name }, 'Created provisioned dashboard');
90+
}
91+
} catch (err) {
92+
logger.error(
93+
{ err, name: dashboard.name },
94+
'Failed to provision dashboard',
95+
);
96+
}
97+
}
98+
}
99+
100+
export async function startDashboardProvisioner() {
101+
stopDashboardProvisioner();
102+
103+
const dir = process.env.DASHBOARD_PROVISIONER_DIR;
104+
if (!dir) return;
105+
106+
const teamId = process.env.DASHBOARD_PROVISIONER_TEAM_ID;
107+
const provisionAllTeams =
108+
process.env.DASHBOARD_PROVISIONER_ALL_TEAMS === 'true';
109+
const intervalMs = parseInt(
110+
process.env.DASHBOARD_PROVISIONER_INTERVAL || '30000',
111+
10,
112+
);
113+
114+
if (isNaN(intervalMs) || intervalMs < 1000) {
115+
logger.error(
116+
{ value: process.env.DASHBOARD_PROVISIONER_INTERVAL },
117+
'Invalid DASHBOARD_PROVISIONER_INTERVAL, must be >= 1000ms',
118+
);
119+
return;
120+
}
121+
122+
if (!teamId && !provisionAllTeams) {
123+
logger.error(
124+
'DASHBOARD_PROVISIONER_TEAM_ID is required (or set DASHBOARD_PROVISIONER_ALL_TEAMS=true)',
125+
);
126+
return;
127+
}
128+
129+
if (teamId && !/^[0-9a-fA-F]{24}$/.test(teamId)) {
130+
logger.error(
131+
{ teamId },
132+
'DASHBOARD_PROVISIONER_TEAM_ID is not a valid ObjectId',
133+
);
134+
return;
135+
}
136+
137+
if (!fs.existsSync(dir)) {
138+
logger.warn(
139+
{ dir },
140+
'Dashboard provisioner directory does not exist, waiting...',
141+
);
142+
}
143+
144+
logger.info(
145+
{ dir, intervalMs, teamId: teamId || (provisionAllTeams ? 'all' : 'none') },
146+
'Dashboard provisioner started',
147+
);
148+
149+
const run = async () => {
150+
if (isSyncing) return;
151+
isSyncing = true;
152+
try {
153+
if (!fs.existsSync(dir)) return;
154+
155+
let teamIds: string[];
156+
if (teamId) {
157+
const teamExists = await Team.exists({ _id: teamId });
158+
if (!teamExists) {
159+
logger.warn({ teamId }, 'Configured team does not exist, skipping sync');
160+
return;
161+
}
162+
teamIds = [teamId];
163+
} else if (provisionAllTeams) {
164+
const teams = await Team.find().select('_id').lean();
165+
teamIds = teams.map(t => t._id.toString());
166+
} else {
167+
return;
168+
}
169+
170+
for (const id of teamIds) {
171+
await syncDashboards(id, dir);
172+
}
173+
} catch (err) {
174+
logger.error({ err }, 'Dashboard provisioner sync failed');
175+
} finally {
176+
isSyncing = false;
177+
}
178+
};
179+
180+
await run();
181+
intervalHandle = setInterval(run, intervalMs);
182+
}
183+
184+
export function stopDashboardProvisioner() {
185+
if (intervalHandle) {
186+
clearInterval(intervalHandle);
187+
intervalHandle = null;
188+
}
189+
isSyncing = false;
190+
}

0 commit comments

Comments
 (0)