Add file-based dashboard provisioner

Author: ZeynelKoca

.changeset/dashboard-provisioner.md

@@ -0,0 +1,5 @@
+---
+"@hyperdx/api": minor
+---
+
+feat: add file-based dashboard provisioner that watches a directory for JSON files and upserts dashboards into MongoDB

docker/hyperdx/entry.prod.sh

@@ -12,10 +12,19 @@ echo ""
 echo "Visit the HyperDX UI at $FRONTEND_URL"
 echo ""
 
-# Use concurrently to run both the API and App servers
+# Optionally include the dashboard provisioner task
+EXTRA_NAMES=""
+EXTRA_CMDS=""
+if [ -n "$DASHBOARD_PROVISIONER_DIR" ]; then
+  EXTRA_NAMES=",DASH-PROVISION"
+  EXTRA_CMDS="./packages/api/bin/hyperdx task provision-dashboards"
+fi
+
+# Use concurrently to run all services
 ./node_modules/.bin/concurrently \
   "--kill-others-on-fail" \
-  "--names=API,APP,ALERT-TASK" \
+  "--names=API,APP,ALERT-TASK${EXTRA_NAMES}" \
   "PORT=${HYPERDX_API_PORT:-8000} HYPERDX_APP_PORT=${HYPERDX_APP_PORT:-8080} ./packages/api/bin/hyperdx api" \
   "cd ./packages/app/packages/app && HOSTNAME='${HYPERDX_APP_LISTEN_HOSTNAME:-0.0.0.0}' HYPERDX_API_PORT=${HYPERDX_API_PORT:-8000} PORT=${HYPERDX_APP_PORT:-8080} node server.js" \
-  "./packages/api/bin/hyperdx task check-alerts"
+  "./packages/api/bin/hyperdx task check-alerts" \
+  ${EXTRA_CMDS:+"$EXTRA_CMDS"}

packages/api/docs/auto_provision/AUTO_PROVISION.md

@@ -163,6 +163,64 @@ services:
 For more complex configurations, you can use environment files or Docker secrets
 to manage these values.
 
+## Dashboard Provisioning
+
+HyperDX supports file-based dashboard provisioning, similar to Grafana's
+provisioning system. A scheduled task reads `.json` files from a directory
+and upserts dashboards into MongoDB, matched by name for idempotency.
+The task runs on the same schedule as other HyperDX tasks (every minute
+when using the built-in scheduler, or on your own schedule when running
+tasks externally).
+
+### Environment Variables
+
+| Variable                            | Required | Default | Description                                                     |
+| ----------------------------------- | -------- | ------- | --------------------------------------------------------------- |
+| `DASHBOARD_PROVISIONER_DIR`         | Yes      |         | Directory to watch for `.json` dashboard files                  |
+| `DASHBOARD_PROVISIONER_TEAM_ID`     | No\*     |         | Scope provisioning to a specific team ID                        |
+| `DASHBOARD_PROVISIONER_ALL_TEAMS`   | No\*     | `false` | Set to `true` to provision dashboards to all teams              |
+
+\*One of `DASHBOARD_PROVISIONER_TEAM_ID` or `DASHBOARD_PROVISIONER_ALL_TEAMS=true`
+is required when `DASHBOARD_PROVISIONER_DIR` is set.
+
+### Dashboard JSON Format
+
+Each `.json` file in the provisioner directory should contain a dashboard object
+with at minimum a `name` and `tiles` array:
+
+```json
+{
+  "name": "My Dashboard",
+  "tiles": [
+    {
+      "id": "tile-1",
+      "x": 0,
+      "y": 0,
+      "w": 6,
+      "h": 4,
+      "config": {
+        "name": "Request Count",
+        "source": "Metrics",
+        "displayType": "line",
+        "select": [{ "aggFn": "count" }]
+      }
+    }
+  ],
+  "tags": ["provisioned"]
+}
+```
+
+### Behavior
+
+- Dashboards are matched by name and team for idempotency
+- Provisioned dashboards are flagged with `provisioned: true` so they never
+  overwrite user-created dashboards with the same name
+- Removing a file from the directory does **not** delete the dashboard from
+  MongoDB (safe by default)
+- Files are validated against the `DashboardWithoutIdSchema` Zod schema; invalid
+  files are skipped with a warning
+
+
 ## Note on Security
 
 While this feature is convenient for development and initial setup, be careful

packages/api/src/models/dashboard.ts

@@ -7,6 +7,7 @@ import type { ObjectId } from '.';
 export interface IDashboard extends z.infer<typeof DashboardSchema> {
   _id: ObjectId;
   team: ObjectId;
+  provisioned?: boolean;
   createdAt: Date;
   updatedAt: Date;
 }
@@ -32,10 +33,14 @@ export default mongoose.model<IDashboard>(
       savedQueryLanguage: { type: String, required: false },
       savedFilterValues: { type: mongoose.Schema.Types.Array, required: false },
       containers: { type: mongoose.Schema.Types.Array, required: false },
+      provisioned: { type: Boolean, default: false },
     },
     {
       timestamps: true,
       toJSON: { getters: true },
     },
+  ).index(
+    { name: 1, team: 1 },
+    { unique: true, partialFilterExpression: { provisioned: true } },
   ),
 );

packages/api/src/tasks/__tests__/types.test.ts

@@ -275,6 +275,21 @@ describe('asTaskArgs', () => {
     });
   });
 
+  describe('provision-dashboards task', () => {
+    it('should accept provision-dashboards task', () => {
+      const validArgs = {
+        _: ['provision-dashboards'],
+      };
+
+      const result = asTaskArgs(validArgs);
+
+      expect(result).toEqual({
+        taskName: 'provision-dashboards',
+      });
+      expect(result.taskName).toBe('provision-dashboards');
+    });
+  });
+
   describe('ping-pong task', () => {
     it('should accept ping-pong task without provider', () => {
       const validArgs = {

packages/api/src/tasks/index.ts

@@ -11,6 +11,7 @@ import {
   timeExec,
 } from '@/tasks/metrics';
 import PingPongTask from '@/tasks/pingPongTask';
+import ProvisionDashboardsTask from '@/tasks/provisionDashboards';
 import { asTaskArgs, HdxTask, TaskArgs, TaskName } from '@/tasks/types';
 import logger from '@/utils/logger';
 
@@ -23,6 +24,8 @@ function createTask(argv: TaskArgs): HdxTask<TaskArgs> {
       return new CheckAlertTask(argv);
     case TaskName.PING_PONG:
       return new PingPongTask(argv);
+    case TaskName.PROVISION_DASHBOARDS:
+      return new ProvisionDashboardsTask(argv);
     default:
       throw new Error(`Unknown task name ${taskName}`);
   }