feat(telemetry): differentiate studio vs CLI renders, add studio frontend events

.fallowrc.jsonc

@@ -53,6 +53,14 @@
       "file": "packages/producer/src/services/fileServer.ts",
       "exports": ["isPathInside"],
     },
+    // Studio telemetry: `trackStudioRenderStart` is consumed by
+    // useRenderQueue.ts (deep relative import) but fallow's static analyzer
+    // doesn't trace it. `trackStudioSessionStart` from the same file resolves
+    // fine via App.tsx, so this is a path-resolution quirk, not dead code.
+    {
+      "file": "packages/studio/src/telemetry/events.ts",
+      "exports": ["trackStudioRenderStart"],
+    },
   ],
   "ignoreDependencies": [
     // Runtime/dynamic deps not visible to static analysis: tsup `external`,

packages/cli/src/server/studioRenderTelemetry.test.ts

@@ -0,0 +1,175 @@
+import { describe, expect, it, vi, beforeEach } from "vitest";
+import type { RenderPerfSummary } from "@hyperframes/producer";
+
+// Mock `../telemetry/events.js` so we can capture trackRenderComplete /
+// trackRenderError calls and verify the payload mapping without firing
+// network requests.
+const trackRenderComplete = vi.fn();
+const trackRenderError = vi.fn();
+vi.mock("../telemetry/events.js", () => ({
+  trackRenderComplete: (...args: unknown[]) => trackRenderComplete(...args),
+  trackRenderError: (...args: unknown[]) => trackRenderError(...args),
+}));
+
+// Imported after the mock is registered so the module picks up the mocked
+// trackRenderComplete / trackRenderError.
+const { emitStudioRenderComplete, emitStudioRenderError } =
+  await import("./studioRenderTelemetry.js");
+
+const opts = {
+  fps: { num: 30, den: 1 } as const,
+  quality: "standard",
+};
+
+const fullPerf: RenderPerfSummary = {
+  renderId: "r-1",
+  totalElapsedMs: 5000,
+  fps: 30,
+  quality: "standard",
+  workers: 4,
+  chunkedEncode: false,
+  chunkSizeFrames: null,
+  compositionDurationSeconds: 10,
+  totalFrames: 300,
+  resolution: { width: 1920, height: 1080 },
+  videoCount: 1,
+  audioCount: 0,
+  stages: {
+    compileMs: 100,
+    videoExtractMs: 200,
+    audioProcessMs: 50,
+    captureMs: 4000,
+    encodeMs: 500,
+    assembleMs: 150,
+  },
+  videoExtractBreakdown: {
+    resolveMs: 10,
+    hdrProbeMs: 20,
+    hdrPreflightMs: 30,
+    hdrPreflightCount: 1,
+    vfrProbeMs: 40,
+    vfrPreflightMs: 50,
+    vfrPreflightCount: 2,
+    extractMs: 60,
+    cacheHits: 3,
+    cacheMisses: 4,
+  },
+  tmpPeakBytes: 1024,
+  captureAvgMs: 13,
+  capturePeakMs: 25,
+};
+
+describe("studioRenderTelemetry", () => {
+  beforeEach(() => {
+    trackRenderComplete.mockClear();
+    trackRenderError.mockClear();
+  });
+
+  describe("emitStudioRenderComplete", () => {
+    it("tags the event with source: 'studio' and fps as a number", () => {
+      emitStudioRenderComplete(opts, 5000, fullPerf);
+      expect(trackRenderComplete).toHaveBeenCalledOnce();
+      const payload = trackRenderComplete.mock.calls[0]![0];
+      expect(payload.source).toBe("studio");
+      expect(payload.fps).toBe(30);
+      expect(payload.quality).toBe("standard");
+      expect(payload.docker).toBe(false);
+      expect(payload.gpu).toBe(false);
+    });
+
+    it("maps every RenderPerfSummary field to the expected payload key", () => {
+      emitStudioRenderComplete(opts, 5000, fullPerf);
+      const p = trackRenderComplete.mock.calls[0]![0];
+      expect(p.durationMs).toBe(5000);
+      expect(p.workers).toBe(4);
+      expect(p.compositionDurationMs).toBe(10_000);
+      expect(p.compositionWidth).toBe(1920);
+      expect(p.compositionHeight).toBe(1080);
+      expect(p.totalFrames).toBe(300);
+      // speedRatio = compositionDurationMs / elapsedMs = 10000 / 5000 = 2
+      expect(p.speedRatio).toBe(2);
+      expect(p.captureAvgMs).toBe(13);
+      expect(p.capturePeakMs).toBe(25);
+      expect(p.tmpPeakBytes).toBe(1024);
+      // stages
+      expect(p.stageCompileMs).toBe(100);
+      expect(p.stageVideoExtractMs).toBe(200);
+      expect(p.stageAudioProcessMs).toBe(50);
+      expect(p.stageCaptureMs).toBe(4000);
+      expect(p.stageEncodeMs).toBe(500);
+      expect(p.stageAssembleMs).toBe(150);
+      // video-extract breakdown
+      expect(p.extractResolveMs).toBe(10);
+      expect(p.extractHdrProbeMs).toBe(20);
+      expect(p.extractHdrPreflightMs).toBe(30);
+      expect(p.extractHdrPreflightCount).toBe(1);
+      expect(p.extractVfrProbeMs).toBe(40);
+      expect(p.extractVfrPreflightMs).toBe(50);
+      expect(p.extractVfrPreflightCount).toBe(2);
+      // `extractMs` on RenderPerfSummary maps to `extractPhase3Ms` on the event
+      // (named for legacy reasons — see packages/cli/src/commands/render.ts).
+      expect(p.extractPhase3Ms).toBe(60);
+      expect(p.extractCacheHits).toBe(3);
+      expect(p.extractCacheMisses).toBe(4);
+    });
+
+    it("omits all perf-derived fields when perfSummary is undefined", () => {
+      emitStudioRenderComplete(opts, 5000, undefined);
+      const p = trackRenderComplete.mock.calls[0]![0];
+      // Identity fields still present
+      expect(p.source).toBe("studio");
+      expect(p.fps).toBe(30);
+      expect(p.durationMs).toBe(5000);
+      // Perf-derived fields all undefined
+      expect(p.workers).toBeUndefined();
+      expect(p.compositionDurationMs).toBeUndefined();
+      expect(p.totalFrames).toBeUndefined();
+      expect(p.speedRatio).toBeUndefined();
+      expect(p.stageCompileMs).toBeUndefined();
+      expect(p.extractResolveMs).toBeUndefined();
+    });
+
+    it("omits videoExtractBreakdown fields when only the breakdown is absent", () => {
+      const perfNoExtract: RenderPerfSummary = { ...fullPerf, videoExtractBreakdown: undefined };
+      emitStudioRenderComplete(opts, 5000, perfNoExtract);
+      const p = trackRenderComplete.mock.calls[0]![0];
+      expect(p.workers).toBe(4);
+      expect(p.extractResolveMs).toBeUndefined();
+      expect(p.extractCacheHits).toBeUndefined();
+    });
+
+    it("leaves speedRatio undefined when elapsedMs is zero", () => {
+      emitStudioRenderComplete(opts, 0, fullPerf);
+      const p = trackRenderComplete.mock.calls[0]![0];
+      expect(p.speedRatio).toBeUndefined();
+    });
+  });
+
+  describe("emitStudioRenderError", () => {
+    it("tags with source: 'studio' and forwards failedStage + elapsedMs", () => {
+      emitStudioRenderError(opts, 1200, "encode", new Error("boom"));
+      expect(trackRenderError).toHaveBeenCalledOnce();
+      const p = trackRenderError.mock.calls[0]![0];
+      expect(p.source).toBe("studio");
+      expect(p.fps).toBe(30);
+      expect(p.quality).toBe("standard");
+      expect(p.docker).toBe(false);
+      expect(p.failedStage).toBe("encode");
+      expect(p.elapsedMs).toBe(1200);
+      expect(p.errorMessage).toBe("boom");
+    });
+
+    it("stringifies non-Error throwables", () => {
+      emitStudioRenderError(opts, 100, undefined, "string error");
+      expect(trackRenderError.mock.calls[0]![0].errorMessage).toBe("string error");
+    });
+
+    it("does not include a workers field on the error event payload", () => {
+      // Documented behavior: studio renders don't request a worker count,
+      // and the early-failure path doesn't have perfSummary to read it from.
+      emitStudioRenderError(opts, 100, undefined, new Error("x"));
+      const p = trackRenderError.mock.calls[0]![0];
+      expect(p.workers).toBeUndefined();
+    });
+  });
+});

packages/cli/src/server/studioRenderTelemetry.ts

@@ -0,0 +1,121 @@
+// ---------------------------------------------------------------------------
+// Maps studio-triggered renders into the existing `render_complete` /
+// `render_error` telemetry events with `source: "studio"`, so they land
+// alongside CLI renders in one unified taxonomy.
+//
+// Kept in its own file so `studioServer.ts` only needs two function calls.
+// ---------------------------------------------------------------------------
+
+import { freemem } from "node:os";
+import type { Fps } from "@hyperframes/core";
+import { fpsToNumber } from "@hyperframes/core";
+import type { RenderPerfSummary } from "@hyperframes/producer";
+import { trackRenderComplete, trackRenderError } from "../telemetry/events.js";
+import { bytesToMb } from "../telemetry/system.js";
+
+export interface StudioRenderOpts {
+  fps: Fps;
+  quality: string;
+}
+
+type RenderCompleteProps = Parameters<typeof trackRenderComplete>[0];
+
+function memSnapshot(): { peakMemoryMb: number; memoryFreeMb: number } {
+  return {
+    peakMemoryMb: bytesToMb(process.memoryUsage.rss()),
+    memoryFreeMb: bytesToMb(freemem()),
+  };
+}
+
+function stagesPayload(stages: Record<string, number>): Partial<RenderCompleteProps> {
+  return {
+    stageCompileMs: stages.compileMs,
+    stageVideoExtractMs: stages.videoExtractMs,
+    stageAudioProcessMs: stages.audioProcessMs,
+    stageCaptureMs: stages.captureMs,
+    stageEncodeMs: stages.encodeMs,
+    stageAssembleMs: stages.assembleMs,
+  };
+}
+
+function extractPayload(
+  extract: RenderPerfSummary["videoExtractBreakdown"],
+): Partial<RenderCompleteProps> {
+  if (!extract) return {};
+  return {
+    extractResolveMs: extract.resolveMs,
+    extractHdrProbeMs: extract.hdrProbeMs,
+    extractHdrPreflightMs: extract.hdrPreflightMs,
+    extractHdrPreflightCount: extract.hdrPreflightCount,
+    extractVfrProbeMs: extract.vfrProbeMs,
+    extractVfrPreflightMs: extract.vfrPreflightMs,
+    extractVfrPreflightCount: extract.vfrPreflightCount,
+    extractPhase3Ms: extract.extractMs,
+    extractCacheHits: extract.cacheHits,
+    extractCacheMisses: extract.cacheMisses,
+  };
+}
+
+function perfPayload(
+  perf: RenderPerfSummary | undefined,
+  elapsedMs: number,
+): Partial<RenderCompleteProps> {
+  if (!perf) return {};
+  const compositionDurationMs = Math.round(perf.compositionDurationSeconds * 1000);
+  const speedRatio =
+    compositionDurationMs > 0 && elapsedMs > 0
+      ? Math.round((compositionDurationMs / elapsedMs) * 100) / 100
+      : undefined;
+  return {
+    workers: perf.workers,
+    compositionDurationMs,
+    compositionWidth: perf.resolution.width,
+    compositionHeight: perf.resolution.height,
+    totalFrames: perf.totalFrames,
+    speedRatio,
+    captureAvgMs: perf.captureAvgMs,
+    capturePeakMs: perf.capturePeakMs,
+    tmpPeakBytes: perf.tmpPeakBytes,
+    ...stagesPayload(perf.stages),
+    ...extractPayload(perf.videoExtractBreakdown),
+  };
+}
+
+export function emitStudioRenderError(
+  opts: StudioRenderOpts,
+  elapsedMs: number,
+  failedStage: string | undefined,
+  err: unknown,
+): void {
+  // `workers` is intentionally omitted: studio renders don't accept a
+  // user-supplied worker count (the producer picks its default), so on early
+  // failures we genuinely don't know one. The CLI side has the value from
+  // `options.workers` even before `job.perfSummary` exists; studio doesn't.
+  trackRenderError({
+    fps: fpsToNumber(opts.fps),
+    quality: opts.quality,
+    docker: false,
+    source: "studio",
+    failedStage,
+    errorMessage: err instanceof Error ? err.message : String(err),
+    elapsedMs,
+    ...memSnapshot(),
+  });
+}
+
+export function emitStudioRenderComplete(
+  opts: StudioRenderOpts,
+  elapsedMs: number,
+  perf: RenderPerfSummary | undefined,
+): void {
+  trackRenderComplete({
+    durationMs: elapsedMs,
+    fps: fpsToNumber(opts.fps),
+    quality: opts.quality,
+    docker: false,
+    gpu: false,
+    source: "studio",
+    ...perfPayload(perf, elapsedMs),
+    ...memSnapshot(),
+  });
+}

packages/cli/src/server/studioServer.ts

@@ -12,6 +12,7 @@ import { resolve, join, basename } from "node:path";
 import { createProjectWatcher, type ProjectWatcher } from "./fileWatcher.js";
 import { loadRuntimeSource } from "./runtimeSource.js";
 import { VERSION as version } from "../version.js";
+import { emitStudioRenderComplete, emitStudioRenderError } from "./studioRenderTelemetry.js";
 import {
   createStudioManualEditsRenderBodyScript,
   createStudioApi,
@@ -253,6 +254,7 @@ export function createStudioServer(options: StudioServerOptions): StudioServer {
       };
 
       // Run render asynchronously, mutating the state object
+      const startTime = Date.now();
       (async () => {
         try {
           const { createRenderJob, executeRenderJob } = await import("@hyperframes/producer");
@@ -279,7 +281,6 @@ export function createStudioServer(options: StudioServerOptions): StudioServer {
             ...(manualEditsRenderScript ? { renderBodyScripts: [manualEditsRenderScript] } : {}),
             ...(opts.composition ? { entryFile: opts.composition } : {}),
           });
-          const startTime = Date.now();
           const onProgress = (j: { progress: number; currentStage?: string }) => {
             state.progress = j.progress;
             if (j.currentStage) state.stage = j.currentStage;
@@ -292,9 +293,11 @@ export function createStudioServer(options: StudioServerOptions): StudioServer {
             metaPath,
             JSON.stringify({ status: "complete", durationMs: Date.now() - startTime }),
           );
+          emitStudioRenderComplete(opts, Date.now() - startTime, job.perfSummary);
         } catch (err) {
           state.status = "failed";
           state.error = err instanceof Error ? err.message : String(err);
+          emitStudioRenderError(opts, Date.now() - startTime, state.stage, err);
           try {
             const metaPath = opts.outputPath.replace(/\.(mp4|webm|mov)$/, ".meta.json");
             writeFileSync(metaPath, JSON.stringify({ status: "failed" }));

packages/cli/src/telemetry/events.ts

@@ -11,6 +11,9 @@ export function trackRenderComplete(props: {
   workers?: number;
   docker: boolean;
   gpu: boolean;
+  // "cli" when triggered by `hyperframes render` (default), "studio" when
+  // triggered by a studio preview-server render (POST /api/projects/:id/render).
+  source?: "cli" | "studio";
   // Composition metadata
   compositionDurationMs?: number;
   compositionWidth?: number;
@@ -50,6 +53,7 @@ export function trackRenderComplete(props: {
     workers: props.workers,
     docker: props.docker,
     gpu: props.gpu,
+    source: props.source ?? "cli",
     composition_duration_ms: props.compositionDurationMs,
     composition_width: props.compositionWidth,
     composition_height: props.compositionHeight,
@@ -85,6 +89,7 @@ export function trackRenderError(props: {
   docker: boolean;
   workers?: number;
   gpu?: boolean;
+  source?: "cli" | "studio";
   failedStage?: string;
   errorMessage?: string;
   elapsedMs?: number;
@@ -97,6 +102,7 @@ export function trackRenderError(props: {
     docker: props.docker,
     workers: props.workers,
     gpu: props.gpu,
+    source: props.source ?? "cli",
     failed_stage: props.failedStage,
     error_message: props.errorMessage,
     elapsed_ms: props.elapsedMs,