From 924898a67e3c908ca08be19527ef65c6f5aee480 Mon Sep 17 00:00:00 2001 From: Brian Love Date: Sat, 16 May 2026 10:36:15 -0700 Subject: [PATCH] feat: validate telemetry install and dogfood runtime events --- .../src/app/shell/demo-shell.component.ts | 57 ++---- .../src/app/shell/runtime-telemetry.spec.ts | 57 ++++++ .../src/app/shell/runtime-telemetry.ts | 34 ++++ .../scripts/smoke-install-telemetry.mjs | 192 ++++++++++++++++++ .../scripts/smoke-install-telemetry.spec.mjs | 35 ++++ nx.json | 2 +- package.json | 3 +- .../posthog/dashboards/runtime-telemetry.json | 2 +- .../runtime-instances-by-transport.json | 2 +- .../runtime-requests-by-transport.json | 2 +- .../runtime-stream-ends-by-transport.json | 2 +- .../runtime-stream-errors-by-transport.json | 2 +- .../runtime-stream-starts-by-transport.json | 2 +- 13 files changed, 343 insertions(+), 49 deletions(-) create mode 100644 examples/chat/angular/src/app/shell/runtime-telemetry.spec.ts create mode 100644 examples/chat/angular/src/app/shell/runtime-telemetry.ts create mode 100644 libs/telemetry/scripts/smoke-install-telemetry.mjs create mode 100644 libs/telemetry/scripts/smoke-install-telemetry.spec.mjs diff --git a/examples/chat/angular/src/app/shell/demo-shell.component.ts b/examples/chat/angular/src/app/shell/demo-shell.component.ts index 134ca0ca8..5e35a2ab1 100644 --- a/examples/chat/angular/src/app/shell/demo-shell.component.ts +++ b/examples/chat/angular/src/app/shell/demo-shell.component.ts @@ -36,6 +36,7 @@ import { PalettePersistence } from './palette-persistence.service'; import { ThreadsService } from './threads.service'; import { ProjectsService } from './projects.service'; import { DEMO_AGENT } from './shell-tokens'; +import { createCanonicalDemoRuntimeTelemetrySink } from './runtime-telemetry'; import { environment } from '../../environments/environment'; export type DemoMode = 'embed' | 'popup' | 'sidebar'; @@ -48,10 +49,6 @@ function modeFromUrl(url: string): DemoMode { return (MODES as readonly string[]).includes(seg) ? (seg as DemoMode) : 'embed'; } -function nowMs(): number { - return globalThis.performance?.now?.() ?? Date.now(); -} - @Component({ selector: 'demo-shell', standalone: true, @@ -346,51 +343,29 @@ export class DemoShell { // subagent dispatches and to materialize agent.subagents() from the // resulting tools:-namespaced stream events. subagentToolNames: ['research'], + telemetry: createCanonicalDemoRuntimeTelemetrySink( + this.telemetry, + () => this.model(), + ), }); void this.telemetry.capture('ngaf:browser_chat_init', { surface: TELEMETRY_SURFACE }); - void this.telemetry.captureRuntimeInstanceCreated({ - transport: 'langgraph', - surface: TELEMETRY_SURFACE, - model: this.model(), - }); const orig = a.submit.bind(a); (a as { submit: typeof a.submit }).submit = (async ( input: Parameters[0], opts?: Parameters[1], ) => { - const start = nowMs(); - const baseTelemetry = { - transport: 'langgraph', - surface: TELEMETRY_SURFACE, - model: this.model(), - }; - void this.telemetry.captureStreamStarted(baseTelemetry); - try { - const result = await orig( - { - ...(input ?? {}), - state: { - ...((input as { state?: Record })?.state ?? {}), - model: this.model(), - reasoning_effort: this.effort(), - gen_ui_mode: this.genUiMode(), - }, + return orig( + { + ...(input ?? {}), + state: { + ...((input as { state?: Record })?.state ?? {}), + model: this.model(), + reasoning_effort: this.effort(), + gen_ui_mode: this.genUiMode(), }, - opts, - ); - void this.telemetry.captureStreamEnded({ - ...baseTelemetry, - durationMs: Math.round(nowMs() - start), - }); - return result; - } catch (error) { - void this.telemetry.captureStreamErrored({ - ...baseTelemetry, - durationMs: Math.round(nowMs() - start), - error, - }); - throw error; - } + }, + opts, + ); }) as typeof a.submit; return a; })(); diff --git a/examples/chat/angular/src/app/shell/runtime-telemetry.spec.ts b/examples/chat/angular/src/app/shell/runtime-telemetry.spec.ts new file mode 100644 index 000000000..3bb560390 --- /dev/null +++ b/examples/chat/angular/src/app/shell/runtime-telemetry.spec.ts @@ -0,0 +1,57 @@ +import { describe, expect, it, vi } from 'vitest'; +import { createCanonicalDemoRuntimeTelemetrySink } from './runtime-telemetry'; + +describe('createCanonicalDemoRuntimeTelemetrySink', () => { + it('forwards runtime events through browser telemetry with canonical demo properties', async () => { + const capture = vi.fn().mockResolvedValue(undefined); + const sink = createCanonicalDemoRuntimeTelemetrySink( + { capture }, + () => 'gpt-5-mini', + ); + + await sink({ + event: 'ngaf:runtime_request_created', + properties: { + transport: 'langgraph', + surface: 'agent', + requestType: 'submit', + }, + }); + + expect(capture).toHaveBeenCalledWith('ngaf:runtime_request_created', { + transport: 'langgraph', + surface: 'canonical_demo', + requestType: 'submit', + model: 'gpt-5-mini', + }); + }); + + it('does not forward request content fields from runtime telemetry payloads', async () => { + const capture = vi.fn().mockResolvedValue(undefined); + const sink = createCanonicalDemoRuntimeTelemetrySink( + { capture }, + () => 'gpt-5-mini', + ); + + const propertiesWithUnexpectedContent = { + transport: 'langgraph', + surface: 'agent', + requestType: 'submit', + messages: [{ content: 'hello' }], + threadId: 'thread-1', + assistantId: 'chat', + apiUrl: '/api', + } as Parameters[0]['properties'] & Record; + + await sink({ + event: 'ngaf:runtime_request_created', + properties: propertiesWithUnexpectedContent, + }); + + const forwarded = capture.mock.calls[0]?.[1]; + expect(forwarded).not.toHaveProperty('messages'); + expect(forwarded).not.toHaveProperty('threadId'); + expect(forwarded).not.toHaveProperty('assistantId'); + expect(forwarded).not.toHaveProperty('apiUrl'); + }); +}); diff --git a/examples/chat/angular/src/app/shell/runtime-telemetry.ts b/examples/chat/angular/src/app/shell/runtime-telemetry.ts new file mode 100644 index 000000000..5f3ae50db --- /dev/null +++ b/examples/chat/angular/src/app/shell/runtime-telemetry.ts @@ -0,0 +1,34 @@ +// SPDX-License-Identifier: MIT +import type { + AgentRuntimeTelemetryEvent, + AgentRuntimeTelemetrySink, +} from '@ngaf/chat'; + +const CANONICAL_DEMO_SURFACE = 'canonical_demo'; +const BLOCKED_PROPERTY_KEYS = new Set([ + 'messages', + 'threadId', + 'assistantId', + 'apiUrl', +]); + +export interface BrowserTelemetryCapture { + capture(event: AgentRuntimeTelemetryEvent, properties?: Record): Promise; +} + +export function createCanonicalDemoRuntimeTelemetrySink( + telemetry: BrowserTelemetryCapture, + readModel: () => string, +): AgentRuntimeTelemetrySink { + return ({ event, properties }) => { + const safeProperties: Record = {}; + for (const [key, value] of Object.entries(properties ?? {})) { + if (!BLOCKED_PROPERTY_KEYS.has(key)) safeProperties[key] = value; + } + return telemetry.capture(event, { + ...safeProperties, + surface: CANONICAL_DEMO_SURFACE, + model: readModel(), + }); + }; +} diff --git a/libs/telemetry/scripts/smoke-install-telemetry.mjs b/libs/telemetry/scripts/smoke-install-telemetry.mjs new file mode 100644 index 000000000..665805f7b --- /dev/null +++ b/libs/telemetry/scripts/smoke-install-telemetry.mjs @@ -0,0 +1,192 @@ +#!/usr/bin/env node +import { createServer } from 'node:http'; +import { writeFileSync } from 'node:fs'; +import { mkdir, mkdtemp, readFile, rm } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { basename, join, resolve } from 'node:path'; +import { execFileSync, spawn } from 'node:child_process'; +import { pathToFileURL } from 'node:url'; + +const POSTINSTALL_EVENT = 'ngaf:postinstall'; + +export function expectedPostinstallPackages(packageRoots) { + return packageRoots + .filter(({ manifest }) => + typeof manifest?.name === 'string' + && manifest.name.startsWith('@ngaf/') + && typeof manifest.scripts?.postinstall === 'string' + ) + .map(({ manifest }) => manifest.name); +} + +export function assertObservedPostinstallEvents({ expectedPackages, events }) { + const observed = new Set( + events + .filter((event) => event?.event === POSTINSTALL_EVENT) + .map((event) => event?.properties?.pkg) + .filter((pkg) => typeof pkg === 'string'), + ); + const missing = expectedPackages.filter((pkg) => !observed.has(pkg)); + if (missing.length > 0) { + throw new Error(`Missing ngaf:postinstall events for ${missing.join(', ')}`); + } +} + +async function loadPackageRoots(roots) { + return Promise.all(roots.map(async (root) => { + const absoluteRoot = resolve(root); + const manifest = JSON.parse(await readFile(join(absoluteRoot, 'package.json'), 'utf8')); + return { root: absoluteRoot, manifest }; + })); +} + +function npmCommand() { + return process.platform === 'win32' ? 'npm.cmd' : 'npm'; +} + +function packPackage(root, tarballDir) { + const output = execFileSync(npmCommand(), [ + 'pack', + root, + '--json', + '--pack-destination', + tarballDir, + ], { encoding: 'utf8', stdio: ['ignore', 'pipe', 'pipe'] }); + const [packed] = JSON.parse(output); + if (!packed?.filename) { + throw new Error(`npm pack did not return a filename for ${root}`); + } + return join(tarballDir, basename(packed.filename)); +} + +async function startIngestServer(events) { + const server = createServer((req, res) => { + if (req.method !== 'POST') { + res.writeHead(405); + res.end(); + return; + } + const chunks = []; + req.on('data', (chunk) => chunks.push(chunk)); + req.on('end', () => { + try { + events.push(JSON.parse(Buffer.concat(chunks).toString('utf8'))); + res.writeHead(204); + } catch { + res.writeHead(400); + } + res.end(); + }); + }); + await new Promise((resolveListen, reject) => { + server.once('error', reject); + server.listen(0, '127.0.0.1', resolveListen); + }); + const address = server.address(); + if (!address || typeof address === 'string') { + throw new Error('Failed to allocate local ingest server port'); + } + return { + url: `http://127.0.0.1:${address.port}/ingest`, + close: () => new Promise((resolveClose, reject) => { + server.close((err) => err ? reject(err) : resolveClose()); + }), + }; +} + +async function installTarballs({ tarballs, tempProject, ingestUrl }) { + writeFileSync(join(tempProject, 'package.json'), JSON.stringify({ + private: true, + name: 'ngaf-install-telemetry-smoke', + version: '0.0.0', + }, null, 2) + '\n'); + + const env = { + ...process.env, + NGAF_TELEMETRY_INGEST_URL: ingestUrl, + NGAF_TELEMETRY_SAMPLE_RATE: '1', + DEBUG: process.env.DEBUG ?? '', + CI: 'false', + GITHUB_ACTIONS: 'false', + CONTINUOUS_INTEGRATION: 'false', + BUILDKITE: 'false', + CIRCLECI: 'false', + DO_NOT_TRACK: '0', + NGAF_TELEMETRY_DISABLED: '0', + }; + delete env.npm_config_do_not_track; + delete env.NPM_CONFIG_DO_NOT_TRACK; + + await new Promise((resolveInstall, reject) => { + const child = spawn(npmCommand(), [ + 'install', + '--foreground-scripts', + '--package-lock=false', + '--no-audit', + '--no-fund', + '--legacy-peer-deps', + ...tarballs, + ], { + cwd: tempProject, + env, + stdio: 'inherit', + }); + child.once('error', reject); + child.once('exit', (code, signal) => { + if (code === 0) { + resolveInstall(); + } else { + reject(new Error(`npm install failed with ${signal ?? `exit code ${code}`}`)); + } + }); + }); +} + +export async function smokeInstallTelemetry(packageRootArgs) { + if (packageRootArgs.length === 0) { + throw new Error('Usage: node libs/telemetry/scripts/smoke-install-telemetry.mjs [...]'); + } + + const packageRoots = await loadPackageRoots(packageRootArgs); + const expectedPackages = expectedPostinstallPackages(packageRoots); + if (expectedPackages.length === 0) { + throw new Error('No publishable @ngaf/* packages require postinstall telemetry in the provided roots'); + } + + const tempRoot = await mkdtemp(join(tmpdir(), 'ngaf-install-telemetry-')); + const tarballDir = join(tempRoot, 'tarballs'); + const tempProject = join(tempRoot, 'project'); + await Promise.all([ + mkdir(tarballDir, { recursive: true }), + mkdir(tempProject, { recursive: true }), + ]); + + const events = []; + const ingest = await startIngestServer(events); + try { + const tarballs = packageRoots.map(({ root }) => packPackage(root, tarballDir)); + await installTarballs({ tarballs, tempProject, ingestUrl: ingest.url }); + assertObservedPostinstallEvents({ expectedPackages, events }); + console.log(`[install-telemetry-smoke] observed ${POSTINSTALL_EVENT} for ${expectedPackages.join(', ')}`); + } finally { + await ingest.close(); + if (process.env.NGAF_TELEMETRY_KEEP_SMOKE_TMP !== '1') { + await rm(tempRoot, { recursive: true, force: true }); + } else { + console.log(`[install-telemetry-smoke] kept temp dir ${tempRoot}`); + } + } +} + +async function main() { + try { + await smokeInstallTelemetry(process.argv.slice(2)); + } catch (err) { + console.error(err instanceof Error ? err.message : err); + process.exit(1); + } +} + +if (import.meta.url === pathToFileURL(process.argv[1] ?? '').href) { + await main(); +} diff --git a/libs/telemetry/scripts/smoke-install-telemetry.spec.mjs b/libs/telemetry/scripts/smoke-install-telemetry.spec.mjs new file mode 100644 index 000000000..de8941054 --- /dev/null +++ b/libs/telemetry/scripts/smoke-install-telemetry.spec.mjs @@ -0,0 +1,35 @@ +import { describe, expect, it } from 'vitest'; +import { + assertObservedPostinstallEvents, + expectedPostinstallPackages, +} from './smoke-install-telemetry.mjs'; + +describe('smoke-install-telemetry', () => { + it('expects every publishable package with a postinstall hook to emit install telemetry', () => { + const roots = [ + { manifest: { name: '@ngaf/chat', scripts: { postinstall: 'ngaf-telemetry-postinstall || true' } }, root: 'dist/libs/chat' }, + { manifest: { name: '@ngaf/langgraph', scripts: { postinstall: 'ngaf-telemetry-postinstall || true' } }, root: 'dist/libs/langgraph' }, + { manifest: { name: '@ngaf/telemetry', scripts: { postinstall: 'node ./node/postinstall.js || true' } }, root: 'dist/libs/telemetry' }, + ]; + + expect(expectedPostinstallPackages(roots)).toEqual(['@ngaf/chat', '@ngaf/langgraph', '@ngaf/telemetry']); + }); + + it('fails when an expected package did not send a postinstall event', () => { + expect(() => assertObservedPostinstallEvents({ + expectedPackages: ['@ngaf/chat', '@ngaf/langgraph'], + events: [ + { event: 'ngaf:postinstall', properties: { pkg: '@ngaf/chat' } }, + ], + })).toThrow(/Missing ngaf:postinstall events for @ngaf\/langgraph/); + }); + + it('ignores non-postinstall events when proving package coverage', () => { + expect(() => assertObservedPostinstallEvents({ + expectedPackages: ['@ngaf/chat'], + events: [ + { event: 'ngaf:runtime_request_created', properties: { pkg: '@ngaf/chat' } }, + ], + })).toThrow(/Missing ngaf:postinstall events for @ngaf\/chat/); + }); +}); diff --git a/nx.json b/nx.json index 1d018b37a..eb610af5c 100644 --- a/nx.json +++ b/nx.json @@ -58,7 +58,7 @@ } }, "version": { - "preVersionCommand": "npx nx run-many -t build --projects=chat,langgraph,ag-ui,render,a2ui,licensing,telemetry && node libs/telemetry/scripts/apply-install-telemetry.mjs dist/libs/chat dist/libs/langgraph dist/libs/ag-ui dist/libs/render dist/libs/a2ui dist/libs/licensing && node libs/telemetry/scripts/verify-install-telemetry.mjs dist/libs/chat dist/libs/langgraph dist/libs/ag-ui dist/libs/render dist/libs/a2ui dist/libs/licensing dist/libs/telemetry", + "preVersionCommand": "npx nx run-many -t build --projects=chat,langgraph,ag-ui,render,a2ui,licensing,telemetry && node libs/telemetry/scripts/apply-install-telemetry.mjs dist/libs/chat dist/libs/langgraph dist/libs/ag-ui dist/libs/render dist/libs/a2ui dist/libs/licensing && node libs/telemetry/scripts/verify-install-telemetry.mjs dist/libs/chat dist/libs/langgraph dist/libs/ag-ui dist/libs/render dist/libs/a2ui dist/libs/licensing dist/libs/telemetry && node libs/telemetry/scripts/smoke-install-telemetry.mjs dist/libs/chat dist/libs/langgraph dist/libs/ag-ui dist/libs/render dist/libs/a2ui dist/libs/licensing dist/libs/telemetry", "updateDependents": "auto", "preserveLocalDependencyProtocols": true, "currentVersionResolver": "git-tag", diff --git a/package.json b/package.json index 1c66c4fda..65e53c295 100644 --- a/package.json +++ b/package.json @@ -12,7 +12,8 @@ "posthog:sync": "nx run posthog-tools:sync:plan", "posthog:apply": "nx run posthog-tools:sync:apply", "posthog:report": "nx run posthog-tools:report", - "posthog:generate-types": "nx run posthog-tools:generate-types" + "posthog:generate-types": "nx run posthog-tools:generate-types", + "telemetry:install-smoke": "node libs/telemetry/scripts/smoke-install-telemetry.mjs dist/libs/chat dist/libs/langgraph dist/libs/ag-ui dist/libs/render dist/libs/a2ui dist/libs/licensing dist/libs/telemetry" }, "private": true, "overrides": { diff --git a/tools/posthog/dashboards/runtime-telemetry.json b/tools/posthog/dashboards/runtime-telemetry.json index b61e33d18..fe1e45431 100644 --- a/tools/posthog/dashboards/runtime-telemetry.json +++ b/tools/posthog/dashboards/runtime-telemetry.json @@ -1,6 +1,6 @@ { "slug": "runtime-telemetry", - "posthog_id": null, + "posthog_id": 1592941, "name": "GTM ยท Runtime telemetry", "description": "Opt-in browser and opt-out Node runtime telemetry for @ngaf/* adapters.", "tags": [ diff --git a/tools/posthog/insights/runtime-instances-by-transport.json b/tools/posthog/insights/runtime-instances-by-transport.json index e4355999e..d6ce0a117 100644 --- a/tools/posthog/insights/runtime-instances-by-transport.json +++ b/tools/posthog/insights/runtime-instances-by-transport.json @@ -1,6 +1,6 @@ { "slug": "runtime-instances-by-transport", - "posthog_id": null, + "posthog_id": 8646617, "kind": "trends", "name": "Runtime instances by transport", "events": [ diff --git a/tools/posthog/insights/runtime-requests-by-transport.json b/tools/posthog/insights/runtime-requests-by-transport.json index 08cd28344..b96a90f54 100644 --- a/tools/posthog/insights/runtime-requests-by-transport.json +++ b/tools/posthog/insights/runtime-requests-by-transport.json @@ -1,6 +1,6 @@ { "slug": "runtime-requests-by-transport", - "posthog_id": null, + "posthog_id": 8646618, "kind": "trends", "name": "Runtime requests by transport", "events": [ diff --git a/tools/posthog/insights/runtime-stream-ends-by-transport.json b/tools/posthog/insights/runtime-stream-ends-by-transport.json index a14b21f67..7f5e67c20 100644 --- a/tools/posthog/insights/runtime-stream-ends-by-transport.json +++ b/tools/posthog/insights/runtime-stream-ends-by-transport.json @@ -1,6 +1,6 @@ { "slug": "runtime-stream-ends-by-transport", - "posthog_id": null, + "posthog_id": 8646619, "kind": "trends", "name": "Runtime stream completions by transport", "events": [ diff --git a/tools/posthog/insights/runtime-stream-errors-by-transport.json b/tools/posthog/insights/runtime-stream-errors-by-transport.json index d67a3869b..7f58db51d 100644 --- a/tools/posthog/insights/runtime-stream-errors-by-transport.json +++ b/tools/posthog/insights/runtime-stream-errors-by-transport.json @@ -1,6 +1,6 @@ { "slug": "runtime-stream-errors-by-transport", - "posthog_id": null, + "posthog_id": 8646620, "kind": "trends", "name": "Runtime stream errors by transport", "events": [ diff --git a/tools/posthog/insights/runtime-stream-starts-by-transport.json b/tools/posthog/insights/runtime-stream-starts-by-transport.json index 9162a3b23..5f5bcaf8b 100644 --- a/tools/posthog/insights/runtime-stream-starts-by-transport.json +++ b/tools/posthog/insights/runtime-stream-starts-by-transport.json @@ -1,6 +1,6 @@ { "slug": "runtime-stream-starts-by-transport", - "posthog_id": null, + "posthog_id": 8646621, "kind": "trends", "name": "Runtime stream starts by transport", "events": [