From c6ad33ce065bbcdc14b24a9e52477d8abca2d5ad Mon Sep 17 00:00:00 2001 From: aarroyo Date: Sun, 12 Jul 2026 22:13:48 -0500 Subject: [PATCH] =?UTF-8?q?feat(observability):=20Langfuse=20=E2=86=92=20p?= =?UTF-8?q?ortable=20canonical=20evidence=20(GT-530)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bring AI-execution evidence to governance gates without rebuilding an LLM telemetry platform (§8.1/§9-5). - domain/observability-evidence.ts: provider-neutral ObservabilityEvidence (traceId, model, promptVersion, costUsd, latencyMs, totalTokens, toolCalls, evaluations, url); mapLangfuseTrace aggregates cost/tokens/latency across observations, takes model/prompt from the first GENERATION, collects distinct tool-calls, maps scores→evaluations, returns null for an id-less trace; IObservabilityEvidenceSource port for the (follow-on) connector. Verified: core-domain 945/945 (+6), tsc clean, tracking guard green. GT-530 IN-PROGRESS; remaining is the read-only Langfuse API fetch behind the port. Co-Authored-By: Claude Opus 4.8 --- .../gaps/gap-reference-catalog.es.md | 6 +- .../gaps/gap-reference-catalog.md | 6 +- .../control-center/gaps/gap-tracking.es.md | 4 +- .../core/control-center/gaps/gap-tracking.md | 4 +- .../src/domain/observability-evidence.spec.ts | 66 +++++++++++ .../src/domain/observability-evidence.ts | 112 ++++++++++++++++++ 6 files changed, 188 insertions(+), 10 deletions(-) create mode 100644 src/packages/core-domain/src/domain/observability-evidence.spec.ts create mode 100644 src/packages/core-domain/src/domain/observability-evidence.ts diff --git a/reference/core/control-center/gaps/gap-reference-catalog.es.md b/reference/core/control-center/gaps/gap-reference-catalog.es.md index 29c2c2859..86fd7fa91 100644 --- a/reference/core/control-center/gaps/gap-reference-catalog.es.md +++ b/reference/core/control-center/gaps/gap-reference-catalog.es.md @@ -364,10 +364,10 @@ Este catálogo explica cada gap: problema, propósito, evidencia, criterios de c - **Criticality:** P2 · **Complexity:** L - **Proposed fix:** `LangfuseEvidenceAdapter` tras un puerto de observabilidad que normaliza traces/evaluaciones/costo/latencia/versión-de-prompt/tool-calls al modelo de evidencia de Evolith. - **Acceptance criteria:** - - [ ] Un trace/evaluación de Langfuse se mapea a evidencia canónica consumible por un gate. - - [ ] El adaptador está aislado tras un puerto (proveedor de observabilidad reemplazable). + - [x] Un trace/evaluación de Langfuse se mapea a evidencia canónica consumible por un gate. _(`mapLangfuseTrace`→`ObservabilityEvidence` con costo/latencia/tokens/prompt/tool-calls/scores)_ + - [x] El adaptador está aislado tras un puerto (proveedor de observabilidad reemplazable). _(`IObservabilityEvidenceSource`; la forma es provider-neutral)_ - **Dependencies:** GT-511. -- **Status:** `PENDING` +- **Status:** `IN-PROGRESS` #### GT-531 diff --git a/reference/core/control-center/gaps/gap-reference-catalog.md b/reference/core/control-center/gaps/gap-reference-catalog.md index 7ae560f6d..cb917fbbf 100644 --- a/reference/core/control-center/gaps/gap-reference-catalog.md +++ b/reference/core/control-center/gaps/gap-reference-catalog.md @@ -364,10 +364,10 @@ This catalog explains each gap: problem, purpose, evidence, closure criteria, an - **Criticality:** P2 · **Complexity:** L - **Proposed fix:** `LangfuseEvidenceAdapter` behind an observability port that normalizes traces/evaluations/cost/latency/prompt-version/tool-calls to Evolith's evidence model. - **Acceptance criteria:** - - [ ] A Langfuse trace/evaluation maps to canonical evidence consumable by a gate. - - [ ] The adapter is isolated behind a port (replaceable observability provider). + - [x] A Langfuse trace/evaluation maps to canonical evidence consumable by a gate. _(`mapLangfuseTrace`→`ObservabilityEvidence` with cost/latency/tokens/prompt/tool-calls/scores)_ + - [x] The adapter is isolated behind a port (replaceable observability provider). _(`IObservabilityEvidenceSource`; the shape is provider-neutral)_ - **Dependencies:** GT-511. -- **Status:** `PENDING` +- **Status:** `IN-PROGRESS` #### GT-531 diff --git a/reference/core/control-center/gaps/gap-tracking.es.md b/reference/core/control-center/gaps/gap-tracking.es.md index ac89b358e..ac62ece65 100644 --- a/reference/core/control-center/gaps/gap-tracking.es.md +++ b/reference/core/control-center/gaps/gap-tracking.es.md @@ -34,7 +34,7 @@ Este tablero es la única fuente de verdad para deuda técnica, gaps, oportunida | [`GT-527`](./gap-reference-catalog.es.md#gt-527) | **[Wedge · eje 2] Conectores de ingesta de ownership sin lock-in: Port / Cortex / OpsLevel + Backstage.** Ingerir blueprints de IDP y `catalog-info.yaml` como fuente de ownership/servicios para enriquecer violaciones y ADRs sin depender del proveedor; Evolith bloquea donde ellos solo miden (§13.2). Fix: conectores read-only vía ACL que normalizan a la forma canónica de ownership. **EN-PROGRESO (`domain/ownership.ts`):** aterrizó el núcleo puro de normalización — `OwnershipEntry` canónico, `parseBackstageCatalog` (`catalog-info.yaml` kind Component → owner + pathPrefix vía anotación `evolith.io/path`/`source-location` relativo; descarta no-Components/incompletos), `parseBlueprintOwnership` (genérico Port/Cortex/OpsLevel: `component`\|`identifier` + `owner`\|`team`), `resolveOwner` (file→owner por longest-prefix) y `enrichViolationsWithOwner` (puebla `owner` sin sobrescribir, fuera del fingerprint). **Completa la cadena violación→owner→compliance** (compone con GT-525). Verificado: core-domain 925/925 (+8), tsc limpio. **Pendiente (conector/infra):** leer el `catalog-info.yaml` del repo (vía el port de config-parser) y el fetch read-only de las APIs Port/Cortex. | `Evolith Core` | Cross | P2 | L | `EN-PROGRESO` | | [`GT-528`](./gap-reference-catalog.es.md#gt-528) | **[Wedge · eje 2] Ingesta de DSL Structurizr / C4 → ADR ejecutable.** Convertir modelos Structurizr/C4 (intención de arquitectura, hoy prosa/diagrama) en reglas `enforce:` verificables contra el código real (§13.2). Fix: parser del DSL + mapeo a `NormalizedRule.enforce`; complementa GT-516. **EN-PROGRESO (`c4-compiler.ts`):** aterrizó el compilador — `compileC4ToBoundaryRules(model)` transforma un `C4Model` normalizado (elementos con `path`/`importPrefix` + relaciones permitidas) en `EditBoundaryRule` de GT-526, derivando el denylist desde el allowlist del modelo (un elemento solo puede depender de lo que declara; el resto queda prohibido), con `ruleId` `C4-`, ADR y severidad. **El diagrama C4 se vuelve ejecutable** y se enchufa al gate edit-time (GT-526) y al PR/CI. Verificado end-to-end: un edit de `src/domain` que importa `src/infrastructure` → bloqueado; core-domain 933/933 (+5), tsc limpio. **Pendiente (ingesta):** parsear el `.dsl` crudo de Structurizr (o su export JSON) al `C4Model` normalizado. | `Evolith Core` | Cross | P2 | M | `EN-PROGRESO` | | [`GT-529`](./gap-reference-catalog.es.md#gt-529) | **[Surround · eje 1] Contrato ACL + referencia de integración Jira Enterprise.** Mapear ideas/epics/stories/aprobaciones/releases de Jira a artefactos Evolith preservando origen/identidad/timestamps/linaje, con salvaguardas de transición (completar un workflow de Jira no autoriza una transición de fase). §8.3 / §12. Fix: ACL de sistemas de trabajo externos + guía de integración. **EN-PROGRESO (`domain/external-work-acl.ts`):** aterrizó el ACL puro — `CanonicalWorkItem` con `WorkItemProvenance` (source/externalId/externalKey/url/created/updated), `parseJiraIssue`/`parseJiraIssues` (mapea issues de Jira preservando procedencia; rechaza sin `id` en vez de fabricar identidad), `mapJiraIssueType` (Epic/Story/Task/Version→kind canónico). **Salvaguarda de transición (§9-6):** `authorizesPhaseTransition: false` por contrato + `externalWorkAuthorizesTransition`⇒`false` (completar un workflow de Jira NO autoriza un phase gate). Verificado: core-domain 939/939 (+6), tsc limpio. **Pendiente (conector/infra + doc):** fetch read-only de la Jira REST API + la guía de integración documentada. | `Evolith Core` | Cross | P1 | L | `EN-PROGRESO` | -| [`GT-530`](./gap-reference-catalog.es.md#gt-530) | **[Surround · eje 1] Adaptador Langfuse → evidencia canónica.** Mapear traces/evaluaciones/costo/latencia/versión-de-prompt/tool-calls de Langfuse al modelo de evidencia de Evolith, para no reconstruir una plataforma de telemetría LLM. §8.1 / §12. Fix: `LangfuseEvidenceAdapter` vía puerto de observabilidad. | `Evolith Core` | Cross | P2 | L | `PENDIENTE` | +| [`GT-530`](./gap-reference-catalog.es.md#gt-530) | **[Surround · eje 1] Adaptador Langfuse → evidencia canónica.** Mapear traces/evaluaciones/costo/latencia/versión-de-prompt/tool-calls de Langfuse al modelo de evidencia de Evolith, para no reconstruir una plataforma de telemetría LLM. §8.1 / §12. Fix: `LangfuseEvidenceAdapter` vía puerto de observabilidad. **EN-PROGRESO (`domain/observability-evidence.ts`):** aterrizó el mapper puro + forma portable — `ObservabilityEvidence` (traceId/model/promptVersion/costUsd/latencyMs/totalTokens/toolCalls/evaluations/url, provider-neutral §9-5), `mapLangfuseTrace` (agrega costo/tokens/latencia de las observaciones, toma model/prompt de la 1ª GENERATION, colecta tool-calls distintos, mapea scores→evaluations; trace sin id⇒null) y el puerto `IObservabilityEvidenceSource`. Verificado: core-domain 945/945 (+6), tsc limpio. **Pendiente (conector/infra):** el fetch read-only de la Langfuse API detrás del puerto. | `Evolith Core` | Cross | P2 | L | `EN-PROGRESO` | | [`GT-531`](./gap-reference-catalog.es.md#gt-531) | **[Surround · eje 1] Adaptador Cowork/Claude como ejecutor gobernado acotado.** Ejecutor de actividades con permisos/planes/aprobaciones/captura de evidencia, tratando a Claude como uno de varios ejecutores reemplazables (§8.2, §9). Extiende el épico agent-runtime GT-383…394 (HITL GT-441 / adapters GT-438). | `agent-runtime` | Cross | P2 | M | `PENDIENTE` | | [`GT-532`](./gap-reference-catalog.es.md#gt-532) | **[Surround · eje 1] Vistas ejecutivas de portafolio + adaptadores marketplace + paquetes de gobernanza por tenant.** Mejora de adopción empresarial y escala de ecosistema (§12 P2); mayormente Tracker (plano de captura de valor enterprise). | `Tracker` | Cross | P3 | XL | `PENDIENTE` | | [`GT-486`](./gap-reference-catalog.es.md#gt-486) | **el CLI no emitió un envelope ADR-0073 parseable para sdlc-status.** La salida no pudo parsearse en un envelope { success, data\|error, meta }. (Auto-detectado por el agente de pruebas exploratorias en la operación `sdlc-status`.) | `Evolith CLI` | Cross | P2 | S | `COMPLETADO` | @@ -547,7 +547,7 @@ Este tablero es la única fuente de verdad para deuda técnica, gaps, oportunida | [`GT-246`](./gap-reference-catalog.es.md#gt-246) | Implementar experimentos Chaos Mesh/Litmus | `QA` | Cross | P3 | L | `COMPLETADO` | -**Progreso:** 496 / 532 completados · 18 en progreso · 16 pendientes · 2 diferido +**Progreso:** 496 / 532 completados · 19 en progreso · 15 pendientes · 2 diferido **Oleada 2026-06-23 (auditoría profunda de Winston III):** Añadidos 14 gaps nuevos `GT-212`…`GT-225` del Winston Audit Playbook que cubren: higiene de estado ADR (GT-212), metadata + presupuestos operativos + corpus de guías por topología (GT-213, GT-217, GT-219), observabilidad + OpenAPI en controladores REST (GT-214, GT-215), paridad de input-schemas OPA + densidad de tests por topología (GT-216, GT-222), plantillas de rollback + on-call de Fase 05 (GT-218), cobertura de ramas CLI + paridad de envelope --format + limpieza de skip-list (GT-220, GT-224, GT-225), audit logging HTTP de MCP (GT-221), y tests e2e de paridad cross-surface (GT-223). diff --git a/reference/core/control-center/gaps/gap-tracking.md b/reference/core/control-center/gaps/gap-tracking.md index da0d52600..40a779721 100644 --- a/reference/core/control-center/gaps/gap-tracking.md +++ b/reference/core/control-center/gaps/gap-tracking.md @@ -34,7 +34,7 @@ This board is the single source of truth for technical debt, gaps, opportunities | [`GT-527`](./gap-reference-catalog.md#gt-527) | **[Wedge · axis 2] Lock-in-free ownership ingestion connectors: Port / Cortex / OpsLevel + Backstage.** Ingest IDP blueprints and `catalog-info.yaml` as an ownership/service source to enrich violations and ADRs without vendor lock-in; Evolith blocks where they only measure (§13.2). Fix: read-only connectors via ACL normalizing to the canonical ownership shape. **IN-PROGRESS (`domain/ownership.ts`):** landed the pure normalization core — canonical `OwnershipEntry`, `parseBackstageCatalog` (`catalog-info.yaml` kind Component → owner + pathPrefix via `evolith.io/path`/relative `source-location`; drops non-Components/incomplete), `parseBlueprintOwnership` (generic Port/Cortex/OpsLevel: `component`\|`identifier` + `owner`\|`team`), `resolveOwner` (file→owner longest-prefix) and `enrichViolationsWithOwner` (fills `owner` without overwriting, excluded from the fingerprint). **Completes the violation→owner→compliance chain** (composes with GT-525). Verified: core-domain 925/925 (+8), tsc clean. **Remaining (connector/infra):** reading the repo's `catalog-info.yaml` (via the config-parser port) and the read-only Port/Cortex API fetch. | `Evolith Core` | Cross | P2 | L | `IN-PROGRESS` | | [`GT-528`](./gap-reference-catalog.md#gt-528) | **[Wedge · axis 2] Structurizr / C4 DSL ingestion → executable ADR.** Turn Structurizr/C4 models (architecture intent, today prose/diagram) into `enforce:` rules verifiable against real code (§13.2). Fix: DSL parser + mapping to `NormalizedRule.enforce`; complements GT-516. **IN-PROGRESS (`c4-compiler.ts`):** landed the compiler — `compileC4ToBoundaryRules(model)` turns a normalized `C4Model` (elements with `path`/`importPrefix` + allowed relationships) into GT-526 `EditBoundaryRule`s, deriving the denylist from the model's allowlist (an element may only depend on what it declares; everything else is forbidden), with `ruleId` `C4-`, ADR and severity. **The C4 diagram becomes executable** and feeds the edit-time gate (GT-526) and PR/CI. Verified end-to-end: a `src/domain` edit importing `src/infrastructure` → blocked; core-domain 933/933 (+5), tsc clean. **Remaining (ingestion):** parse the raw Structurizr `.dsl` (or its JSON export) into the normalized `C4Model`. | `Evolith Core` | Cross | P2 | M | `IN-PROGRESS` | | [`GT-529`](./gap-reference-catalog.md#gt-529) | **[Surround · axis 1] ACL contract + Jira Enterprise integration reference.** Map Jira ideas/epics/stories/approvals/releases to Evolith artifacts preserving origin/identity/timestamps/lineage, with transition safeguards (completing a Jira workflow does not authorize a phase transition). §8.3 / §12. Fix: external work-system ACL + integration guide. **IN-PROGRESS (`domain/external-work-acl.ts`):** landed the pure ACL — `CanonicalWorkItem` with `WorkItemProvenance` (source/externalId/externalKey/url/created/updated), `parseJiraIssue`/`parseJiraIssues` (maps Jira issues preserving provenance; rejects a missing `id` rather than fabricating identity), `mapJiraIssueType` (Epic/Story/Task/Version→canonical kind). **Transition safeguard (§9-6):** `authorizesPhaseTransition: false` by contract + `externalWorkAuthorizesTransition`⇒`false` (completing a Jira workflow does NOT authorize a phase gate). Verified: core-domain 939/939 (+6), tsc clean. **Remaining (connector/infra + doc):** the read-only Jira REST API fetch + the documented integration guide. | `Evolith Core` | Cross | P1 | L | `IN-PROGRESS` | -| [`GT-530`](./gap-reference-catalog.md#gt-530) | **[Surround · axis 1] Langfuse adapter → canonical evidence.** Map Langfuse traces/evaluations/cost/latency/prompt-version/tool-calls to Evolith's evidence model, to avoid rebuilding an LLM telemetry platform. §8.1 / §12. Fix: `LangfuseEvidenceAdapter` via the observability port. | `Evolith Core` | Cross | P2 | L | `PENDING` | +| [`GT-530`](./gap-reference-catalog.md#gt-530) | **[Surround · axis 1] Langfuse adapter → canonical evidence.** Map Langfuse traces/evaluations/cost/latency/prompt-version/tool-calls to Evolith's evidence model, to avoid rebuilding an LLM telemetry platform. §8.1 / §12. Fix: `LangfuseEvidenceAdapter` via the observability port. **IN-PROGRESS (`domain/observability-evidence.ts`):** landed the pure mapper + portable shape — `ObservabilityEvidence` (traceId/model/promptVersion/costUsd/latencyMs/totalTokens/toolCalls/evaluations/url, provider-neutral §9-5), `mapLangfuseTrace` (aggregates cost/tokens/latency across observations, takes model/prompt from the first GENERATION, collects distinct tool-calls, maps scores→evaluations; a trace with no id⇒null) and the `IObservabilityEvidenceSource` port. Verified: core-domain 945/945 (+6), tsc clean. **Remaining (connector/infra):** the read-only Langfuse API fetch behind the port. | `Evolith Core` | Cross | P2 | L | `IN-PROGRESS` | | [`GT-531`](./gap-reference-catalog.md#gt-531) | **[Surround · axis 1] Cowork/Claude adapter as a bounded governed executor.** Activity executor with permissions/plans/approvals/evidence capture, treating Claude as one of several replaceable executors (§8.2, §9). Extends the agent-runtime epic GT-383…394 (HITL GT-441 / adapters GT-438). | `agent-runtime` | Cross | P2 | M | `PENDING` | | [`GT-532`](./gap-reference-catalog.md#gt-532) | **[Surround · axis 1] Executive portfolio views + marketplace adapters + per-tenant governance packages.** Enterprise-adoption and ecosystem-scale improvement (§12 P2); mostly Tracker (enterprise value-capture plane). | `Tracker` | Cross | P3 | XL | `PENDING` | | [`GT-486`](./gap-reference-catalog.md#gt-486) | **cli did not emit a parseable ADR-0073 envelope for sdlc-status.** Output could not be parsed into an { success, data\|error, meta } envelope. (Auto-detected by the exploratory test agent on operation `sdlc-status`.) | `Evolith CLI` | Cross | P2 | S | `DONE` | @@ -547,7 +547,7 @@ This board is the single source of truth for technical debt, gaps, opportunities | [`GT-246`](./gap-reference-catalog.md#gt-246) | Implement Chaos Mesh/Litmus experiments | `QA` | Cross | P3 | L | `DONE` | -**Progress:** 496 / 532 done · 18 in progress · 16 pending · 2 deferred +**Progress:** 496 / 532 done · 19 in progress · 15 pending · 2 deferred **Wave 2026-06-23 (Winston deep audit III):** Added 14 new gaps `GT-212`…`GT-225` from the Winston Audit Playbook covering: ADR status hygiene (GT-212), topology manifest metadata + operational budgets + guidance corpus (GT-213, GT-217, GT-219), REST controller observability + OpenAPI (GT-214, GT-215), OPA input-schema parity + per-topology test density (GT-216, GT-222), SDLC Phase 05 rollback + on-call templates (GT-218), CLI branch coverage + envelope format coverage + skip-list cleanup (GT-220, GT-224, GT-225), MCP HTTP audit logging (GT-221), and cross-surface parity e2e tests (GT-223). diff --git a/src/packages/core-domain/src/domain/observability-evidence.spec.ts b/src/packages/core-domain/src/domain/observability-evidence.spec.ts new file mode 100644 index 000000000..7eb672f8b --- /dev/null +++ b/src/packages/core-domain/src/domain/observability-evidence.spec.ts @@ -0,0 +1,66 @@ +import { mapLangfuseTrace, type LangfuseTrace } from './observability-evidence'; + +const trace = (over: Partial = {}): LangfuseTrace => ({ + id: 'trace-1', + timestamp: '2026-07-12T00:00:00.000Z', + htmlPath: 'https://cloud.langfuse.com/trace/trace-1', + observations: [ + { + type: 'GENERATION', + model: 'claude-opus-4-8', + promptName: 'summarize', + promptVersion: 3, + usage: { totalTokens: 1200 }, + calculatedTotalCost: 0.021, + latency: 800, + toolCalls: [{ name: 'search' }, { name: 'fetch' }], + }, + { type: 'SPAN', calculatedTotalCost: 0.004, latency: 120, toolCalls: [{ name: 'search' }] }, + ], + scores: [{ name: 'helpfulness', value: 0.9 }, { name: 'grounded', value: 'yes' }], + ...over, +}); + +describe('mapLangfuseTrace (GT-530 — Langfuse → portable evidence)', () => { + it('maps model, prompt version, and the trace url/timestamp', () => { + const e = mapLangfuseTrace(trace())!; + expect(e).toMatchObject({ + traceId: 'trace-1', + source: 'langfuse', + model: 'claude-opus-4-8', + promptName: 'summarize', + promptVersion: 3, + url: 'https://cloud.langfuse.com/trace/trace-1', + timestamp: '2026-07-12T00:00:00.000Z', + }); + }); + + it('aggregates cost, tokens and latency across observations', () => { + const e = mapLangfuseTrace(trace())!; + expect(e.costUsd).toBeCloseTo(0.025); // 0.021 + 0.004 + expect(e.totalTokens).toBe(1200); + expect(e.latencyMs).toBe(920); // 800 + 120 + }); + + it('prefers trace-level cost/latency when present', () => { + const e = mapLangfuseTrace(trace({ totalCost: 0.05, latency: 1000 }))!; + expect(e.costUsd).toBe(0.05); + expect(e.latencyMs).toBe(1000); + }); + + it('collects distinct tool-call names and maps scores to evaluations', () => { + const e = mapLangfuseTrace(trace())!; + expect(e.toolCalls.sort()).toEqual(['fetch', 'search']); + expect(e.evaluations).toEqual([{ name: 'helpfulness', value: 0.9 }, { name: 'grounded', value: 'yes' }]); + }); + + it('returns null for a trace with no id (no identity → not evidence)', () => { + expect(mapLangfuseTrace({ observations: [] })).toBeNull(); + }); + + it('handles a sparse trace (no observations/scores) without throwing', () => { + const e = mapLangfuseTrace({ id: 't' })!; + expect(e).toMatchObject({ traceId: 't', source: 'langfuse', toolCalls: [], evaluations: [] }); + expect(e.costUsd).toBeUndefined(); + }); +}); diff --git a/src/packages/core-domain/src/domain/observability-evidence.ts b/src/packages/core-domain/src/domain/observability-evidence.ts new file mode 100644 index 000000000..70ab98eac --- /dev/null +++ b/src/packages/core-domain/src/domain/observability-evidence.ts @@ -0,0 +1,112 @@ +/** + * Observability evidence adapter (GT-530 · axis 1 — positioning §8.1 / §9-5). + * + * Maps an LLM/agent-execution TRACE from a specialized observability platform (Langfuse first) + * to Evolith's canonical, PORTABLE evidence shape — trace id, model, prompt version, cost, + * latency, tokens, tool calls, evaluations — so a governance gate can consume AI-execution + * evidence WITHOUT Evolith rebuilding an LLM telemetry platform. §9-5 requires this evidence to + * be portable; {@link ObservabilityEvidence} is that provider-neutral shape. + * + * Layering: PURE. Fetching a trace from the Langfuse API is the connector/infra step behind + * {@link IObservabilityEvidenceSource} (follow-on); this maps an already-fetched trace. + */ + +/** One evaluation/score attached to a trace (e.g. a Langfuse score). */ +export interface EvaluationScore { + readonly name: string; + readonly value: number | string; +} + +/** Provider-neutral evidence for one AI execution, portable across observability backends. */ +export interface ObservabilityEvidence { + readonly traceId: string; + readonly source: string; // 'langfuse' + readonly model?: string; + readonly promptName?: string; + readonly promptVersion?: string | number; + readonly costUsd?: number; + readonly latencyMs?: number; + readonly totalTokens?: number; + readonly toolCalls: readonly string[]; + readonly evaluations: readonly EvaluationScore[]; + readonly url?: string; + readonly timestamp?: string; +} + +/** The port an observability connector implements to supply a trace by id (infra, follow-on). */ +export interface IObservabilityEvidenceSource { + fetchEvidence(traceId: string): Promise; +} + +// --------------------------------------------------------------------------- +// Langfuse mapper (the first observability backend) +// --------------------------------------------------------------------------- + +export interface LangfuseObservation { + readonly type?: string; // 'GENERATION' | 'SPAN' | 'EVENT' + readonly name?: string; + readonly model?: string; + readonly promptName?: string; + readonly promptVersion?: string | number; + readonly usage?: { readonly totalTokens?: number }; + readonly calculatedTotalCost?: number; + readonly latency?: number; // ms + readonly toolCalls?: readonly { readonly name?: string }[]; +} + +export interface LangfuseScore { + readonly name?: string; + readonly value?: number | string; +} + +export interface LangfuseTrace { + readonly id?: string; + readonly timestamp?: string; + readonly htmlPath?: string; // url back to the trace + readonly latency?: number; // trace-level ms, when present + readonly totalCost?: number; // trace-level cost, when present + readonly observations?: readonly LangfuseObservation[]; + readonly scores?: readonly LangfuseScore[]; +} + +function sum(values: readonly (number | undefined)[]): number | undefined { + const nums = values.filter((v): v is number => typeof v === 'number'); + return nums.length ? nums.reduce((a, b) => a + b, 0) : undefined; +} + +/** + * Map a Langfuse trace to canonical {@link ObservabilityEvidence}. Aggregates cost/tokens/latency + * across observations (trace-level values win when present), takes model/prompt from the first + * GENERATION, collects tool-call names, and maps scores to evaluations. Returns `null` for a + * trace with no id (no identity → cannot be evidence). + */ +export function mapLangfuseTrace(trace: LangfuseTrace): ObservabilityEvidence | null { + const traceId = trace.id; + if (!traceId) return null; + const observations = trace.observations ?? []; + const generation = observations.find((o) => o.type === 'GENERATION') ?? observations[0]; + + const toolCalls = [ + ...new Set( + observations.flatMap((o) => (o.toolCalls ?? []).map((t) => t.name).filter((n): n is string => !!n)), + ), + ]; + const evaluations: EvaluationScore[] = (trace.scores ?? []) + .filter((s): s is Required> => s.name != null && s.value != null) + .map((s) => ({ name: s.name, value: s.value })); + + return { + traceId, + source: 'langfuse', + model: generation?.model, + promptName: generation?.promptName, + promptVersion: generation?.promptVersion, + costUsd: trace.totalCost ?? sum(observations.map((o) => o.calculatedTotalCost)), + latencyMs: trace.latency ?? sum(observations.map((o) => o.latency)), + totalTokens: sum(observations.map((o) => o.usage?.totalTokens)), + toolCalls, + evaluations, + url: trace.htmlPath, + timestamp: trace.timestamp, + }; +}