Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 6 additions & 0 deletions .server-changes/otlp-postgres-store-fallback.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
---
area: webapp
type: fix
---

Fixes OTLP ingest endpoints returning HTTP 500 for runs on environments that use a Postgres-backed task event store. This caused the OpenTelemetry collector to drop entire span batches as non-retryable, resulting in real span loss.
17 changes: 13 additions & 4 deletions apps/webapp/app/v3/otlpExporter.server.ts
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@ import type { ClickhouseFactory } from "~/services/clickhouse/clickhouseFactory.
import { clickhouseFactory } from "~/services/clickhouse/clickhouseFactoryInstance.server";

import { generateSpanId } from "./eventRepository/common.server";
import { eventRepository } from "./eventRepository/eventRepository.server";
import type {
CreatableEventKind,
CreatableEventStatus,
Expand Down Expand Up @@ -120,10 +121,18 @@ class OTLPExporter {
const routeKey = `${event.organizationId}\0${taskEventStore}`;
let resolved = routeCache.get(routeKey);
if (!resolved) {
resolved = this._clickhouseFactory.getEventRepositoryForOrganizationSync(
taskEventStore,
event.organizationId
);
// Non-ClickHouse stores (taskEvent / taskEventPartitioned) are Postgres-backed.
// The ClickHouse factory only handles clickhouse/clickhouse_v2 and throws otherwise.
if (taskEventStore !== "clickhouse" && taskEventStore !== "clickhouse_v2") {
// Non-ClickHouse stores (taskEvent / taskEventPartitioned) are Postgres-backed.
// The ClickHouse factory only handles clickhouse/clickhouse_v2 and throws otherwise.
resolved = { key: "postgres:default", repository: eventRepository };
} else {
resolved = this._clickhouseFactory.getEventRepositoryForOrganizationSync(
taskEventStore,
event.organizationId
);
}
routeCache.set(routeKey, resolved);
}

Expand Down