feat: OTel infrastructure — MeterProvider, exporters, attribute constants (#97)

* feat: OTel infrastructure — MeterProvider, exporters, attribute constants

Phase 1 of the OTel telemetry migration. Sets up the foundation with
no metric changes — validates the bridge works alongside controller-runtime.

MeterProvider setup (cmd/telemetry.go):
- Prometheus exporter registers into controller-runtime's metrics.Registry
  so OTel instruments will appear on the existing /metrics endpoint
- OTLP exporter enabled when OTEL_EXPORTER_OTLP_ENDPOINT is set
- Resource attributes: service.name, service.version, k8s.pod.name,
  k8s.namespace.name (via downward API env vars)
- Proper shutdown handling via defer

Attribute constants (observability/attributes.go):
- 9 per-datapoint attribute keys matching existing Prometheus label names
- No overloaded "type" — split into replica_state and condition_type

Build changes:
- Makefile and Dockerfile: go build ./cmd/ (package, not single file)
- OTel SDK dependencies added to go.mod

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>

* fix: use signal-cancellable context for MeterProvider init

Create the signal handler context once via ctrl.SetupSignalHandler()
and share it between the MeterProvider init and mgr.Start. The OTLP
exporter's gRPC connection setup respects cancellation on SIGTERM.

The shutdown defer intentionally uses context.Background() so the
OTLP exporter can flush its final batch after the signal fires.

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>

* fix: add timeout to MeterProvider shutdown

Use a 5-second timeout on the shutdown context so the OTLP exporter
doesn't hang indefinitely if the collector is unreachable. The fresh
context.Background() is still correct (signal ctx is already cancelled
when defer runs), but now it's bounded.

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>

---------

Co-authored-by: Claude Opus 4.6 (1M context) <noreply@anthropic.com>

Author: bdchatham

Dockerfile

@@ -9,7 +9,7 @@ RUN go mod download
 
 COPY . .
 
-RUN CGO_ENABLED=0 GOOS=${TARGETOS:-linux} GOARCH=${TARGETARCH} go build -a -o manager cmd/main.go
+RUN CGO_ENABLED=0 GOOS=${TARGETOS:-linux} GOARCH=${TARGETARCH} go build -a -o manager ./cmd/
 
 FROM gcr.io/distroless/static:nonroot
 WORKDIR /

Makefile

@@ -4,7 +4,7 @@ GOLANGCI_LINT ?= $(shell which golangci-lint 2>/dev/null || echo $(HOME)/go/bin/
 .PHONY: build test lint manifests generate ci docker-build docker-push
 
 build: ## Build manager binary.
-	go build -o bin/manager cmd/main.go
+	go build -o bin/manager ./cmd/
 
 test: ## Run tests.
 	go test $$(go list ./... | grep -v /e2e) -coverprofile cover.out

cmd/main.go

@@ -7,6 +7,7 @@ import (
 	"fmt"
 	"os"
 	"sort"
+	"time"
 
 	_ "k8s.io/client-go/plugin/pkg/client/auth"
 
@@ -99,6 +100,21 @@ func main() {
 		metricsServerOptions.KeyName = metricsCertKey
 	}
 
+	ctx := ctrl.SetupSignalHandler()
+
+	mp, err := initMeterProvider(ctx)
+	if err != nil {
+		setupLog.Error(err, "Failed to initialize OTel MeterProvider")
+		os.Exit(1)
+	}
+	defer func() {
+		shutdownCtx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
+		defer cancel()
+		if err := mp.Shutdown(shutdownCtx); err != nil {
+			setupLog.Error(err, "Failed to shutdown MeterProvider")
+		}
+	}()
+
 	mgr, err := ctrl.NewManager(ctrl.GetConfigOrDie(), ctrl.Options{
 		Scheme:                        scheme,
 		Metrics:                       metricsServerOptions,
@@ -230,7 +246,7 @@ func main() {
 	}
 
 	setupLog.Info("Starting manager")
-	if err := mgr.Start(ctrl.SetupSignalHandler()); err != nil {
+	if err := mgr.Start(ctx); err != nil {
 		setupLog.Error(err, "Failed to run manager")
 		os.Exit(1)
 	}

cmd/telemetry.go

@@ -0,0 +1,74 @@
+package main
+
+import (
+	"context"
+	"fmt"
+	"os"
+
+	"go.opentelemetry.io/otel"
+	"go.opentelemetry.io/otel/attribute"
+	"go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc"
+	promexporter "go.opentelemetry.io/otel/exporters/prometheus"
+	sdkmetric "go.opentelemetry.io/otel/sdk/metric"
+	"go.opentelemetry.io/otel/sdk/resource"
+	semconv "go.opentelemetry.io/otel/semconv/v1.26.0"
+	crmetrics "sigs.k8s.io/controller-runtime/pkg/metrics"
+)
+
+// buildVersion is set via ldflags at build time.
+var buildVersion = "dev"
+
+// initMeterProvider creates an OTel MeterProvider with two readers:
+//   - A Prometheus exporter that registers into controller-runtime's metrics.Registry,
+//     making OTel instruments appear on the existing /metrics endpoint alongside
+//     controller-runtime's own workqueue/reconcile metrics.
+//   - An OTLP exporter (when OTEL_EXPORTER_OTLP_ENDPOINT is set) that pushes to
+//     whatever backend the user configures.
+func initMeterProvider(ctx context.Context) (*sdkmetric.MeterProvider, error) {
+	res, err := resource.Merge(
+		resource.Default(),
+		resource.NewWithAttributes(
+			semconv.SchemaURL,
+			semconv.ServiceName("sei-k8s-controller"),
+			semconv.ServiceVersion(buildVersion),
+			attribute.String("k8s.pod.name", os.Getenv("POD_NAME")),
+			attribute.String("k8s.namespace.name", os.Getenv("POD_NAMESPACE")),
+		),
+	)
+	if err != nil {
+		return nil, fmt.Errorf("building OTel resource: %w", err)
+	}
+
+	var opts []sdkmetric.Option
+	opts = append(opts, sdkmetric.WithResource(res))
+
+	// Prometheus reader: bridges OTel instruments into controller-runtime's
+	// existing /metrics endpoint. WithoutScopeInfo and WithoutTargetInfo
+	// suppress otel_scope_info and target_info metrics that would add noise
+	// alongside controller-runtime's own Prometheus output.
+	promReader, err := promexporter.New(
+		promexporter.WithRegisterer(crmetrics.Registry),
+		promexporter.WithoutScopeInfo(),
+		promexporter.WithoutTargetInfo(),
+	)
+	if err != nil {
+		return nil, fmt.Errorf("creating Prometheus exporter: %w", err)
+	}
+	opts = append(opts, sdkmetric.WithReader(promReader))
+
+	// OTLP reader: optional, enabled by standard OTel env var.
+	if os.Getenv("OTEL_EXPORTER_OTLP_ENDPOINT") != "" {
+		otlpExporter, err := otlpmetricgrpc.New(ctx)
+		if err != nil {
+			return nil, fmt.Errorf("creating OTLP exporter: %w", err)
+		}
+		opts = append(opts, sdkmetric.WithReader(
+			sdkmetric.NewPeriodicReader(otlpExporter),
+		))
+	}
+
+	mp := sdkmetric.NewMeterProvider(opts...)
+	otel.SetMeterProvider(mp)
+
+	return mp, nil
+}

go.mod

@@ -11,6 +11,11 @@ require (
 	github.com/prometheus/client_golang v1.23.2
 	github.com/sei-protocol/sei-config v0.0.11
 	github.com/sei-protocol/seictl v0.0.30
+	go.opentelemetry.io/otel v1.43.0
+	go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v1.43.0
+	go.opentelemetry.io/otel/exporters/prometheus v0.65.0
+	go.opentelemetry.io/otel/sdk v1.43.0
+	go.opentelemetry.io/otel/sdk/metric v1.43.0
 	k8s.io/api v0.35.0
 	k8s.io/apiextensions-apiserver v0.35.0
 	k8s.io/apimachinery v0.35.0
@@ -20,7 +25,7 @@ require (
 )
 
 require (
-	cel.dev/expr v0.24.0 // indirect
+	cel.dev/expr v0.25.1 // indirect
 	github.com/BurntSushi/toml v1.5.0 // indirect
 	github.com/antlr4-go/antlr/v4 v4.13.0 // indirect
 	github.com/apapsch/go-jsonmerge/v2 v2.0.0 // indirect
@@ -42,7 +47,7 @@ require (
 	github.com/aws/smithy-go v1.24.2 // indirect
 	github.com/beorn7/perks v1.0.1 // indirect
 	github.com/blang/semver/v4 v4.0.0 // indirect
-	github.com/cenkalti/backoff/v5 v5.0.2 // indirect
+	github.com/cenkalti/backoff/v5 v5.0.3 // indirect
 	github.com/cespare/xxhash/v2 v2.3.0 // indirect
 	github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect
 	github.com/dustin/go-humanize v1.0.1 // indirect
@@ -61,7 +66,7 @@ require (
 	github.com/google/cel-go v0.26.0 // indirect
 	github.com/google/gnostic-models v0.7.0 // indirect
 	github.com/google/go-cmp v0.7.0 // indirect
-	github.com/grpc-ecosystem/grpc-gateway/v2 v2.27.1 // indirect
+	github.com/grpc-ecosystem/grpc-gateway/v2 v2.28.0 // indirect
 	github.com/inconshreveable/mousetrap v1.1.0 // indirect
 	github.com/josharian/intern v1.0.0 // indirect
 	github.com/json-iterator/go v1.1.12 // indirect
@@ -74,8 +79,9 @@ require (
 	github.com/oapi-codegen/runtime v1.2.0 // indirect
 	github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect
 	github.com/prometheus/client_model v0.6.2 // indirect
-	github.com/prometheus/common v0.66.1 // indirect
-	github.com/prometheus/procfs v0.17.0 // indirect
+	github.com/prometheus/common v0.67.5 // indirect
+	github.com/prometheus/otlptranslator v1.0.0 // indirect
+	github.com/prometheus/procfs v0.20.1 // indirect
 	github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect
 	github.com/sei-protocol/seilog v0.0.3 // indirect
 	github.com/spf13/cobra v1.10.2 // indirect
@@ -84,31 +90,29 @@ require (
 	github.com/x448/float16 v0.8.4 // indirect
 	go.opentelemetry.io/auto/sdk v1.2.1 // indirect
 	go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.62.0 // indirect
-	go.opentelemetry.io/otel v1.39.0 // indirect
 	go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.37.0 // indirect
 	go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.37.0 // indirect
-	go.opentelemetry.io/otel/metric v1.39.0 // indirect
-	go.opentelemetry.io/otel/sdk v1.39.0 // indirect
-	go.opentelemetry.io/otel/trace v1.39.0 // indirect
-	go.opentelemetry.io/proto/otlp v1.7.0 // indirect
+	go.opentelemetry.io/otel/metric v1.43.0 // indirect
+	go.opentelemetry.io/otel/trace v1.43.0 // indirect
+	go.opentelemetry.io/proto/otlp v1.10.0 // indirect
 	go.uber.org/multierr v1.11.0 // indirect
 	go.uber.org/zap v1.27.0 // indirect
-	go.yaml.in/yaml/v2 v2.4.3 // indirect
+	go.yaml.in/yaml/v2 v2.4.4 // indirect
 	go.yaml.in/yaml/v3 v3.0.4 // indirect
 	golang.org/x/exp v0.0.0-20260112195511-716be5621a96 // indirect
-	golang.org/x/mod v0.32.0 // indirect
-	golang.org/x/net v0.49.0 // indirect
+	golang.org/x/mod v0.33.0 // indirect
+	golang.org/x/net v0.52.0 // indirect
 	golang.org/x/oauth2 v0.35.0 // indirect
-	golang.org/x/sync v0.19.0 // indirect
-	golang.org/x/sys v0.40.0 // indirect
-	golang.org/x/term v0.39.0 // indirect
-	golang.org/x/text v0.34.0 // indirect
+	golang.org/x/sync v0.20.0 // indirect
+	golang.org/x/sys v0.42.0 // indirect
+	golang.org/x/term v0.41.0 // indirect
+	golang.org/x/text v0.35.0 // indirect
 	golang.org/x/time v0.14.0 // indirect
-	golang.org/x/tools v0.41.0 // indirect
+	golang.org/x/tools v0.42.0 // indirect
 	gomodules.xyz/jsonpatch/v2 v2.4.0 // indirect
-	google.golang.org/genproto/googleapis/api v0.0.0-20260226221140-a57be14db171 // indirect
-	google.golang.org/genproto/googleapis/rpc v0.0.0-20260226221140-a57be14db171 // indirect
-	google.golang.org/grpc v1.78.0 // indirect
+	google.golang.org/genproto/googleapis/api v0.0.0-20260401024825-9d38bb4040a9 // indirect
+	google.golang.org/genproto/googleapis/rpc v0.0.0-20260401024825-9d38bb4040a9 // indirect
+	google.golang.org/grpc v1.80.0 // indirect
 	google.golang.org/protobuf v1.36.11 // indirect
 	gopkg.in/evanphx/json-patch.v4 v4.13.0 // indirect
 	gopkg.in/inf.v0 v0.9.1 // indirect