From 181f1182e0befdd50a126b8d037ad3edea3f6f2d Mon Sep 17 00:00:00 2001 From: Sam Calder-Mason Date: Mon, 6 Jul 2026 10:55:31 +1000 Subject: [PATCH 1/2] telemetry: add OTLP traces pipeline (Vector otlp_in -> OTLP sink) Forward-ports the traces pipeline from ethpandaops/blob-devnets (devnet-0) to devnet-6: Vector gains an otlp_in source (gRPC :4317, HTTP :4318) with use_otlp_decoding, plus an otlp_traces sink forwarding span batches to the prod analytics OTLP gateway. geth exports RPC spans via --rpc.telemetry, lighthouse via --telemetry-collector-url. Vector joins the shared docker network so clients can reach it by name. --- .../devnet-6/group_vars/all/all.yaml | 35 +++++++++++++++++-- .../devnet-6/group_vars/bootnode.yaml | 4 +++ .../inventories/devnet-6/group_vars/geth.yaml | 5 +++ .../devnet-6/group_vars/lighthouse.yaml | 2 ++ 4 files changed, 44 insertions(+), 2 deletions(-) diff --git a/ansible/inventories/devnet-6/group_vars/all/all.yaml b/ansible/inventories/devnet-6/group_vars/all/all.yaml index 1b9810a..df7569c 100644 --- a/ansible/inventories/devnet-6/group_vars/all/all.yaml +++ b/ansible/inventories/devnet-6/group_vars/all/all.yaml @@ -345,15 +345,18 @@ docker_nginx_proxy_wildcard_cert: "{{ network_server_subdomain }}" docker_nginx_proxy_wildcard_cert_url: http://cert.{{ network_server_subdomain }}/{{ network_server_subdomain }}-latest.tar.enc docker_nginx_proxy_wildcard_cert_psk: "{{ secret_cert_encryption_psk }}" -# OTLP egress to the prod analytics gateway (Vector ships container logs here). +# OTLP egress to the prod analytics gateway (shared by Vector's logs + traces sinks). # Ingress identity (auth username + ingress_user tag) is the current devnet name # rather than secret_loki.username: the sops username drifts stale between devnet # iterations, while the gateway only validates the (constant) password — so using -# the network name keeps log attribution correct no matter what sops holds. +# the network name keeps log/trace attribution correct no matter what sops holds. otlp_endpoint: https://otlp.analytics.production.platform.ethpandaops.io otlp_deployment_env: production # role: ethpandaops.general.vector +# Shared docker network so the clients can reach Vector's OTLP listeners and +# Vector can read the other containers' logs. +vector_container_networks: "{{ docker_networks_shared }}" vector_config: | # Docker container logs (clean per-container metadata straight from the Docker API) [sources.in] @@ -367,6 +370,19 @@ vector_config: | "snooper-", ] + # OTLP traces pushed by the clients (beacon -> :4317 grpc, geth -> :4318 http). + # use_otlp_decoding preserves the resourceSpans envelope so the sink forwards + # natively with the client's batching intact (1 request in = 1 request out). + [sources.otlp_in] + type = "opentelemetry" + use_otlp_decoding = true + + [sources.otlp_in.grpc] + address = "[::]:4317" + + [sources.otlp_in.http] + address = "[::]:4318" + # Shape docker_logs events into an OTLP resourceLogs envelope with full metadata. [transforms.otel_shape] type = "remap" @@ -492,3 +508,18 @@ vector_config: | # max_events MUST be 1 — OTLP/HTTP allows one envelope per request. batch.max_events = 1 batch.timeout_secs = 5 + + [sinks.otlp_traces] + type = "opentelemetry" + inputs = ["otlp_in.traces"] + [sinks.otlp_traces.protocol] + type = "http" + uri = "{{ otlp_endpoint }}/v1/traces" + method = "post" + encoding.codec = "otlp" + auth.strategy = "basic" + auth.user = "{{ ethereum_network_name }}" + auth.password = "{{ secret_loki.password }}" + # use_otlp_decoding => one event already carries the client's full span batch. + batch.max_events = 1 + batch.timeout_secs = 5 diff --git a/ansible/inventories/devnet-6/group_vars/bootnode.yaml b/ansible/inventories/devnet-6/group_vars/bootnode.yaml index f43c8b7..fb599e5 100644 --- a/ansible/inventories/devnet-6/group_vars/bootnode.yaml +++ b/ansible/inventories/devnet-6/group_vars/bootnode.yaml @@ -117,6 +117,10 @@ geth_container_command_extra_args: - --http.vhosts=* - --networkid={{ ethereum_network_id }} - --syncmode=full + # OTLP traces → local otelcol sidecar (handles upstream auth + endpoint). + - --rpc.telemetry + - --rpc.telemetry.endpoint=http://vector:4318/v1/traces + - --rpc.telemetry.instance-id={{ ethereum_network_name }}-{{ inventory_hostname }} geth_container_pull: true # role: ethpandaops.general.prometheus prometheus_remote_push_url: https://victoriametrics.ethdevops.io/insert/0/prometheus/api/v1/write diff --git a/ansible/inventories/devnet-6/group_vars/geth.yaml b/ansible/inventories/devnet-6/group_vars/geth.yaml index 4f76420..3c8d05e 100644 --- a/ansible/inventories/devnet-6/group_vars/geth.yaml +++ b/ansible/inventories/devnet-6/group_vars/geth.yaml @@ -26,6 +26,11 @@ geth_container_command_extra_args: - --networkid={{ ethereum_network_id }} - --syncmode=full - --bootnodes={{ ethereum_el_bootnodes | join(',') }} + - --rpc.telemetry + - --rpc.telemetry.endpoint=http://vector:4318/v1/traces + - --rpc.telemetry.sample-ratio=1 # Required until geth ships the fix for the SampleRatio default https://github.com/ethereum/go-ethereum/pull/34948 + - --rpc.telemetry.instance-id={{ ethereum_network_name }}-{{ inventory_hostname }} + - --rpc.telemetry.tags=execution_client={{ ethereum_node_el }},consensus_client={{ ethereum_node_cl }},supernode={{ ethereum_node_cl_supernode_enabled | bool | default(false) }},network={{ ethereum_network_name }},instance-id={{ ethereum_network_name }}-{{ inventory_hostname }} geth_container_pull: true prometheus_config: | diff --git a/ansible/inventories/devnet-6/group_vars/lighthouse.yaml b/ansible/inventories/devnet-6/group_vars/lighthouse.yaml index 16557a9..f0745ff 100644 --- a/ansible/inventories/devnet-6/group_vars/lighthouse.yaml +++ b/ansible/inventories/devnet-6/group_vars/lighthouse.yaml @@ -41,6 +41,8 @@ lighthouse_container_command_extra_simple_args: - --suggested-fee-recipient={{ lighthouse_validator_fee_recipient }} - --ignore-ws-check - --enable-mplex + - --telemetry-collector-url=http://vector:4317 + - --telemetry-service-name={{ ethereum_network_name }}-{{ inventory_hostname }} lighthouse_validator_container_volumes: - "{{ lighthouse_validator_datadir }}:/validator-data" - "{{ eth_testnet_config_dir }}:/network-config:ro" From 1b6a8dce3bba58a59b170bd0ce6b241d51d0b6b1 Mon Sep 17 00:00:00 2001 From: Sam Calder-Mason Date: Mon, 6 Jul 2026 12:58:42 +1000 Subject: [PATCH 2/2] telemetry: wire reth/prysm/grandine OTLP traces --- ansible/inventories/devnet-6/group_vars/grandine.yaml | 2 ++ ansible/inventories/devnet-6/group_vars/prysm.yaml | 4 ++++ ansible/inventories/devnet-6/group_vars/reth.yaml | 5 +++++ 3 files changed, 11 insertions(+) diff --git a/ansible/inventories/devnet-6/group_vars/grandine.yaml b/ansible/inventories/devnet-6/group_vars/grandine.yaml index e7b19cd..cec5049 100644 --- a/ansible/inventories/devnet-6/group_vars/grandine.yaml +++ b/ansible/inventories/devnet-6/group_vars/grandine.yaml @@ -45,6 +45,8 @@ grandine_container_command_extra_args: - --graffiti={{ ansible_hostname }} - --features=LogHttpRequests - --default-gas-limit=190000000 + - --telemetry-metrics-url=http://vector:4317 # despite the name this exports tracing spans (grandine #487); gRPC only + - --telemetry-service-name={{ ethereum_network_name }}-{{ inventory_hostname }} grandine_validator_container_volumes: - "{{ grandine_validator_datadir }}:/validator-data" diff --git a/ansible/inventories/devnet-6/group_vars/prysm.yaml b/ansible/inventories/devnet-6/group_vars/prysm.yaml index 8009dd8..8e9d496 100644 --- a/ansible/inventories/devnet-6/group_vars/prysm.yaml +++ b/ansible/inventories/devnet-6/group_vars/prysm.yaml @@ -51,6 +51,10 @@ prysm_container_command_extra_simple_args: - --verbosity=debug - --subscribe-all-subnets - --p2p-colocation-whitelist=0.0.0.0/0,::/0 + - --enable-tracing + - --tracing-endpoint=http://vector:4318/v1/traces # the flag default is a stale Jaeger URL; always set explicitly + - --tracing-process-name={{ ethereum_network_name }}-{{ inventory_hostname }} + - --trace-sample-fraction=1.0 # default samples only 20% prysm_container_command_extra_bootnode_args: >- {{ ethereum_cl_bootnodes | map('regex_replace', '^', '--bootstrap-node=') | list }} diff --git a/ansible/inventories/devnet-6/group_vars/reth.yaml b/ansible/inventories/devnet-6/group_vars/reth.yaml index 5910d11..68619ae 100644 --- a/ansible/inventories/devnet-6/group_vars/reth.yaml +++ b/ansible/inventories/devnet-6/group_vars/reth.yaml @@ -15,6 +15,8 @@ reth_container_env: VIRTUAL_PORT: "{{ ethereum_node_el_ports_http_rpc | string }}" LETSENCRYPT_HOST: "{{ ethereum_node_rcp_hostname }}" RUST_BACKTRACE: full + # reth hardcodes service.name=reth; the instance id has to come in via resource attributes + OTEL_RESOURCE_ATTRIBUTES: "service.instance.id={{ ethereum_network_name }}-{{ inventory_hostname }}" reth_container_volumes: - "{{ reth_datadir }}:/data" - "{{ reth_auth_jwt_path }}:/execution-auth.jwt:ro" @@ -24,6 +26,9 @@ reth_container_command_extra_args: - --bootnodes={{ ethereum_el_bootnodes | join(',') }} - --http.api=trace,rpc,eth,net,debug,web3,admin,txpool - --builder.gaslimit=150000000 + - --tracing-otlp=http://vector:4318/v1/traces + - --tracing-otlp.filter=info # the default 'debug' produced 50GB of spans during a sync (reth #21608) + - --tracing-otlp.sample-ratio=1.0 prometheus_config: | global: scrape_interval: 30s