feat: structured benchmark result output via BENCH_RESULT_OUTPUT (#3195)

* feat(benchmarking): add structured result output via BENCH_RESULT_OUTPUT

emit full benchmark run metadata (config, tags, metrics, block range,
spamoor stats) as JSON when BENCH_RESULT_OUTPUT is set. consumed by
external matrix runner for table generation.

* fix: address PR review feedback for structured benchmark output

- Deduplicate overhead/reth-rate computation: move stats-based helpers
  to helpers.go, make span-based wrappers delegate to them
- Fix sub-millisecond precision loss in engine span timings by using
  microsecond-based float division instead of integer truncation
- Add spamoor stats to TestGasBurner for consistency with other tests

* refactor: make spamoor config fully configurable via BENCH_* env vars

- Add MaxPending, Rebroadcast, BaseFee, TipFee to benchConfig
- Fix ERC20 test hardcoding max_wallets=200 instead of using cfg
- Replace all hardcoded spamoor params with cfg fields across tests

* feat: extract host metadata from OTEL resource attributes in trace spans

- Add resourceAttrs struct with host, OS, and service fields
- Extract attributes from VictoriaTraces LogsQL span data via
  resourceAttrCollector interface
- Include host metadata in structured benchmark result JSON

* fix: defer emitRunResult so results are written even on test failure

Move emitRunResult into a deferred closure in all three test functions.
If the test fails after metrics are collected, the structured JSON is
still written. If it fails before result data exists, the defer is a
no-op.

* fix: state pressure benchmark CI failure and align with other tests

Remove the 3-second sleep before requireSpammersRunning that caused all
transactions to be mined before the measurement window started, leaving
SteadyState at 0s. Also add deferred emitRunResult, configurable spamoor
params, and spamoorStats collection to match the other benchmark tests.

* fix: use deployment-level service names for trace queries in external mode

In external mode the sequencer reports spans as "ev-node" (not the
test-specific name like "ev-node-erc20"), so trace queries returned
zero spans. Store service names on env: local mode uses the
test-specific name, external mode defaults to "ev-node"/"ev-reth"
with BENCH_EVNODE_SERVICE_NAME/BENCH_EVRETH_SERVICE_NAME overrides.

* perf: use limit=1 for resource attribute trace queries

fetchResourceAttrs only needs one span but was streaming the full
result set from VictoriaTraces. Add limit=1 to the LogsQL query to
avoid wasting bandwidth on long-lived instances with many spans.

* docs: add missing doc comments to run_result.go functions

Author: chatton

test/e2e/benchmark/config.go

@@ -27,6 +27,10 @@ type benchConfig struct {
 	WarmupTxs       int
 	GasUnitsToBurn  int
 	MaxWallets      int
+	MaxPending      int
+	Rebroadcast     int
+	BaseFee         int
+	TipFee          int
 	WaitTimeout     time.Duration
 }
 
@@ -43,6 +47,10 @@ func newBenchConfig(serviceName string) benchConfig {
 		WarmupTxs:       envInt("BENCH_WARMUP_TXS", 200),
 		GasUnitsToBurn:  envInt("BENCH_GAS_UNITS_TO_BURN", 1_000_000),
 		MaxWallets:      envInt("BENCH_MAX_WALLETS", 500),
+		MaxPending:      envInt("BENCH_MAX_PENDING", 50_000),
+		Rebroadcast:     envInt("BENCH_REBROADCAST", 0),
+		BaseFee:         envInt("BENCH_BASE_FEE", 20),
+		TipFee:          envInt("BENCH_TIP_FEE", 2),
 		WaitTimeout:     envDuration("BENCH_WAIT_TIMEOUT", 10*time.Minute),
 	}
 }

test/e2e/benchmark/gasburner_test.go

@@ -23,6 +23,15 @@ func (s *SpamoorSuite) TestGasBurner() {
 
 	cfg.log(t)
 
+	var result *benchmarkResult
+	var wallClock time.Duration
+	var spamoorStats *runSpamoorStats
+	defer func() {
+		if result != nil {
+			emitRunResult(t, cfg, result, wallClock, spamoorStats)
+		}
+	}()
+
 	e := s.setupEnv(cfg)
 	api := e.spamoorAPI
 
@@ -32,11 +41,11 @@ func (s *SpamoorSuite) TestGasBurner() {
 		"gas_units_to_burn": cfg.GasUnitsToBurn,
 		"total_count":       cfg.CountPerSpammer,
 		"throughput":        cfg.Throughput,
-		"max_pending":       50000,
+		"max_pending":       cfg.MaxPending,
 		"max_wallets":       cfg.MaxWallets,
-		"rebroadcast":       5,
-		"base_fee":          100,
-		"tip_fee":           50,
+		"rebroadcast":       cfg.Rebroadcast,
+		"base_fee":          cfg.BaseFee,
+		"tip_fee":           cfg.TipFee,
 		"refill_amount":     "500000000000000000000",
 		"refill_balance":    "200000000000000000000",
 		"refill_interval":   300,
@@ -83,7 +92,7 @@ func (s *SpamoorSuite) TestGasBurner() {
 	if err := waitForDrain(drainCtx, t.Logf, e.ethClient, 10); err != nil {
 		t.Logf("warning: %v", err)
 	}
-	wallClock := time.Since(loadStart)
+	wallClock = time.Since(loadStart)
 
 	endHeader, err := e.ethClient.HeaderByNumber(ctx, nil)
 	s.Require().NoError(err, "failed to get end block header")
@@ -94,10 +103,16 @@ func (s *SpamoorSuite) TestGasBurner() {
 	bm, err := collectBlockMetrics(ctx, e.ethClient, startBlock, endBlock)
 	s.Require().NoError(err, "failed to collect block metrics")
 
-	traces := s.collectTraces(e, cfg.ServiceName)
+	traces := s.collectTraces(e)
 
-	result := newBenchmarkResult("GasBurner", bm, traces)
+	result = newBenchmarkResult("GasBurner", bm, traces)
 	s.Require().Greater(result.summary.SteadyState, time.Duration(0), "expected non-zero steady-state duration")
 	result.log(t, wallClock)
 	w.addEntries(result.entries())
+
+	metrics, mErr := api.GetMetrics()
+	s.Require().NoError(mErr, "failed to get final metrics")
+	sent := sumCounter(metrics["spamoor_transactions_sent_total"])
+	failed := sumCounter(metrics["spamoor_transactions_failed_total"])
+	spamoorStats = &runSpamoorStats{Sent: sent, Failed: failed}
 }

test/e2e/benchmark/helpers.go

@@ -382,23 +382,16 @@ func (s *blockMetricsSummary) entries(prefix string) []entry {
 	}
 }
 
-// evNodeOverhead computes the fraction of block production time spent outside
-// EVM execution. It looks up the average durations of BlockExecutor.ProduceBlock
-// (the outer span covering the full block lifecycle) and Executor.ExecuteTxs
-// (the inner span covering only EVM tx execution), then returns:
+// overheadFromStats computes ev-node overhead from pre-aggregated span stats.
 //
 //	overhead% = (avgProduce - avgExecute) / avgProduce * 100
-//
-// This captures time spent on sequencing, DA submission, header construction,
-// and other ev-node orchestration work. Returns false if either span is missing.
-func evNodeOverhead(spans []e2e.TraceSpan) (float64, bool) {
-	stats := e2e.AggregateSpanStats(spans)
+func overheadFromStats(stats map[string]*e2e.SpanStats) (float64, bool) {
 	produce, ok := stats[spanProduceBlock]
-	if !ok {
+	if !ok || produce.Count == 0 {
 		return 0, false
 	}
 	execute, ok := stats[spanExecuteTxs]
-	if !ok {
+	if !ok || execute.Count == 0 {
 		return 0, false
 	}
 	produceAvg := float64(produce.Total.Microseconds()) / float64(produce.Count)
@@ -409,20 +402,25 @@ func evNodeOverhead(spans []e2e.TraceSpan) (float64, bool) {
 	return (produceAvg - executeAvg) / produceAvg * 100, true
 }
 
-// rethExecutionRate computes ev-reth's effective execution throughput in GGas/s
-// based on the total gas processed and the cumulative Engine.NewPayload duration.
-// NewPayload is the engine API call where reth validates and executes all state
-// transitions for a block (EVM execution + state root + disk commit).
-func rethExecutionRate(spans []e2e.TraceSpan, totalGasUsed uint64) (float64, bool) {
-	stats := e2e.AggregateSpanStats(spans)
+// evNodeOverhead aggregates spans then computes overhead.
+func evNodeOverhead(spans []e2e.TraceSpan) (float64, bool) {
+	return overheadFromStats(e2e.AggregateSpanStats(spans))
+}
+
+// rethRateFromStats computes ev-reth GGas/s from pre-aggregated span stats.
+func rethRateFromStats(stats map[string]*e2e.SpanStats, totalGasUsed uint64) (float64, bool) {
 	np, ok := stats[spanNewPayload]
 	if !ok || np.Total <= 0 || totalGasUsed == 0 {
 		return 0, false
 	}
-	// GGas/s = totalGas / newPayloadSeconds / 1e9
 	return float64(totalGasUsed) / np.Total.Seconds() / 1e9, true
 }
 
+// rethExecutionRate aggregates spans then computes GGas/s.
+func rethExecutionRate(spans []e2e.TraceSpan, totalGasUsed uint64) (float64, bool) {
+	return rethRateFromStats(e2e.AggregateSpanStats(spans), totalGasUsed)
+}
+
 // engineSpanEntries extracts ProduceBlock, Engine.GetPayload, and
 // Engine.NewPayload timing stats from ev-node spans and returns them as
 // result writer entries. these are the key metrics for answering "does block
@@ -443,11 +441,13 @@ func engineSpanEntries(prefix string, spans []e2e.TraceSpan) []entry {
 		if !ok || s.Count == 0 {
 			continue
 		}
-		avg := s.Total / time.Duration(s.Count)
+		avg := float64(s.Total.Microseconds()) / float64(s.Count) / 1000.0
+		min := float64(s.Min.Microseconds()) / 1000.0
+		max := float64(s.Max.Microseconds()) / 1000.0
 		entries = append(entries,
-			entry{Name: prefix + " - " + k.label + " avg", Unit: "ms", Value: float64(avg.Milliseconds())},
-			entry{Name: prefix + " - " + k.label + " min", Unit: "ms", Value: float64(s.Min.Milliseconds())},
-			entry{Name: prefix + " - " + k.label + " max", Unit: "ms", Value: float64(s.Max.Milliseconds())},
+			entry{Name: prefix + " - " + k.label + " avg", Unit: "ms", Value: avg},
+			entry{Name: prefix + " - " + k.label + " min", Unit: "ms", Value: min},
+			entry{Name: prefix + " - " + k.label + " max", Unit: "ms", Value: max},
 		)
 	}
 	return entries

test/e2e/benchmark/result.go

@@ -20,6 +20,10 @@ type traceResult struct {
 	// empty when the trace provider doesn't support rich span collection.
 	evNodeRich []richSpan
 	evRethRich []richSpan
+
+	// resource attributes extracted from trace spans (OTEL_RESOURCE_ATTRIBUTES).
+	evNodeAttrs *resourceAttrs
+	evRethAttrs *resourceAttrs
 }
 
 // displayFlowcharts renders ASCII flowcharts from rich spans. Falls back to