Add benchmarks and optimize diagnose overhead

.gitignore

@@ -9,10 +9,7 @@ diagnose-*/
 
 .env
 .venv/
-
-# Local Phoenix DB for evals/eval.py
-evals/.phoenix/
-__pycache__/
+*.test
 
 # Agents
 .antigravitycli/

AGENTS.md

@@ -3,6 +3,7 @@
 - An easy to import and use test rig for Go projects
 - Can be used as a package or CLI
 - Used to find and fix flaky tests
+- Minimal CPU, RAM, and timing overhead added to test execution
 
 ## Validate changes
 
@@ -18,7 +19,6 @@ go test ./...                 # Test
 - `internal/runner/` — core test execution. `Diagnose` is the main entry point; `diagnoseRunHooks` carries iteration hooks as `func(context.Context) error` fields.
 - `internal/config/` — Cobra flag registry config loading. `config.App` is the unified config struct.
 - `internal/output/` — output printer abstraction. `--ai-output` flag controls format.
-- `internal/repo/` — git/module helpers.
 
 ## Critical decisions
 

example_test.go

@@ -12,18 +12,12 @@ func ExampleRun() {
 		// GlobalSetup runs once before any tests start.
 		testrig.GlobalSetup(func(_ context.Context) error {
 			fmt.Println("Starting mock background service...")
-			// Simulate starting a dependency, e.g.:
-			// cmd := exec.CommandContext(ctx, "docker", "compose", "up", "-d")
-			// return cmd.Run()
 			return nil
 		}),
 
 		// IterationSetup runs before each diagnose iteration.
 		testrig.IterationSetup(func(_ context.Context) error {
 			fmt.Println("Resetting database state for next iteration...")
-			// Simulate resetting state:
-			// cmd := exec.CommandContext(ctx, "psql", "-c", "TRUNCATE events")
-			// return cmd.Run()
 			return nil
 		}),
 

go.mod

@@ -15,6 +15,7 @@ require (
 	github.com/spf13/cobra v1.10.2
 	github.com/spf13/pflag v1.0.10
 	github.com/stretchr/testify v1.11.1
+	golang.org/x/sync v0.20.0
 )
 
 require (
@@ -50,7 +51,6 @@ require (
 	github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e // indirect
 	golang.org/x/exp v0.0.0-20260410095643-746e56fc9e2f // indirect
 	golang.org/x/mod v0.35.0 // indirect
-	golang.org/x/sync v0.20.0 // indirect
 	golang.org/x/sys v0.45.0 // indirect
 	golang.org/x/term v0.43.0 // indirect
 	golang.org/x/text v0.37.0 // indirect

internal/runner/analyze.go

@@ -14,6 +14,7 @@ import (
 	"sort"
 	"strconv"
 	"strings"
+	"sync"
 	"time"
 
 	"charm.land/lipgloss/v2"
@@ -206,12 +207,18 @@ func (rep *Report) TestGroups() []TestGroup {
 // coupling the parser to the filesystem.
 type LogMap map[testKey]map[int]string
 
+var readerPool = sync.Pool{
+	New: func() any {
+		return bufio.NewReaderSize(nil, 1024*1024)
+	},
+}
+
 // Analyze reads per-iteration test2json streams and classifies tests.
 // Malformed lines are silently skipped (go test can interleave non-JSON).
 func Analyze(iterations []io.Reader, slowThreshold time.Duration) (*Report, LogMap, error) {
 	aggs := make(map[testKey]*aggregate)
 	for i, r := range iterations {
-		if err := scanIterationJSONL(r, i, aggs, nil); err != nil {
+		if err := scanIterationJSONL(r, i, aggs, nil, slowThreshold); err != nil {
 			return nil, nil, err
 		}
 	}
@@ -241,16 +248,35 @@ func (a *aggregate) recordElapsed(iterIdx int, d time.Duration) {
 
 // scanIterationJSONL merges one iteration's JSONL stream into aggs at iterIdx.
 // meta may be nil; when set, records e.g. compile/build failure from FailedBuild on fail events.
-func scanIterationJSONL(r io.Reader, iterIdx int, aggs map[testKey]*aggregate, meta *iterationScanMeta) error {
-	reader := bufio.NewReaderSize(r, 1024*1024)
+func scanIterationJSONL(
+	r io.Reader,
+	iterIdx int,
+	aggs map[testKey]*aggregate,
+	meta *iterationScanMeta,
+	slowThreshold time.Duration,
+) error {
+	reader := readerPool.Get().(*bufio.Reader)
+	reader.Reset(r)
+	defer func() {
+		reader.Reset(nil)
+		readerPool.Put(reader)
+	}()
+
 	for {
-		line, err := reader.ReadBytes('\n')
+		line, err := reader.ReadSlice('\n')
+		if err == bufio.ErrBufferFull {
+			rest, err2 := reader.ReadBytes('\n')
+			line = append(append([]byte(nil), line...), rest...)
+			err = err2
+		}
+
 		if len(line) > 0 && line[0] == '{' {
 			var ev TestEvent
 			if json.Unmarshal(line, &ev) == nil {
-				applyTestEvent(aggs, iterIdx, &ev, meta)
+				applyTestEvent(aggs, iterIdx, &ev, meta, slowThreshold)
 			}
 		}
+
 		if err != nil {
 			if err != io.EOF {
 				return fmt.Errorf("reading iteration %d: %w", iterIdx, err)
@@ -260,7 +286,13 @@ func scanIterationJSONL(r io.Reader, iterIdx int, aggs map[testKey]*aggregate, m
 	}
 }
 
-func applyTestEvent(aggs map[testKey]*aggregate, iterIdx int, ev *TestEvent, meta *iterationScanMeta) {
+func applyTestEvent(
+	aggs map[testKey]*aggregate,
+	iterIdx int,
+	ev *TestEvent,
+	meta *iterationScanMeta,
+	slowThreshold time.Duration,
+) {
 	key := testKey{Package: ev.Package, Test: ev.Test}
 	a := aggs[key]
 	if a == nil {
@@ -271,7 +303,11 @@ func applyTestEvent(aggs map[testKey]*aggregate, iterIdx int, ev *TestEvent, met
 	case "pass":
 		a.passes++
 		a.iterations[iterIdx] = struct{}{}
-		a.recordElapsed(iterIdx, seconds(ev.Elapsed))
+		el := seconds(ev.Elapsed)
+		a.recordElapsed(iterIdx, el)
+		if !a.timedOut && (slowThreshold == 0 || el <= slowThreshold) {
+			delete(a.outputs, iterIdx)
+		}
 	case "fail":
 		if meta != nil && ev.FailedBuild != "" {
 			meta.sawFailedBuild = true
@@ -284,7 +320,11 @@ func applyTestEvent(aggs map[testKey]*aggregate, iterIdx int, ev *TestEvent, met
 		a.skips++
 		a.iterations[iterIdx] = struct{}{}
 		a.skipIters[iterIdx] = true
-		a.recordElapsed(iterIdx, seconds(ev.Elapsed))
+		el := seconds(ev.Elapsed)
+		a.recordElapsed(iterIdx, el)
+		if !a.timedOut {
+			delete(a.outputs, iterIdx)
+		}
 	case "output":
 		if strings.Contains(ev.Output, timeoutPanic) {
 			a.timedOut = true
@@ -559,7 +599,7 @@ func countNamedTestsSkippedInAggs(aggs map[testKey]*aggregate) int {
 func DigestIterationJSONL(r io.Reader, slowThreshold time.Duration) (IterationDigest, error) {
 	aggs := make(map[testKey]*aggregate)
 	var meta iterationScanMeta
-	if err := scanIterationJSONL(r, 0, aggs, &meta); err != nil {
+	if err := scanIterationJSONL(r, 0, aggs, &meta, slowThreshold); err != nil {
 		return IterationDigest{}, err
 	}
 	reattributeTimeouts(aggs, newAggregate)

internal/runner/diagnose_progress.go

@@ -1,13 +1,8 @@
 package runner
 
 import (
-	"context"
-	"encoding/json"
-	"errors"
 	"fmt"
 	"io"
-	"os"
-	"os/exec"
 	"slices"
 	"strings"
 	"sync"
@@ -70,44 +65,6 @@ func packagePatternsFromEnd(args []string) []string {
 	return pkgs
 }
 
-// listTestPackageCount runs `go list -test -e` for the trailing package patterns
-// in go test arguments (see packagePatternsFromEnd). On error or no patterns,
-// returns an error or zero packages.
-func listTestPackageCount(ctx context.Context, repoRoot string, goTestArgs []string) (int, error) {
-	pkgs := packagePatternsFromEnd(goTestArgs)
-	if len(pkgs) == 0 {
-		return 0, errors.New("no package patterns in go test arguments (put packages last, after flags)")
-	}
-	// Binary is fixed ("go"); pkgs come from the user's CLI package patterns by design.
-	//nolint:gosec // G204: forwarded package patterns from CLI invocation
-	cmd := exec.CommandContext(ctx, "go", append([]string{"list", "-test", "-e", "-f", "{{.ImportPath}}"}, pkgs...)...)
-	cmd.Dir = repoRoot
-	cmd.Env = os.Environ()
-	out, err := cmd.Output()
-	if err != nil {
-		return 0, err
-	}
-	n := 0
-	for line := range strings.SplitSeq(string(out), "\n") {
-		if strings.TrimSpace(line) != "" {
-			n++
-		}
-	}
-	if n == 0 {
-		return 0, errors.New("go list returned no packages")
-	}
-	return n, nil
-}
-
-// diagnoseProgress tracks completed packages from a go test -json stream.
-type diagnoseProgress struct {
-	mu         sync.Mutex
-	done       map[string]struct{}
-	lastPkg    string
-	pkgOutcome map[string]string // package import path → pass|fail|skip (package-level events only)
-	total      int               // -1 when denominator is unknown (go list failed or empty)
-}
-
 type parallelDiagnoseProgress struct {
 	mu              sync.Mutex
 	renderMu        sync.Mutex
@@ -138,14 +95,6 @@ func newParallelDiagnoseProgressAt(totalIterations int, poolStartedAt time.Time)
 	}
 }
 
-func newDiagnoseProgress(totalPackages int) *diagnoseProgress {
-	return &diagnoseProgress{
-		done:       make(map[string]struct{}),
-		pkgOutcome: make(map[string]string),
-		total:      totalPackages,
-	}
-}
-
 func (p *parallelDiagnoseProgress) start(iteration int) {
 	if p == nil {
 		return
@@ -214,50 +163,6 @@ func (p *parallelDiagnoseProgress) renderSnapshot(
 	return completed, total, actives, poolElapsed
 }
 
-// onTestJSONLine updates state from one JSONL line. Returns true if the number
-// of completed packages increased (for throttled redraws).
-func (p *diagnoseProgress) onTestJSONLine(line []byte) (completedIncreased bool) {
-	if len(line) == 0 || line[0] != '{' {
-		return false
-	}
-	var ev TestEvent
-	if err := json.Unmarshal(line, &ev); err != nil {
-		return false
-	}
-	if ev.Package != "" {
-		p.mu.Lock()
-		p.lastPkg = ev.Package
-		p.mu.Unlock()
-	}
-	if !isPackageTerminalEvent(&ev) {
-		return false
-	}
-	p.mu.Lock()
-	defer p.mu.Unlock()
-	p.pkgOutcome[ev.Package] = ev.Action
-	before := len(p.done)
-	p.done[ev.Package] = struct{}{}
-	return len(p.done) > before
-}
-
-func isPackageTerminalEvent(ev *TestEvent) bool {
-	if ev.Package == "" || ev.Test != "" {
-		return false
-	}
-	switch ev.Action {
-	case "pass", "fail", "skip":
-		return true
-	default:
-		return false
-	}
-}
-
-func (p *diagnoseProgress) snapshot() (completed int, total int, lastPkg string, outcome string) {
-	p.mu.Lock()
-	defer p.mu.Unlock()
-	return len(p.done), p.total, p.lastPkg, p.pkgOutcome[p.lastPkg]
-}
-
 // progressBracket wraps inner (already styled) in muted square brackets.
 func progressBracket(inner string) string {
 	return termstyle.Muted.Render("[") + inner + termstyle.Muted.Render("]")

internal/runner/diagnose_progress_test.go

@@ -11,61 +11,6 @@ import (
 	"github.com/smartcontractkit/testrig/internal/output"
 )
 
-func TestDiagnoseProgress_onTestJSONLine_packageTerminal(t *testing.T) {
-	t.Parallel()
-	p := newDiagnoseProgress(2)
-
-	require.False(t, p.onTestJSONLine([]byte(`not json`)))
-	require.False(t, p.onTestJSONLine([]byte(`{"Action":"run","Package":"a/b","Test":"TestX"}`)))
-
-	require.True(t, p.onTestJSONLine([]byte(`{"Action":"pass","Package":"a/b"}`)))
-	c, tot, _, _ := p.snapshot()
-	require.Equal(t, 1, c)
-	require.Equal(t, 2, tot)
-
-	// Duplicate package-level pass must not report a second completion tick.
-	require.False(t, p.onTestJSONLine([]byte(`{"Action":"pass","Package":"a/b"}`)))
-	c, _, _, _ = p.snapshot()
-	require.Equal(t, 1, c)
-
-	require.True(t, p.onTestJSONLine([]byte(`{"Action":"fail","Package":"c/d"}`)))
-	c, _, _, _ = p.snapshot()
-	require.Equal(t, 2, c)
-}
-
-func TestDiagnoseProgress_onTestJSONLine_skipFail(t *testing.T) {
-	t.Parallel()
-	p := newDiagnoseProgress(1)
-	require.True(t, p.onTestJSONLine([]byte(`{"Action":"skip","Package":"p"}`)))
-	c, _, _, _ := p.snapshot()
-	require.Equal(t, 1, c)
-
-	p2 := newDiagnoseProgress(1)
-	require.True(t, p2.onTestJSONLine([]byte(`{"Action":"fail","Package":"p"}`)))
-	c2, _, _, _ := p2.snapshot()
-	require.Equal(t, 1, c2)
-}
-
-func TestDiagnoseProgress_lastPkgUpdates(t *testing.T) {
-	t.Parallel()
-	p := newDiagnoseProgress(10)
-	p.onTestJSONLine([]byte(`{"Action":"run","Package":"x/y","Test":"TestZ"}`))
-	_, _, last, _ := p.snapshot()
-	require.Equal(t, "x/y", last)
-}
-
-func TestDiagnoseProgress_pkgOutcomeOnTerminal(t *testing.T) {
-	t.Parallel()
-	p := newDiagnoseProgress(5)
-	p.onTestJSONLine([]byte(`{"Action":"run","Package":"p/q","Test":"TestZ"}`))
-	_, _, _, out := p.snapshot()
-	require.Empty(t, out)
-	p.onTestJSONLine([]byte(`{"Action":"pass","Package":"p/q"}`))
-	_, _, last, out := p.snapshot()
-	require.Equal(t, "p/q", last)
-	require.Equal(t, "pass", out)
-}
-
 func TestEllipsizeRight(t *testing.T) {
 	t.Parallel()
 	require.Equal(t, "short", ellipsizeRight("short", 10))