Validate llm judge

Signed-off-by: David Gageot <david.gageot@docker.com>

Author: dgageot

pkg/evaluation/eval.go

@@ -117,6 +117,20 @@ func (r *Runner) Run(ctx context.Context, ttyOut, out io.Writer, isTTY bool) ([]
 		return nil, fmt.Errorf("loading evaluations: %w", err)
 	}
 
+	// Check whether any evaluations require relevance checking.
+	// If so, the judge must be configured and working; validate eagerly
+	// to fail fast on configuration issues (bad API key, wrong model, etc.)
+	// instead of silently producing zero-relevance results.
+	if needsJudge(evals) {
+		if r.judge == nil {
+			return nil, errors.New("some evaluations have relevance criteria but no judge model is configured (use --judge-model)")
+		}
+		fmt.Fprintln(out, "Validating judge model...")
+		if err := r.judge.Validate(ctx); err != nil {
+			return nil, fmt.Errorf("%w", err)
+		}
+	}
+
 	// Pre-build all unique Docker images in parallel before running evaluations.
 	// This avoids serialized builds when multiple workers need the same image.
 	if err := r.preBuildImages(ctx, out, evals); err != nil {
@@ -341,12 +355,12 @@ func (r *Runner) runSingleEval(ctx context.Context, evalSess *InputSession) (Res
 	if r.judge != nil && len(evals.Relevance) > 0 {
 		// Use transcript for relevance checking to preserve temporal ordering
 		transcript := buildTranscript(events)
-		passed, failed, errs := r.judge.CheckRelevance(ctx, transcript, evals.Relevance)
+		passed, failed, err := r.judge.CheckRelevance(ctx, transcript, evals.Relevance)
+		if err != nil {
+			return result, fmt.Errorf("relevance check failed: %w", err)
+		}
 		result.RelevancePassed = float64(passed)
 		result.FailedRelevance = failed
-		for _, e := range errs {
-			slog.Warn("Relevance check error", "title", evalSess.Title, "error", e)
-		}
 	}
 
 	slog.Debug("Evaluation complete", "title", evalSess.Title, "duration", time.Since(startTime))
@@ -590,6 +604,14 @@ func matchesAnyPattern(name string, patterns []string) bool {
 	})
 }
 
+// needsJudge returns true if any evaluation session has relevance criteria,
+// meaning a judge model is required to evaluate them.
+func needsJudge(evals []InputSession) bool {
+	return slices.ContainsFunc(evals, func(s InputSession) bool {
+		return s.Evals != nil && len(s.Evals.Relevance) > 0
+	})
+}
+
 // createJudgeModel creates a provider.Provider from a model string (format: provider/model).
 // Returns nil if judgeModel is empty.
 func createJudgeModel(ctx context.Context, judgeModel string, runConfig *config.RuntimeConfig) (provider.Provider, error) {

pkg/evaluation/eval_test.go

@@ -12,6 +12,8 @@ import (
 
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
+
+	"github.com/docker/docker-agent/pkg/session"
 )
 
 func TestToolCallF1Score(t *testing.T) {
@@ -1009,3 +1011,58 @@ func TestMatchesAnyPattern(t *testing.T) {
 		})
 	}
 }
+
+func TestNeedsJudge(t *testing.T) {
+	t.Parallel()
+
+	tests := []struct {
+		name  string
+		evals []InputSession
+		want  bool
+	}{
+		{
+			name:  "no evals",
+			evals: nil,
+			want:  false,
+		},
+		{
+			name: "evals without relevance criteria",
+			evals: []InputSession{
+				{Session: &session.Session{Evals: &session.EvalCriteria{Size: "M"}}},
+				{Session: &session.Session{Evals: &session.EvalCriteria{}}},
+			},
+			want: false,
+		},
+		{
+			name: "evals with nil Evals field",
+			evals: []InputSession{
+				{Session: &session.Session{}},
+			},
+			want: false,
+		},
+		{
+			name: "some evals with relevance criteria",
+			evals: []InputSession{
+				{Session: &session.Session{Evals: &session.EvalCriteria{}}},
+				{Session: &session.Session{Evals: &session.EvalCriteria{Relevance: []string{"criterion1"}}}},
+			},
+			want: true,
+		},
+		{
+			name: "all evals with relevance criteria",
+			evals: []InputSession{
+				{Session: &session.Session{Evals: &session.EvalCriteria{Relevance: []string{"a", "b"}}}},
+				{Session: &session.Session{Evals: &session.EvalCriteria{Relevance: []string{"c"}}}},
+			},
+			want: true,
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			t.Parallel()
+			got := needsJudge(tt.evals)
+			assert.Equal(t, tt.want, got)
+		})
+	}
+}

pkg/evaluation/judge.go

@@ -82,15 +82,41 @@ func NewJudge(model provider.Provider, runConfig *config.RuntimeConfig, concurre
 	}
 }
 
+// Validate performs an end-to-end check of the judge model by sending a
+// trivial relevance prompt and verifying the response is valid structured
+// JSON. This catches configuration errors (bad API key, unsupported model,
+// missing structured-output support, etc.) before running any evaluations,
+// allowing the framework to fail fast.
+func (j *Judge) Validate(ctx context.Context) error {
+	const (
+		testResponse  = "The sky is blue."
+		testCriterion = "The response mentions a color."
+	)
+
+	passed, _, err := j.checkSingle(ctx, testResponse, testCriterion)
+	if err != nil {
+		return fmt.Errorf("judge model validation failed: %w", err)
+	}
+
+	if !passed {
+		return errors.New("judge model validation failed: expected the test criterion to pass but the judge returned 'fail'")
+	}
+
+	return nil
+}
+
 // RelevanceResult contains the result of a single relevance check.
 type RelevanceResult struct {
 	Criterion string `json:"criterion"`
 	Reason    string `json:"reason"`
 }
 
 // CheckRelevance runs all relevance checks concurrently with the configured concurrency.
-// It returns the number of passed checks, a slice of failed results with reasons, and any errors encountered.
-func (j *Judge) CheckRelevance(ctx context.Context, response string, criteria []string) (passed int, failed []RelevanceResult, errs []string) {
+// It returns the number of passed checks, a slice of failed results with reasons, and an error
+// if any check encountered an error (e.g. judge model misconfiguration). Errors cause a hard
+// failure so that configuration issues are surfaced immediately rather than silently producing
+// zero-relevance results.
+func (j *Judge) CheckRelevance(ctx context.Context, response string, criteria []string) (passed int, failed []RelevanceResult, err error) {
 	if len(criteria) == 0 {
 		return 0, nil, nil
 	}
@@ -122,17 +148,19 @@ func (j *Judge) CheckRelevance(ctx context.Context, response string, criteria []
 					results[item.index] = result{err: fmt.Errorf("context cancelled: %w", ctx.Err())}
 					continue
 				}
-				pass, reason, err := j.checkSingle(ctx, response, item.criterion)
-				results[item.index] = result{passed: pass, reason: reason, err: err}
+				pass, reason, checkErr := j.checkSingle(ctx, response, item.criterion)
+				results[item.index] = result{passed: pass, reason: reason, err: checkErr}
 			}
 		})
 	}
 	wg.Wait()
 
-	// Aggregate results
+	// Aggregate results. Any error is fatal — return it immediately so the
+	// caller can fail fast on judge misconfiguration.
+	var errs []error
 	for i, r := range results {
 		if r.err != nil {
-			errs = append(errs, fmt.Sprintf("error checking %q: %v", criteria[i], r.err))
+			errs = append(errs, fmt.Errorf("checking %q: %w", criteria[i], r.err))
 			continue
 		}
 		if r.passed {
@@ -145,7 +173,11 @@ func (j *Judge) CheckRelevance(ctx context.Context, response string, criteria []
 		}
 	}
 
-	return passed, failed, errs
+	if len(errs) > 0 {
+		return passed, failed, errors.Join(errs...)
+	}
+
+	return passed, failed, nil
 }
 
 // getOrCreateJudgeWithSchema returns a provider pre-configured with structured output.

pkg/evaluation/judge_test.go

@@ -5,6 +5,7 @@ import (
 	"testing"
 
 	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
 )
 
 func TestNewJudge(t *testing.T) {
@@ -46,11 +47,11 @@ func TestJudge_CheckRelevance_EmptyCriteria(t *testing.T) {
 	t.Parallel()
 
 	judge := NewJudge(nil, nil, 1)
-	passed, failed, errs := judge.CheckRelevance(t.Context(), "some response", nil)
+	passed, failed, err := judge.CheckRelevance(t.Context(), "some response", nil)
 
 	assert.Equal(t, 0, passed)
 	assert.Empty(t, failed)
-	assert.Empty(t, errs)
+	assert.NoError(t, err)
 }
 
 func TestJudge_CheckRelevance_ContextCanceled(t *testing.T) {
@@ -62,13 +63,11 @@ func TestJudge_CheckRelevance_ContextCanceled(t *testing.T) {
 	cancel() // Cancel immediately
 
 	criteria := []string{"criterion1", "criterion2", "criterion3"}
-	passed, failed, errs := judge.CheckRelevance(ctx, "some response", criteria)
+	passed, failed, err := judge.CheckRelevance(ctx, "some response", criteria)
 
 	// All should have errors due to context cancellation
 	assert.Equal(t, 0, passed)
 	assert.Empty(t, failed)
-	assert.Len(t, errs, 3)
-	for _, err := range errs {
-		assert.Contains(t, err, "context cancelled")
-	}
+	require.Error(t, err)
+	assert.Contains(t, err.Error(), "context cancelled")
 }