Merge pull request #1516 from dgageot/eval-base-image

Pass a custom base image to cagent eval

Author: dgageot

cmd/root/eval.go

@@ -28,6 +28,7 @@ type evalFlags struct {
 	judgeModel  string
 	outputDir   string
 	only        []string
+	baseImage   string
 }
 
 func newEvalCmd() *cobra.Command {
@@ -46,6 +47,7 @@ func newEvalCmd() *cobra.Command {
 	cmd.Flags().StringVar(&flags.judgeModel, "judge-model", defaultJudgeModel, "Model to use for relevance checking (format: provider/model)")
 	cmd.Flags().StringVar(&flags.outputDir, "output", "", "Directory for results and logs (default: <eval-dir>/results)")
 	cmd.Flags().StringSliceVar(&flags.only, "only", nil, "Only run evaluations with file names matching these patterns (can be specified multiple times)")
+	cmd.Flags().StringVar(&flags.baseImage, "base-image", "", "Custom base Docker image for running evaluations")
 
 	return cmd
 }
@@ -137,7 +139,7 @@ func (f *evalFlags) runEvalCommand(cmd *cobra.Command, args []string) error {
 	}
 
 	// Run evaluation
-	run, evalErr := evaluation.EvaluateWithName(ctx, teeOut, isTTY, ttyFd, runName, agentFilename, evalsDir, &f.runConfig, f.concurrency, judgeModel, f.only)
+	run, evalErr := evaluation.EvaluateWithName(ctx, teeOut, isTTY, ttyFd, runName, agentFilename, evalsDir, &f.runConfig, f.concurrency, judgeModel, f.only, f.baseImage)
 	if run == nil {
 		return evalErr
 	}

pkg/evaluation/Dockerfile.custom.template

@@ -0,0 +1,10 @@
+# syntax=docker/dockerfile:1
+
+FROM {{.BaseImage}}
+COPY --from=docker/cagent:edge /cagent /
+WORKDIR /working_dir
+ENV TELEMETRY_ENABLED=false
+ENV CAGENT_HIDE_TELEMETRY_BANNER=1
+ENTRYPOINT ["/cagent", "exec", "--yolo", "--json"]
+{{if .CopyWorkingDir}}COPY . ./
+{{end}}

pkg/evaluation/Dockerfile.template

@@ -23,13 +23,12 @@ exec "$@"
 EOF
 RUN chmod +x /run.sh
 
-FROM docker/cagent:edge AS cagent
-
 FROM scratch
 COPY --from=dind / /
-COPY --from=cagent /cagent /
+COPY --from=docker/cagent:edge /cagent /
 WORKDIR /working_dir
-{{if .CopyWorkingDir}}COPY . ./
-{{end}}ENV TELEMETRY_ENABLED=false
+ENV TELEMETRY_ENABLED=false
 ENV CAGENT_HIDE_TELEMETRY_BANNER=1
 ENTRYPOINT ["/run.sh", "/cagent", "exec", "--yolo", "--json"]
+{{if .CopyWorkingDir}}COPY . ./
+{{end}}

pkg/evaluation/build.go

@@ -17,13 +17,18 @@ var (
 	//go:embed Dockerfile.template
 	dockerfileTmpl string
 
-	dockerfileTemplate = template.Must(template.New("Dockerfile").Parse(dockerfileTmpl))
+	//go:embed Dockerfile.custom.template
+	dockerfileCustomTmpl string
+
+	dockerfileTemplate       = template.Must(template.New("Dockerfile").Parse(dockerfileTmpl))
+	dockerfileCustomTemplate = template.Must(template.New("DockerfileCustom").Parse(dockerfileCustomTmpl))
 )
 
 func (r *Runner) buildEvalImage(ctx context.Context, workingDir string) (string, error) {
 	var buildContext string
 	var data struct {
 		CopyWorkingDir bool
+		BaseImage      string
 	}
 
 	if workingDir == "" {
@@ -37,8 +42,15 @@ func (r *Runner) buildEvalImage(ctx context.Context, workingDir string) (string,
 		data.CopyWorkingDir = true
 	}
 
+	// Choose template based on whether a custom base image is provided
+	tmpl := dockerfileTemplate
+	if r.baseImage != "" {
+		tmpl = dockerfileCustomTemplate
+		data.BaseImage = r.baseImage
+	}
+
 	var dockerfile bytes.Buffer
-	if err := dockerfileTemplate.Execute(&dockerfile, data); err != nil {
+	if err := tmpl.Execute(&dockerfile, data); err != nil {
 		return "", fmt.Errorf("executing dockerfile template: %w", err)
 	}
 

pkg/evaluation/eval.go

@@ -35,6 +35,7 @@ type Runner struct {
 	envProvider   environment.Provider
 	ttyFd         int
 	only          []string
+	baseImage     string
 }
 
 // NewRunner creates a new evaluation runner.
@@ -48,16 +49,17 @@ func NewRunner(agentSource config.Source, runConfig *config.RuntimeConfig, evals
 		envProvider:   runConfig.EnvProvider(),
 		ttyFd:         cfg.TTYFd,
 		only:          cfg.Only,
+		baseImage:     cfg.BaseImage,
 	}
 }
 
 // Evaluate is the main entry point for running evaluations.
-func Evaluate(ctx context.Context, out io.Writer, isTTY bool, ttyFd int, agentFilename, evalsDir string, runConfig *config.RuntimeConfig, concurrency int, judgeModel provider.Provider, only []string) (*EvalRun, error) {
-	return EvaluateWithName(ctx, out, isTTY, ttyFd, GenerateRunName(), agentFilename, evalsDir, runConfig, concurrency, judgeModel, only)
+func Evaluate(ctx context.Context, out io.Writer, isTTY bool, ttyFd int, agentFilename, evalsDir string, runConfig *config.RuntimeConfig, concurrency int, judgeModel provider.Provider, only []string, baseImage string) (*EvalRun, error) {
+	return EvaluateWithName(ctx, out, isTTY, ttyFd, GenerateRunName(), agentFilename, evalsDir, runConfig, concurrency, judgeModel, only, baseImage)
 }
 
 // EvaluateWithName runs evaluations with a specified run name.
-func EvaluateWithName(ctx context.Context, out io.Writer, isTTY bool, ttyFd int, runName, agentFilename, evalsDir string, runConfig *config.RuntimeConfig, concurrency int, judgeModel provider.Provider, only []string) (*EvalRun, error) {
+func EvaluateWithName(ctx context.Context, out io.Writer, isTTY bool, ttyFd int, runName, agentFilename, evalsDir string, runConfig *config.RuntimeConfig, concurrency int, judgeModel provider.Provider, only []string, baseImage string) (*EvalRun, error) {
 	agentSource, err := config.Resolve(agentFilename)
 	if err != nil {
 		return nil, fmt.Errorf("resolving agent: %w", err)
@@ -68,6 +70,7 @@ func EvaluateWithName(ctx context.Context, out io.Writer, isTTY bool, ttyFd int,
 		JudgeModel:  judgeModel,
 		TTYFd:       ttyFd,
 		Only:        only,
+		BaseImage:   baseImage,
 	})
 
 	fmt.Fprintf(out, "Evaluation run: %s\n", runName)

pkg/evaluation/types.go

@@ -114,6 +114,7 @@ type Config struct {
 	Concurrency int               // Number of concurrent runs (0 = number of CPUs)
 	TTYFd       int               // File descriptor for terminal size queries (e.g., int(os.Stdout.Fd()))
 	Only        []string          // Only run evaluations matching these patterns
+	BaseImage   string            // Custom base Docker image for running evaluations
 }
 
 // Session helper functions