Merge pull request #608 from rumpl/eval-command

Add "Eval" command to the command palette

Author: dgageot

cmd/root/run.go

@@ -618,9 +618,9 @@ func runUserCommand(userInput string, sess *session.Session, rt runtime.Runtime,
 	case "/exit":
 		os.Exit(0)
 	case "/eval":
-		err := evaluation.Save(sess)
+		evalFile, err := evaluation.Save(sess)
 		if err == nil {
-			fmt.Printf("%s\n", yellow("Evaluation saved"))
+			fmt.Printf("%s\n", yellow("Evaluation saved to file %s", evalFile))
 			return true, err
 		}
 		return true, nil

pkg/app/app.go

@@ -98,6 +98,10 @@ func (a *App) NewSession() {
 	a.session = session.New()
 }
 
+func (a *App) Session() *session.Session {
+	return a.session
+}
+
 func (a *App) CompactSession() {
 	if a.runtime != nil && a.session != nil {
 		events := make(chan runtime.Event, 100)

pkg/evaluation/evaluation.go

@@ -165,9 +165,9 @@ func toolTrajectoryScore(expectedToolMessages, actualToolMessages []session.Mess
 	return score / float64(len(expectedToolMessages))
 }
 
-func Save(sess *session.Session) error {
+func Save(sess *session.Session) (string, error) {
 	if err := os.MkdirAll("evals", 0o755); err != nil {
-		return err
+		return "", err
 	}
 
 	fileName := sess.ID + ".json"
@@ -182,11 +182,12 @@ func Save(sess *session.Session) error {
 		}
 	}
 
-	file, err := os.Create(filepath.Join("evals", fileName))
+	evalFile := filepath.Join("evals", fileName)
+	file, err := os.Create(evalFile)
 	if err != nil {
-		return err
+		return "", err
 	}
 	defer file.Close()
 
-	return json.NewEncoder(file).Encode(sess)
+	return evalFile, json.NewEncoder(file).Encode(sess)
 }

pkg/tui/components/messages/messages.go

@@ -25,6 +25,10 @@ type StreamCancelledMsg struct {
 	ShowMessage bool // Whether to show a cancellation message after cleanup
 }
 
+type EvalSavedMsg struct {
+	File string
+}
+
 // Model represents a chat message list component
 type Model interface {
 	layout.Model
@@ -104,7 +108,10 @@ func (m *model) Update(msg tea.Msg) (tea.Model, tea.Cmd) {
 		m.removeSpinner()
 		m.cancelPendingToolCalls()
 		return m, nil
-
+	case EvalSavedMsg:
+		m.AddSystemMessage(fmt.Sprintf("Eval saved to file %s", msg.File))
+		cmd := m.ScrollToBottom()
+		return m, cmd
 	case tea.WindowSizeMsg:
 		cmd := m.SetSize(msg.Width, msg.Height)
 		if cmd != nil {

pkg/tui/dialog/command_palette.go

@@ -133,7 +133,7 @@ func (d *commandPaletteDialog) Update(msg tea.Msg) (tea.Model, tea.Cmd) {
 				if selectedCmd.Execute != nil {
 					cmds = append(cmds, selectedCmd.Execute())
 				}
-				return d, tea.Batch(cmds...)
+				return d, tea.Sequence(cmds...)
 			}
 			return d, nil
 

pkg/tui/page/chat/chat.go

@@ -187,9 +187,8 @@ func (p *chatPage) Update(msg tea.Msg) (tea.Model, tea.Cmd) {
 		p.messages = model.(messages.Model)
 
 		var cmds []tea.Cmd
-		if cmd != nil {
-			cmds = append(cmds, cmd)
-		}
+		cmds = append(cmds, cmd)
+
 		if msg.ShowMessage {
 			cmds = append(cmds, p.messages.AddCancelledMessage())
 		}

pkg/tui/tui.go

@@ -9,7 +9,9 @@ import (
 	"github.com/charmbracelet/lipgloss/v2"
 
 	"github.com/docker/cagent/pkg/app"
+	"github.com/docker/cagent/pkg/evaluation"
 	"github.com/docker/cagent/pkg/runtime"
+	"github.com/docker/cagent/pkg/tui/components/messages"
 	"github.com/docker/cagent/pkg/tui/components/statusbar"
 	"github.com/docker/cagent/pkg/tui/core"
 	"github.com/docker/cagent/pkg/tui/dialog"
@@ -327,6 +329,16 @@ func (a *appModel) buildCommandCategories() []dialog.CommandCategory {
 						return a.chatPage.CopySessionToClipboard()
 					},
 				},
+				{
+					ID:          "session.eval",
+					Label:       "Eval",
+					Description: "Create an evaluation report for the current conversation",
+					Category:    "Session",
+					Execute: func() tea.Cmd {
+						evalFile, _ := evaluation.Save(a.application.Session())
+						return core.CmdHandler(messages.EvalSavedMsg{File: evalFile})
+					},
+				},
 			},
 		},
 	}