Put back the sessions json. It was useful after all

Signed-off-by: David Gageot <david.gageot@docker.com>

Author: dgageot

cmd/root/eval.go

@@ -124,7 +124,15 @@ func (f *evalFlags) runEvalCommand(cmd *cobra.Command, args []string) error {
 	if err != nil {
 		slog.Error("Failed to save sessions database", "error", err)
 	} else {
-		fmt.Fprintf(teeOut, "\nSessions: %s\n", dbPath)
+		fmt.Fprintf(teeOut, "\nSessions DB: %s\n", dbPath)
+	}
+
+	// Save sessions to JSON file (same format as /eval produces)
+	sessionsPath, err := evaluation.SaveRunSessionsJSON(run, outputDir)
+	if err != nil {
+		slog.Error("Failed to save sessions JSON", "error", err)
+	} else {
+		fmt.Fprintf(teeOut, "Sessions JSON: %s\n", sessionsPath)
 	}
 
 	fmt.Fprintf(teeOut, "Log: %s\n", logPath)

pkg/evaluation/save.go

@@ -307,6 +307,23 @@ func SaveRunJSON(run *EvalRun, outputDir string) (string, error) {
 	return saveJSON(run, filepath.Join(outputDir, run.Name+".json"))
 }
 
+// SaveRunSessionsJSON saves all eval sessions to a single JSON file.
+// Each session is saved in the same format as /eval produces (session.Session).
+// This complements SaveRunSessions which saves to SQLite, providing a
+// human-readable format for inspection.
+func SaveRunSessionsJSON(run *EvalRun, outputDir string) (string, error) {
+	// Collect all sessions from results
+	var sessions []*session.Session
+	for i := range run.Results {
+		if run.Results[i].Session != nil {
+			sessions = append(sessions, run.Results[i].Session)
+		}
+	}
+
+	outputPath := filepath.Join(outputDir, run.Name+".json")
+	return saveJSON(sessions, outputPath)
+}
+
 func Save(sess *session.Session, filename string) (string, error) {
 	baseName := cmp.Or(filename, sess.ID)
 

pkg/evaluation/save_test.go

@@ -1,6 +1,8 @@
 package evaluation
 
 import (
+	"encoding/json"
+	"os"
 	"path/filepath"
 	"testing"
 	"time"
@@ -107,6 +109,92 @@ func TestSaveRunSessions(t *testing.T) {
 	assert.True(t, titles["eval-test-2"], "should have eval-test-2")
 }
 
+func TestSaveRunSessionsJSON(t *testing.T) {
+	t.Parallel()
+
+	outputDir := t.TempDir()
+
+	// Create sessions with different content
+	sess1 := session.New(
+		session.WithTitle("eval-json-1"),
+		session.WithUserMessage("What is the capital of France?"),
+	)
+	sess1.InputTokens = 100
+	sess1.OutputTokens = 50
+	sess1.Cost = 0.01
+
+	sess2 := session.New(
+		session.WithTitle("eval-json-2"),
+		session.WithUserMessage("What is 2+2?"),
+	)
+	sess2.InputTokens = 80
+	sess2.OutputTokens = 30
+	sess2.Cost = 0.005
+
+	// Create an eval run with sessions
+	run := &EvalRun{
+		Name:      "test-json-001",
+		Timestamp: time.Now(),
+		Results: []Result{
+			{
+				Title:    "eval-json-1",
+				Question: "What is the capital of France?",
+				Response: "Paris is the capital of France.",
+				Session:  sess1,
+			},
+			{
+				Title:    "eval-json-2",
+				Question: "What is 2+2?",
+				Response: "4",
+				Session:  sess2,
+			},
+			{
+				// Result without a session (error case)
+				Title:   "eval-json-3",
+				Error:   "container failed",
+				Session: nil,
+			},
+		},
+	}
+
+	// Save sessions to JSON
+	sessionsPath, err := SaveRunSessionsJSON(run, outputDir)
+	require.NoError(t, err)
+	assert.Equal(t, filepath.Join(outputDir, "test-json-001.json"), sessionsPath)
+	assert.FileExists(t, sessionsPath)
+
+	// Read and parse the JSON file
+	data, err := os.ReadFile(sessionsPath)
+	require.NoError(t, err)
+
+	var loadedSessions []*session.Session
+	err = json.Unmarshal(data, &loadedSessions)
+	require.NoError(t, err)
+
+	// Should have 2 sessions (excluding the error case)
+	assert.Len(t, loadedSessions, 2)
+
+	// Verify session content
+	titles := make(map[string]*session.Session)
+	for _, sess := range loadedSessions {
+		titles[sess.Title] = sess
+	}
+
+	assert.Contains(t, titles, "eval-json-1")
+	assert.Contains(t, titles, "eval-json-2")
+
+	// Verify cost and token data is preserved
+	sess1Loaded := titles["eval-json-1"]
+	assert.Equal(t, int64(100), sess1Loaded.InputTokens)
+	assert.Equal(t, int64(50), sess1Loaded.OutputTokens)
+	assert.InDelta(t, 0.01, sess1Loaded.Cost, 0.0001)
+
+	sess2Loaded := titles["eval-json-2"]
+	assert.Equal(t, int64(80), sess2Loaded.InputTokens)
+	assert.Equal(t, int64(30), sess2Loaded.OutputTokens)
+	assert.InDelta(t, 0.005, sess2Loaded.Cost, 0.0001)
+}
+
 func TestSaveRunSessionsWithCost(t *testing.T) {
 	t.Parallel()