Support different number of tool calls

Signed-off-by: David Gageot <david.gageot@docker.com>

Author: dgageot

cmd/root/eval.go

@@ -56,7 +56,6 @@ func (f *evalFlags) runEvalCommand(cmd *cobra.Command, args []string) error {
 		out.Printf("Eval file: %s\n", result.EvalFile)
 		out.Printf("Tool trajectory score: %f\n", result.Score.ToolTrajectoryScore)
 		out.Printf("Rouge-1 score: %f\n", result.Score.Rouge1Score)
-
 	})
 	return err
 }

pkg/evaluation/score.go

@@ -67,18 +67,20 @@ func rouge1(expected, actual string) float64 {
 }
 
 func toolTrajectoryScore(expectedToolMessages, actualToolMessages []session.Message) float64 {
+	maximum := 0.0
 	score := 0.0
 
-	for i := range expectedToolMessages {
+	for i := range min(len(expectedToolMessages), len(actualToolMessages)) {
 		expected := expectedToolMessages[i]
 		actual := actualToolMessages[i]
 
 		for j := range actual.Message.ToolCalls {
+			maximum += 1.0
 			if actual.Message.ToolCalls[j].Function.Name == expected.Message.ToolCalls[j].Function.Name {
 				score += 1.0
 			}
 		}
 	}
 
-	return score / float64(len(expectedToolMessages))
+	return score / maximum
 }