From da92fb8e731e86f027e310ba9ebe7edde672f1cb Mon Sep 17 00:00:00 2001 From: Christopher Tso Date: Thu, 2 Apr 2026 06:19:16 +0000 Subject: [PATCH] fix(core): accept content object shorthands in eval validator The eval validator's validateMessages() rejected input array items that have a 'type' field (e.g., {type: "file", value: "..."}) but no 'role' field, producing spurious errors about invalid role 'undefined'. The runtime (shorthand-expansion.ts) silently filters these via isTestMessage(), so the validator was stricter than actual execution. Add an early-continue for content object shorthands (items with 'type' but no 'role') so the validator accepts what the runtime accepts. Closes #915 Co-Authored-By: Claude Opus 4.6 --- packages/core/src/evaluation/validation/eval-validator.ts | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/packages/core/src/evaluation/validation/eval-validator.ts b/packages/core/src/evaluation/validation/eval-validator.ts index cdf00fe4..923d3a18 100644 --- a/packages/core/src/evaluation/validation/eval-validator.ts +++ b/packages/core/src/evaluation/validation/eval-validator.ts @@ -486,6 +486,12 @@ function validateMessages( continue; } + // Content object shorthand: items with 'type' (e.g., {type: "file", value: "..."}) + // but no 'role' are valid content shorthands — the runtime wraps them implicitly. + if (!('role' in message) && 'type' in message) { + continue; // Accept silently — runtime handles expansion + } + // Validate role field const role = message.role; const validRoles = ['system', 'user', 'assistant'];