From e1decc55c368114f06d3669eaa1697f275ac2828 Mon Sep 17 00:00:00 2001 From: John Kearney Date: Thu, 9 Jul 2026 18:25:43 -0500 Subject: [PATCH] [security] Fix verdict-injection via extract_json_from_llm_response Returns first matching JSON code block, allowing model-under-test to inject verdict JSON that the judge references in its reasoning. Fix: return the LAST parsed JSON object. Same class as deepeval #2868, ragas #2829, guardrails-ai #1566, promptfoo #10036, instructor #2424. --- .../crewai/experimental/evaluation/json_parser.py | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/lib/crewai/src/crewai/experimental/evaluation/json_parser.py b/lib/crewai/src/crewai/experimental/evaluation/json_parser.py index d696b49422..09474f9b55 100644 --- a/lib/crewai/src/crewai/experimental/evaluation/json_parser.py +++ b/lib/crewai/src/crewai/experimental/evaluation/json_parser.py @@ -21,12 +21,22 @@ def extract_json_from_llm_response(text: str) -> dict[str, Any]: r"`([{\\[].*[}\]])`", ] + # Collect ALL parseable JSON from code blocks, then return the LAST one. + # The judge's own verdict is the authoritative JSON and appears last; + # earlier JSON may have originated from model-under-test output that was + # referenced in the judge's reasoning (verdict injection). + all_parsed: list[dict[str, Any]] = [] + for pattern in json_patterns: matches = re.findall(pattern, text, re.IGNORECASE | re.DOTALL) for match in matches: try: parsed: dict[str, Any] = json.loads(match.strip()) - return parsed + all_parsed.append(parsed) except json.JSONDecodeError: # noqa: PERF203 continue + + if all_parsed: + return all_parsed[-1] + raise ValueError("No valid JSON found in the response")