From 29d09f00fc271168af5599e3762da4dcc9387102 Mon Sep 17 00:00:00 2001 From: Adriana Knight Date: Fri, 23 Oct 2026 12:54:59 -0400 Subject: [PATCH 1/5] Use RFC 8785 json canonicalization and normalize text behaviors --- .../evaluations/canonical_json.py | 90 ++++++++++++++ .../evaluations/runner.py | 30 +++-- .../launchdarkly_ai_server/judge_scoring.py | 27 +++-- packages/client/tests/test_canonical_json.py | 114 ++++++++++++++++++ packages/client/tests/test_evaluations_run.py | 37 +++++- packages/client/tests/test_judge_scoring.py | 27 ++++- 6 files changed, 295 insertions(+), 30 deletions(-) create mode 100644 packages/client/src/launchdarkly_ai_server/evaluations/canonical_json.py create mode 100644 packages/client/tests/test_canonical_json.py diff --git a/packages/client/src/launchdarkly_ai_server/evaluations/canonical_json.py b/packages/client/src/launchdarkly_ai_server/evaluations/canonical_json.py new file mode 100644 index 00000000..3f7d28a0 --- /dev/null +++ b/packages/client/src/launchdarkly_ai_server/evaluations/canonical_json.py @@ -0,0 +1,90 @@ +"""RFC 8785 JSON Canonicalization Scheme (JCS), for hashes every SDK must agree on. + +JCS spells JSON the way ECMAScript's ``JSON.stringify`` does: no whitespace, +object keys sorted by UTF-16 code unit, non-ASCII written as is, and numbers in +ECMAScript's shortest round-trip form. That is the native form in JavaScript +and close to it in Go, so the only language that needs this module is Python, +whose ``json.dumps`` writes ``1.0`` where JCS writes ``1``. +""" + +from __future__ import annotations + +import json +import re +from decimal import Decimal +from math import isfinite +from typing import Any + +# Above this an int has no exact IEEE double, so it is written as the double +# nearest to it, as a JavaScript JSON.parse reads it. +_MAX_SAFE_INTEGER = 2**53 - 1 +_SURROGATE = re.compile("[\ud800-\udfff]") + + +def canonical_json(value: Any) -> str: + """The JCS text of a JSON-like value. + + The value is first normalized through ``json.dumps(..., default=str)``, so + a tuple is a list, a non-string key is the string ``json.dumps`` writes for + it, and a value JSON cannot hold is its ``str()``. A non-finite number is + written ``null``, as ``JSON.stringify`` writes it. + """ + return _encode(json.loads(json.dumps(value, default=str))) + + +def _encode(value: Any) -> str: + if value is None or value is True or value is False: + return json.dumps(value) + if isinstance(value, int): + if abs(value) <= _MAX_SAFE_INTEGER: + return str(value) + try: + return _number(float(value)) + except OverflowError: + return "null" + if isinstance(value, float): + return _number(value) + if isinstance(value, str): + return _string(value) + if isinstance(value, list): + return "[" + ",".join(_encode(item) for item in value) + "]" + keys = sorted(value, key=_utf16) + return "{" + ",".join(f"{_string(key)}:{_encode(value[key])}" for key in keys) + "}" + + +def _number(value: float) -> str: + """ECMAScript's Number::toString, from the shortest round-trip digits ``repr`` gives.""" + if not isfinite(value): + return "null" + if value == 0: + return "0" + _, digit_tuple, exponent = Decimal(repr(abs(value))).as_tuple() + # The value is 0. x 10^n, the form Number::toString starts from. + n = len(digit_tuple) + int(exponent) + digits = "".join(map(str, digit_tuple)).rstrip("0") + k = len(digits) + if k <= n <= 21: + text = digits + "0" * (n - k) + elif 0 < n <= 21: + text = f"{digits[:n]}.{digits[n:]}" + elif -6 < n <= 0: + text = f"0.{'0' * -n}{digits}" + else: + fraction = f".{digits[1:]}" if k > 1 else "" + text = f"{digits[0]}{fraction}e{'+' if n - 1 >= 0 else '-'}{abs(n - 1)}" + return ("-" if value < 0 else "") + text + + +def _string(text: str) -> str: + # A surrogate pair held as two code points is joined into one character, as + # it is in a JavaScript string; a lone surrogate is escaped, as + # JSON.stringify escapes it, so the text always has a UTF-8 encoding. + joined = text.encode("utf-16", "surrogatepass").decode("utf-16", "surrogatepass") + return _SURROGATE.sub( + lambda match: f"\\u{ord(match.group()):04x}", + json.dumps(joined, ensure_ascii=False), + ) + + +def _utf16(key: str) -> bytes: + return key.encode("utf-16-be", "surrogatepass") diff --git a/packages/client/src/launchdarkly_ai_server/evaluations/runner.py b/packages/client/src/launchdarkly_ai_server/evaluations/runner.py index e76bf0ea..b12ee5cd 100644 --- a/packages/client/src/launchdarkly_ai_server/evaluations/runner.py +++ b/packages/client/src/launchdarkly_ai_server/evaluations/runner.py @@ -43,6 +43,7 @@ require_string, segment, ) +from .canonical_json import canonical_json from .criteria import Criterion, Judge, Scorer from .events import ( CriterionEventPayload, @@ -95,6 +96,17 @@ def _provides_for( return handler_key(handler) +def _describe_value(value: Any) -> str: + """A value in an error message, as JSON rather than a Python repr, so it reads the same in every SDK. + + A non-finite number is ``NaN`` / ``Infinity``, and a value JSON cannot hold its ``str()``. + """ + try: + return json.dumps(value, ensure_ascii=False, default=str) + except (TypeError, ValueError): + return str(value) + + def describe_handlers(handlers: Sequence[EvalHandler]) -> str: """Return the ``provides_for`` of each handler, for an error message.""" return ", ".join(repr(_provides_for(handler)) for handler in handlers) @@ -733,9 +745,7 @@ def _emit_generation_events( } if dataset.id is None: del identity["datasetId"] - event_id = hashlib.sha256( - json.dumps(identity, sort_keys=True, separators=(",", ":")).encode() - ).hexdigest() + event_id = hashlib.sha256(canonical_json(identity).encode()).hexdigest() error = result.get("error") generated = { "status": result["status"], @@ -758,9 +768,7 @@ def _emit_generation_events( "outputTokens": normalized_usage["output"], } content_hash = hashlib.sha256( - json.dumps( - generated, sort_keys=True, separators=(",", ":"), default=str - ).encode() + canonical_json(generated).encode() ).hexdigest() emitted_at = datetime.now(UTC).isoformat().replace("+00:00", "Z") payload: dict[str, Any] = { @@ -899,7 +907,7 @@ async def _run_scorer_for_result( started_clock, "invalid_score", "scorer fn must return a bool or a finite number, " - f"got {score_value!r}", + f"got {_describe_value(score_value)}", ) score = maybe_score if score < 0 or score > 1: @@ -907,7 +915,7 @@ async def _run_scorer_for_result( base, started_clock, "invalid_score", - f"scorer fn score must be between 0 and 1, got {score_value!r}", + f"scorer fn score must be between 0 and 1, got {_describe_value(score_value)}", ) completed = datetime.now(UTC) return { @@ -990,7 +998,7 @@ async def _run_ld_judge_for_result( base, started_clock, "invalid_score", - f"judge score must be a number between 0 and 1, got {raw_score!r}", + f"judge score must be a number between 0 and 1, got {_describe_value(raw_score)}", ) completed = datetime.now(UTC) # No verdict: the SDK reports the score and LaunchDarkly rules on it. The @@ -1080,9 +1088,7 @@ def _emit_evaluation_events( } if dataset.id is None: del identity["datasetId"] - event_id = hashlib.sha256( - json.dumps(identity, sort_keys=True, separators=(",", ":")).encode() - ).hexdigest() + event_id = hashlib.sha256(canonical_json(identity).encode()).hexdigest() emitted_at = datetime.now(UTC).isoformat().replace("+00:00", "Z") usage: TokenUsage | None = None if result["kind"] == "judge" and isinstance( diff --git a/packages/client/src/launchdarkly_ai_server/judge_scoring.py b/packages/client/src/launchdarkly_ai_server/judge_scoring.py index 54f82b44..151b3501 100644 --- a/packages/client/src/launchdarkly_ai_server/judge_scoring.py +++ b/packages/client/src/launchdarkly_ai_server/judge_scoring.py @@ -42,23 +42,24 @@ def build_message_history( """The conversation a judge is shown, as its ``message_history`` variable. Ordered as it happened: what was asked, what the agent did, what it - answered, then how to format the verdict. Empty parts are skipped, so a - run with no tools yields the history it did before trajectories existed. + answered, then how to format the verdict. A part whose text is empty is + skipped, so a run with no tools yields the history it did before + trajectories existed. The formatting block is appended here, not by callers: judges built from the AI Library's default templates read the JSON shape from ``{{message_history}}``, and one that stopped being told it would return prose and fail every result as invalid output. - A non-string output is shown as JSON, by the rule tool results follow, so a - rubric reads the same text in every SDK. + Every part is rendered by the rule tool results follow, so a non-string + output is shown as JSON and a rubric reads the same text in every SDK. An + output of ``0``, ``False`` or ``{}`` is an answer, and is shown. """ - output_text = tool_result_string(output) if output else None - return "\n\n".join( - str(part) - for part in (user_input, trajectory, output_text, FORMATTING_INSTRUCTIONS) - if part + texts = ( + tool_result_string(part) + for part in (user_input, trajectory, output, FORMATTING_INSTRUCTIONS) ) + return "\n\n".join(text for text in texts if text) def numeric_score(score: Any) -> float | None: @@ -129,6 +130,10 @@ def parse_judge_response(raw: Any) -> tuple[Any, str]: already-decoded mapping. The score is returned untouched — callers apply their own policy to non-numeric values via :func:`numeric_score`. + The reasoning is the first of ``reasoning`` and ``reason`` whose text is + not empty, rendered by the tool-result rule: a string as it is, anything + else as JSON, never as a Python repr. + Raises ``ValueError`` when the response is not a non-empty JSON object. """ parsed: Any @@ -140,8 +145,8 @@ def parse_judge_response(raw: Any) -> tuple[Any, str]: parsed = None if not isinstance(parsed, Mapping) or not parsed: raise ValueError("Invalid JSON from judge") - reasoning = parsed.get("reasoning") or parsed.get("reason") or "" - return parsed.get("score"), str(reasoning) + texts = (tool_result_string(parsed.get(key)) for key in ("reasoning", "reason")) + return parsed.get("score"), next((text for text in texts if text), "") def without_output_format(config: Any, judge_key: str) -> Any: diff --git a/packages/client/tests/test_canonical_json.py b/packages/client/tests/test_canonical_json.py new file mode 100644 index 00000000..ce007d94 --- /dev/null +++ b/packages/client/tests/test_canonical_json.py @@ -0,0 +1,114 @@ +"""RFC 8785 canonical JSON. Each expected string is what JavaScript's JSON.stringify writes.""" + +from __future__ import annotations + +import hashlib +from typing import Any + +import pytest + +from launchdarkly_ai_server.evaluations.canonical_json import canonical_json + + +@pytest.mark.parametrize( + ("value", "text"), + [ + (1.0, "1"), + (-2.0, "-2"), + (-0.0, "0"), + (0.1, "0.1"), + (1 / 3, "0.3333333333333333"), + (1e16, "10000000000000000"), + (1e20, "100000000000000000000"), + (1e21, "1e+21"), + (1.5e300, "1.5e+300"), + (1e-6, "0.000001"), + (1e-7, "1e-7"), + (-2.5e-7, "-2.5e-7"), + (5e-324, "5e-324"), + (2**53 - 1, "9007199254740991"), + (2**64, "18446744073709552000"), + (10**400, "null"), + (float("nan"), "null"), + (float("-inf"), "null"), + ], +) +def test_writes_numbers_as_ecmascript_does(value: Any, text: str) -> None: + assert canonical_json(value) == text + + +def test_writes_strings_as_json_stringify_does() -> None: + assert canonical_json("café 東京 😀") == '"café 東京 😀"' + assert ( + canonical_json('\x00\x1f"\\\n\t\u2028\x7f') + == '"\\u0000\\u001f\\"\\\\\\n\\t\u2028\x7f"' + ) + # A lone surrogate is escaped; a pair held as two code points is one character. + assert canonical_json("a\udc00b") == '"a\\udc00b"' + assert canonical_json("\ud83d\ude00") == '"😀"' + + +def test_sorts_keys_by_utf16_code_unit_at_every_depth() -> None: + # U+1F600 is the surrogate pair D83D DE00, so it sorts before U+FF61. + value = {"z": 1, "\uff61": 2, "\U0001f600": 3, "a": [{"b": 1, "a": 2}]} + assert canonical_json(value) == '{"a":[{"a":2,"b":1}],"z":1,"😀":3,"｡":2}' + + +def test_normalizes_values_as_json_dumps_does() -> None: + assert canonical_json({1: (True, None)}) == '{"1":[true,null]}' + assert canonical_json({"when": object}) == '{"when":""}' + + +# Shared with the JavaScript SDK's contentHash tests: the same generated content +# must hash the same in every language. +@pytest.mark.parametrize( + ("generated", "digest"), + [ + ( + { + "status": "COMPLETE", + "output": "café 東京 😀", + "error": None, + "usage": {"inputTokens": 1, "outputTokens": 2}, + }, + "c413938130f9e5d632edbd07d529139b942e1c43c62fe3fb9ae09552dc7e7f71", + ), + ( + { + "status": "ERROR", + "output": None, + "error": {"code": 5001, "message": "handler raised: boom"}, + "errorMessage": "handler raised: boom", + }, + "04e26552bf66f7afa68ba6307c9f9c162abd2bbd963cb7e42d9dcea337152848", + ), + ( + { + "status": "COMPLETE", + "output": {"b": 1, "a": [True, None, "x"]}, + "error": None, + }, + "504de7cbb5f7d8b3ef260a99c69460c6b7f4fb3ea449c16cedde0ac0e120008c", + ), + ( + { + "status": "COMPLETE", + "output": { + "score": 1.0, + "p": 0.1, + "tiny": 1e-5, + "big": 1e16, + "list": [3, -0.0, 2.5e-7], + "neg": -2, + }, + "error": None, + "usage": {"inputTokens": 7, "outputTokens": 9}, + }, + "ae920410735c7fe62767503ea2121965d264b0d26abe396736d03be6aac77ae7", + ), + ], +) +def test_content_hash_matches_the_javascript_sdk( + generated: dict[str, Any], digest: str +) -> None: + assert hashlib.sha256(canonical_json(generated).encode()).hexdigest() == digest diff --git a/packages/client/tests/test_evaluations_run.py b/packages/client/tests/test_evaluations_run.py index 435d0f4c..6a874eb3 100644 --- a/packages/client/tests/test_evaluations_run.py +++ b/packages/client/tests/test_evaluations_run.py @@ -338,6 +338,14 @@ async def test_complete_run_with_zero_failed_and_error_rows_passes( assert "inputTokens" not in event assert "outputTokens" not in event assert len(event["eventId"]) == len(event["contentHash"]) == 64 + # The canonical JSON of what was generated, which every SDK can produce. + assert ( + event["contentHash"] + == hashlib.sha256( + b'{"error":null,"output":"generated: Order A19","status":"COMPLETE",' + b'"usage":{"inputTokens":10,"outputTokens":4}}' + ).hexdigest() + ) assert event["emittedAt"].endswith("Z") assert datetime.fromisoformat(event["emittedAt"]).tzinfo is not None assert {"input", "expected_output", "metadata", "variables"}.isdisjoint(event) @@ -2125,12 +2133,28 @@ async def fake_extract_variation( @pytest.mark.parametrize( - ("judge_output", "expected_code"), + ("judge_output", "expected_code", "expected_message"), [ - ('{"score": "high (0.9)", "reasoning": "confident"}', "invalid_score"), - ('{"score": 3, "reasoning": "confident"}', "invalid_score"), - ('{"score": NaN, "reasoning": "confident"}', "invalid_score"), - ("the answer looks right to me", "invalid_judge_output"), + ( + '{"score": "high (0.9)", "reasoning": "confident"}', + "invalid_score", + 'judge score must be a number between 0 and 1, got "high (0.9)"', + ), + ( + '{"score": 3, "reasoning": "confident"}', + "invalid_score", + "judge score must be a number between 0 and 1, got 3", + ), + ( + '{"score": NaN, "reasoning": "confident"}', + "invalid_score", + "judge score must be a number between 0 and 1, got NaN", + ), + ( + "the answer looks right to me", + "invalid_judge_output", + "Invalid JSON from judge", + ), ], ) @pytest.mark.asyncio @@ -2139,6 +2163,7 @@ async def test_bad_judge_output_emits_error_event_instead_of_crashing( stub_sdk_client: MagicMock, judge_output: str, expected_code: str, + expected_message: str, ) -> None: transport = judge_run_transport() accuracy_judge_variation(monkeypatch) @@ -2169,6 +2194,8 @@ async def handler( judge_event = next(event for event in events if event.get("kind") == "judge") assert judge_event["status"] == "ERROR" assert judge_event["error"]["code"] == expected_code + # The bad value is shown as JSON, not a Python repr, as in every SDK. + assert judge_event["error"]["message"] == expected_message assert judge_event["errorMessage"] == judge_event["error"]["message"] assert "score" not in judge_event stub_sdk_client.flush.assert_awaited() diff --git a/packages/client/tests/test_judge_scoring.py b/packages/client/tests/test_judge_scoring.py index 3091ecce..3f1360e6 100644 --- a/packages/client/tests/test_judge_scoring.py +++ b/packages/client/tests/test_judge_scoring.py @@ -38,6 +38,18 @@ def test_falls_back_to_reason_key(self) -> None: def test_null_reasoning_becomes_empty_string_not_none_literal(self) -> None: assert parse_judge_response({"score": 0.5, "reasoning": None}) == (0.5, "") + def test_non_string_reasoning_is_json_not_a_repr(self) -> None: + assert parse_judge_response({"score": 1, "reasoning": {"ok": True}}) == ( + 1, + '{"ok": true}', + ) + assert parse_judge_response({"score": 1, "reasoning": 0}) == (1, "0") + + def test_empty_reasoning_falls_back_to_reason(self) -> None: + assert parse_judge_response( + {"score": 1, "reasoning": "", "reason": "fallback"} + ) == (1, "fallback") + def test_score_returned_untouched_for_caller_policy(self) -> None: score, _ = parse_judge_response({"score": "high", "reasoning": "?"}) assert score == "high" @@ -96,6 +108,17 @@ def test_message_history_renders_a_structured_output_as_json() -> None: def test_message_history_keeps_non_ascii_and_skips_an_empty_output() -> None: assert '{"city": "Zürich"}' in build_message_history(output={"city": "Zürich"}) - assert build_message_history(user_input="q", output={}) == "\n\n".join( - ["q", FORMATTING_INSTRUCTIONS] + for empty in (None, ""): + assert build_message_history(user_input="q", output=empty) == "\n\n".join( + ["q", FORMATTING_INSTRUCTIONS] + ) + + +@pytest.mark.parametrize( + ("output", "text"), [(0, "0"), (False, "false"), ({}, "{}"), ([], "[]")] +) +def test_message_history_shows_a_falsy_output(output: Any, text: str) -> None: + # Each is an answer the judge has to grade, not an empty part. + assert build_message_history(user_input="q", output=output) == "\n\n".join( + ["q", text, FORMATTING_INSTRUCTIONS] ) From 4d24d34b43c1a397c67f1d5d30cd09743136ec69 Mon Sep 17 00:00:00 2001 From: Adriana Knight Date: Fri, 23 Oct 2026 13:07:54 -0400 Subject: [PATCH 2/5] fix test --- packages/opencode-agents/tests/test_handler.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/opencode-agents/tests/test_handler.py b/packages/opencode-agents/tests/test_handler.py index cee6de67..2d107bd5 100644 --- a/packages/opencode-agents/tests/test_handler.py +++ b/packages/opencode-agents/tests/test_handler.py @@ -1540,7 +1540,7 @@ async def test_a_fake_opencode_is_started_spoken_to_and_stopped(self) -> None: assert "--print-logs" not in argv assert "--log-level" not in argv assert records[0]["cwd"] == str(script.cwd.resolve()) - assert records[0]["config"] == str(script.config.resolve()) + assert records[0]["config"] == os.path.abspath(script.config) paths = [item["path"] for item in script.requests] assert "/event" in paths assert "/session" in paths From 7d2b7bbbf672b7853fa4fec383c0a8dc7ae4a8fa Mon Sep 17 00:00:00 2001 From: Adriana Knight Date: Fri, 23 Oct 2026 13:42:59 -0400 Subject: [PATCH 3/5] empty objects must still omit message history --- .../launchdarkly_ai_server/judge_scoring.py | 45 +++++++++++++------ packages/client/tests/test_judge_scoring.py | 25 ++++++----- 2 files changed, 46 insertions(+), 24 deletions(-) diff --git a/packages/client/src/launchdarkly_ai_server/judge_scoring.py b/packages/client/src/launchdarkly_ai_server/judge_scoring.py index 151b3501..6985afea 100644 --- a/packages/client/src/launchdarkly_ai_server/judge_scoring.py +++ b/packages/client/src/launchdarkly_ai_server/judge_scoring.py @@ -42,24 +42,36 @@ def build_message_history( """The conversation a judge is shown, as its ``message_history`` variable. Ordered as it happened: what was asked, what the agent did, what it - answered, then how to format the verdict. A part whose text is empty is - skipped, so a run with no tools yields the history it did before - trajectories existed. + answered, then how to format the verdict. An empty part (see + :func:`_is_empty_part`) is skipped, so a run with no tools yields the + history it did before trajectories existed. The formatting block is appended here, not by callers: judges built from the AI Library's default templates read the JSON shape from ``{{message_history}}``, and one that stopped being told it would return prose and fail every result as invalid output. - Every part is rendered by the rule tool results follow, so a non-string - output is shown as JSON and a rubric reads the same text in every SDK. An - output of ``0``, ``False`` or ``{}`` is an answer, and is shown. + Every other part is rendered by the rule tool results follow, so a + non-string output is shown as JSON and a rubric reads the same text in + every SDK. An output of ``0`` or ``False`` is an answer, and is shown. """ - texts = ( + return "\n\n".join( tool_result_string(part) for part in (user_input, trajectory, output, FORMATTING_INSTRUCTIONS) + if not _is_empty_part(part) ) - return "\n\n".join(text for text in texts if text) + + +def _is_empty_part(value: Any) -> bool: + """``None``, ``""``, an empty list or an empty mapping, tested before rendering. + + An empty container is skipped, as it was before structured outputs were + rendered as JSON. Python truthiness is not the test: it would also drop + ``0`` and ``False``, which are answers. + """ + if value is None or value == "": + return True + return isinstance(value, (list, tuple, Mapping)) and not value def numeric_score(score: Any) -> float | None: @@ -130,9 +142,9 @@ def parse_judge_response(raw: Any) -> tuple[Any, str]: already-decoded mapping. The score is returned untouched — callers apply their own policy to non-numeric values via :func:`numeric_score`. - The reasoning is the first of ``reasoning`` and ``reason`` whose text is - not empty, rendered by the tool-result rule: a string as it is, anything - else as JSON, never as a Python repr. + The reasoning is the first of ``reasoning`` and ``reason`` that is not an + empty part (:func:`_is_empty_part`), rendered by the tool-result rule: a + string as it is, anything else as JSON, never as a Python repr. Raises ``ValueError`` when the response is not a non-empty JSON object. """ @@ -145,8 +157,15 @@ def parse_judge_response(raw: Any) -> tuple[Any, str]: parsed = None if not isinstance(parsed, Mapping) or not parsed: raise ValueError("Invalid JSON from judge") - texts = (tool_result_string(parsed.get(key)) for key in ("reasoning", "reason")) - return parsed.get("score"), next((text for text in texts if text), "") + reasoning = next( + ( + tool_result_string(value) + for value in (parsed.get("reasoning"), parsed.get("reason")) + if not _is_empty_part(value) + ), + "", + ) + return parsed.get("score"), reasoning def without_output_format(config: Any, judge_key: str) -> Any: diff --git a/packages/client/tests/test_judge_scoring.py b/packages/client/tests/test_judge_scoring.py index 3f1360e6..cb3b990d 100644 --- a/packages/client/tests/test_judge_scoring.py +++ b/packages/client/tests/test_judge_scoring.py @@ -45,9 +45,10 @@ def test_non_string_reasoning_is_json_not_a_repr(self) -> None: ) assert parse_judge_response({"score": 1, "reasoning": 0}) == (1, "0") - def test_empty_reasoning_falls_back_to_reason(self) -> None: + @pytest.mark.parametrize("empty", [None, "", [], {}]) + def test_empty_reasoning_falls_back_to_reason(self, empty: Any) -> None: assert parse_judge_response( - {"score": 1, "reasoning": "", "reason": "fallback"} + {"score": 1, "reasoning": empty, "reason": "fallback"} ) == (1, "fallback") def test_score_returned_untouched_for_caller_policy(self) -> None: @@ -106,18 +107,20 @@ def test_message_history_renders_a_structured_output_as_json() -> None: ) -def test_message_history_keeps_non_ascii_and_skips_an_empty_output() -> None: +def test_message_history_keeps_non_ascii() -> None: assert '{"city": "Zürich"}' in build_message_history(output={"city": "Zürich"}) - for empty in (None, ""): - assert build_message_history(user_input="q", output=empty) == "\n\n".join( - ["q", FORMATTING_INSTRUCTIONS] - ) -@pytest.mark.parametrize( - ("output", "text"), [(0, "0"), (False, "false"), ({}, "{}"), ([], "[]")] -) -def test_message_history_shows_a_falsy_output(output: Any, text: str) -> None: +@pytest.mark.parametrize("empty", [None, "", {}, []]) +def test_message_history_skips_an_empty_output(empty: Any) -> None: + # Tested before rendering, so an empty container adds no block. + assert build_message_history(user_input="q", output=empty) == "\n\n".join( + ["q", FORMATTING_INSTRUCTIONS] + ) + + +@pytest.mark.parametrize(("output", "text"), [(0, "0"), (False, "false")]) +def test_message_history_shows_a_falsy_answer(output: Any, text: str) -> None: # Each is an answer the judge has to grade, not an empty part. assert build_message_history(user_input="q", output=output) == "\n\n".join( ["q", text, FORMATTING_INSTRUCTIONS] From 850f353f9241aa03c1e347657cdcb31c482f82d6 Mon Sep 17 00:00:00 2001 From: Jeff Dupont Date: Fri, 23 Oct 2026 12:07:33 -0700 Subject: [PATCH 4/5] fix(client): give judges JSON.stringify text for non-string values Judge text (message_history, response_to_evaluate, a non-string reasoning) and error-message values now use canonical_json(v, sort_keys=False), so they match the JS SDK: compact separators, insertion-order keys, 1 not 1.0. _describe_value keeps NaN/Infinity. canonical_json moves out of evaluations because judge_scoring uses it too. Co-Authored-By: Claude Opus 5.5 --- .../{evaluations => }/canonical_json.py | 20 +++++++++---- .../evaluations/runner.py | 17 ++++++----- .../launchdarkly_ai_server/judge_scoring.py | 20 +++++++++++-- packages/client/tests/test_canonical_json.py | 20 ++++++++++++- packages/client/tests/test_evaluations_run.py | 2 +- packages/client/tests/test_judge_scoring.py | 29 +++++++++++++++++-- 6 files changed, 86 insertions(+), 22 deletions(-) rename packages/client/src/launchdarkly_ai_server/{evaluations => }/canonical_json.py (81%) diff --git a/packages/client/src/launchdarkly_ai_server/evaluations/canonical_json.py b/packages/client/src/launchdarkly_ai_server/canonical_json.py similarity index 81% rename from packages/client/src/launchdarkly_ai_server/evaluations/canonical_json.py rename to packages/client/src/launchdarkly_ai_server/canonical_json.py index 3f7d28a0..d30d5ae3 100644 --- a/packages/client/src/launchdarkly_ai_server/evaluations/canonical_json.py +++ b/packages/client/src/launchdarkly_ai_server/canonical_json.py @@ -21,18 +21,22 @@ _SURROGATE = re.compile("[\ud800-\udfff]") -def canonical_json(value: Any) -> str: +def canonical_json(value: Any, *, sort_keys: bool = True) -> str: """The JCS text of a JSON-like value. + With ``sort_keys=False`` object keys keep their insertion order, which is + ``JSON.stringify``'s output exactly: the form judge text uses, so a judge + reads the same text in every SDK. It is not JCS, so never hash it. + The value is first normalized through ``json.dumps(..., default=str)``, so a tuple is a list, a non-string key is the string ``json.dumps`` writes for it, and a value JSON cannot hold is its ``str()``. A non-finite number is written ``null``, as ``JSON.stringify`` writes it. """ - return _encode(json.loads(json.dumps(value, default=str))) + return _encode(json.loads(json.dumps(value, default=str)), sort_keys) -def _encode(value: Any) -> str: +def _encode(value: Any, sort_keys: bool = True) -> str: if value is None or value is True or value is False: return json.dumps(value) if isinstance(value, int): @@ -47,9 +51,13 @@ def _encode(value: Any) -> str: if isinstance(value, str): return _string(value) if isinstance(value, list): - return "[" + ",".join(_encode(item) for item in value) + "]" - keys = sorted(value, key=_utf16) - return "{" + ",".join(f"{_string(key)}:{_encode(value[key])}" for key in keys) + "}" + return "[" + ",".join(_encode(item, sort_keys) for item in value) + "]" + keys = sorted(value, key=_utf16) if sort_keys else list(value) + return ( + "{" + + ",".join(f"{_string(key)}:{_encode(value[key], sort_keys)}" for key in keys) + + "}" + ) def _number(value: float) -> str: diff --git a/packages/client/src/launchdarkly_ai_server/evaluations/runner.py b/packages/client/src/launchdarkly_ai_server/evaluations/runner.py index b12ee5cd..9a06890e 100644 --- a/packages/client/src/launchdarkly_ai_server/evaluations/runner.py +++ b/packages/client/src/launchdarkly_ai_server/evaluations/runner.py @@ -10,12 +10,14 @@ from collections.abc import Awaitable, Callable, Mapping, Sequence from dataclasses import dataclass from datetime import UTC, datetime +from math import isfinite from typing import Any, Literal, cast -from ..content import tool_result_string +from ..canonical_json import canonical_json from ..judge_scoring import ( FORMATTING_INSTRUCTIONS, build_message_history, + judge_text, numeric_score, parse_judge_response, typesafe_judge_entries, @@ -43,7 +45,6 @@ require_string, segment, ) -from .canonical_json import canonical_json from .criteria import Criterion, Judge, Scorer from .events import ( CriterionEventPayload, @@ -99,12 +100,12 @@ def _provides_for( def _describe_value(value: Any) -> str: """A value in an error message, as JSON rather than a Python repr, so it reads the same in every SDK. - A non-finite number is ``NaN`` / ``Infinity``, and a value JSON cannot hold its ``str()``. + Spelled as ``JSON.stringify`` spells it, except that a non-finite number is + ``NaN`` / ``Infinity``, and a value JSON cannot hold is its ``str()``. """ - try: - return json.dumps(value, ensure_ascii=False, default=str) - except (TypeError, ValueError): - return str(value) + if isinstance(value, float) and not isfinite(value): + return json.dumps(value) + return canonical_json(value, sort_keys=False) def describe_handlers(handlers: Sequence[EvalHandler]) -> str: @@ -832,7 +833,7 @@ def _judge_variables( variables.update( { "input": row_result.get("input") or "", - "response_to_evaluate": tool_result_string(output), + "response_to_evaluate": judge_text(output), "message_history": build_message_history( user_input=row_result.get("input"), trajectory=trajectory, diff --git a/packages/client/src/launchdarkly_ai_server/judge_scoring.py b/packages/client/src/launchdarkly_ai_server/judge_scoring.py index 6985afea..67d9113b 100644 --- a/packages/client/src/launchdarkly_ai_server/judge_scoring.py +++ b/packages/client/src/launchdarkly_ai_server/judge_scoring.py @@ -16,7 +16,7 @@ from math import isfinite from typing import Any -from .content import tool_result_string +from .canonical_json import canonical_json from .utils import parse_json_with_possible_fences logger = logging.getLogger(__name__) @@ -56,12 +56,26 @@ def build_message_history( every SDK. An output of ``0`` or ``False`` is an answer, and is shown. """ return "\n\n".join( - tool_result_string(part) + judge_text(part) for part in (user_input, trajectory, output, FORMATTING_INSTRUCTIONS) if not _is_empty_part(part) ) +def judge_text(value: Any) -> str: + """A value as a judge is shown it: a string as it is, ``None`` as ``""``, anything else as JSON. + + The JSON is ``JSON.stringify``'s spelling (no whitespace, keys in insertion + order, ``1`` for ``1.0``), not ``json.dumps``'s, so a rubric reads the same + text in every SDK. + """ + if isinstance(value, str): + return value + if value is None: + return "" + return canonical_json(value, sort_keys=False) + + def _is_empty_part(value: Any) -> bool: """``None``, ``""``, an empty list or an empty mapping, tested before rendering. @@ -159,7 +173,7 @@ def parse_judge_response(raw: Any) -> tuple[Any, str]: raise ValueError("Invalid JSON from judge") reasoning = next( ( - tool_result_string(value) + judge_text(value) for value in (parsed.get("reasoning"), parsed.get("reason")) if not _is_empty_part(value) ), diff --git a/packages/client/tests/test_canonical_json.py b/packages/client/tests/test_canonical_json.py index ce007d94..a7759479 100644 --- a/packages/client/tests/test_canonical_json.py +++ b/packages/client/tests/test_canonical_json.py @@ -7,7 +7,7 @@ import pytest -from launchdarkly_ai_server.evaluations.canonical_json import canonical_json +from launchdarkly_ai_server.canonical_json import canonical_json @pytest.mark.parametrize( @@ -112,3 +112,21 @@ def test_content_hash_matches_the_javascript_sdk( generated: dict[str, Any], digest: str ) -> None: assert hashlib.sha256(canonical_json(generated).encode()).hexdigest() == digest + + +@pytest.mark.parametrize( + ("value", "text"), + [ + (1.0, "1"), + ("high", '"high"'), + ({"b": 2.0, "a": [None]}, '{"b":2,"a":[null]}'), + (float("nan"), "NaN"), + (float("-inf"), "-Infinity"), + ], +) +def test_describe_value_is_json_stringify_text_except_non_finite( + value: Any, text: str +) -> None: + from launchdarkly_ai_server.evaluations.runner import _describe_value + + assert _describe_value(value) == text diff --git a/packages/client/tests/test_evaluations_run.py b/packages/client/tests/test_evaluations_run.py index 6a874eb3..78228c90 100644 --- a/packages/client/tests/test_evaluations_run.py +++ b/packages/client/tests/test_evaluations_run.py @@ -2343,7 +2343,7 @@ async def handler( criteria=[Judge(key="$ld:ai:judge:accuracy")], ) - rendered = '{"answer": "original", "tags": ["a"], "final": true}' + rendered = '{"answer":"original","tags":["a"],"final":true}' assert judged["response"] == rendered assert rendered in judged["history"] diff --git a/packages/client/tests/test_judge_scoring.py b/packages/client/tests/test_judge_scoring.py index cb3b990d..b59eacda 100644 --- a/packages/client/tests/test_judge_scoring.py +++ b/packages/client/tests/test_judge_scoring.py @@ -7,6 +7,7 @@ from launchdarkly_ai_server.judge_scoring import ( FORMATTING_INSTRUCTIONS, build_message_history, + judge_text, parse_judge_response, typesafe_judge_entries, ) @@ -41,7 +42,7 @@ def test_null_reasoning_becomes_empty_string_not_none_literal(self) -> None: def test_non_string_reasoning_is_json_not_a_repr(self) -> None: assert parse_judge_response({"score": 1, "reasoning": {"ok": True}}) == ( 1, - '{"ok": true}', + '{"ok":true}', ) assert parse_judge_response({"score": 1, "reasoning": 0}) == (1, "0") @@ -103,12 +104,12 @@ def test_message_history_renders_a_structured_output_as_json() -> None: user_input="q", output={"answer": "oui", "ok": True, "note": None} ) assert history == "\n\n".join( - ["q", '{"answer": "oui", "ok": true, "note": null}', FORMATTING_INSTRUCTIONS] + ["q", '{"answer":"oui","ok":true,"note":null}', FORMATTING_INSTRUCTIONS] ) def test_message_history_keeps_non_ascii() -> None: - assert '{"city": "Zürich"}' in build_message_history(output={"city": "Zürich"}) + assert '{"city":"Zürich"}' in build_message_history(output={"city": "Zürich"}) @pytest.mark.parametrize("empty", [None, "", {}, []]) @@ -125,3 +126,25 @@ def test_message_history_shows_a_falsy_answer(output: Any, text: str) -> None: assert build_message_history(user_input="q", output=output) == "\n\n".join( ["q", text, FORMATTING_INSTRUCTIONS] ) + + +# Each expected string is what JavaScript's JSON.stringify writes for the same value. +@pytest.mark.parametrize( + ("value", "text"), + [ + ("as is", "as is"), + (None, ""), + (0, "0"), + (False, "false"), + (1.0, "1"), + (-2.0, "-2"), + (0.5, "0.5"), + (1e21, "1e+21"), + ([1.0, "a", None], '[1,"a",null]'), + ({"b": 1.0, "a": [True, {}]}, '{"b":1,"a":[true,{}]}'), + ({"city": "Zürich"}, '{"city":"Zürich"}'), + ({"n": float("nan")}, '{"n":null}'), + ], +) +def test_judge_text_is_json_stringify_text(value: Any, text: str) -> None: + assert judge_text(value) == text From eaa40282c3ea203adf33db8b121476e78fc5cc10 Mon Sep 17 00:00:00 2001 From: Jeff Dupont Date: Fri, 23 Oct 2026 12:36:20 -0700 Subject: [PATCH 5/5] fix(client): give online judges the same response text as offline judges config().invoke() rendered a structured response for its judges with json.dumps (spaces, 1.0, ASCII escapes); it now uses judge_text, as the offline runner does. Co-Authored-By: Claude Opus 5.5 --- .../src/launchdarkly_ai_server/client.py | 9 +++----- packages/client/tests/test_client.py | 23 +++++++++++++++++++ 2 files changed, 26 insertions(+), 6 deletions(-) diff --git a/packages/client/src/launchdarkly_ai_server/client.py b/packages/client/src/launchdarkly_ai_server/client.py index da73be45..be124d87 100644 --- a/packages/client/src/launchdarkly_ai_server/client.py +++ b/packages/client/src/launchdarkly_ai_server/client.py @@ -1,12 +1,12 @@ from __future__ import annotations -import json from collections.abc import AsyncGenerator, Callable from contextlib import aclosing from typing import Any from .agui import StableProjector, judge_completed_stream from .conversation import bind_conversation_id +from .judge_scoring import judge_text from .judges import _build_judge_tasks, run_judges from .lifecycle import extract_variation from .registry import resolve_handlers, resolve_tools @@ -120,11 +120,8 @@ async def _invoke( config.get("outputFormat") if isinstance(config, dict) else None, ) - llm_str = ( - parsed_response - if isinstance(parsed_response, str) - else json.dumps(parsed_response) - ) + # The same text an offline judge reads, so a rubric sees one spelling. + llm_str = judge_text(parsed_response) usage_obj = to_usage_dict(usage) diff --git a/packages/client/tests/test_client.py b/packages/client/tests/test_client.py index 24833549..83fa514a 100644 --- a/packages/client/tests/test_client.py +++ b/packages/client/tests/test_client.py @@ -377,6 +377,29 @@ async def obj_fn( result = await m.invoke("q", CONTEXT) assert result.response == {"already": "parsed"} + async def test_online_judge_reads_a_structured_response_as_json_stringify_text( + self, mock_ld_client: MagicMock, monkeypatch: pytest.MonkeyPatch + ) -> None: + raw = { + "model": {"name": "gpt-4"}, + "provider": {"name": "TestProvider"}, + "instructions": "Reply in JSON", + "_ldMeta": { + "enabled": True, + "variationKey": "v1", + "version": 1, + "mode": "messages", + }, + "outputFormat": {"type": "object"}, + } + mock_ld_client.variation = AsyncMock(return_value=raw) + judges = AsyncMock(return_value={}) + monkeypatch.setattr("launchdarkly_ai_server.client.run_judges", judges) + handler = _make_handler(response='{"score": 1.0, "city": "Zürich"}') + await config(key="flag", handler=handler).invoke("q", CONTEXT) + # The same text an offline judge reads: no spaces, 1 not 1.0, no escapes. + assert judges.await_args.kwargs["llm_response"] == '{"score":1,"city":"Zürich"}' + async def test_no_parsing_when_output_format_absent( self, mock_ld_client: MagicMock ) -> None: