fix(evaluations)!: reject boolean scorer scores

packages/client/README.md

@@ -125,8 +125,8 @@ from launchdarkly_ai_server.experimental.evaluations import (
 )
 
 
-def mentions_policy(row: DatasetRow, output: str | None) -> bool:
-    return "refund policy" in (output or "").lower()
+def mentions_policy(row: DatasetRow, output: str | None) -> float:
+    return 1.0 if "refund policy" in (output or "").lower() else 0.0
 
 
 result = await init_evaluations(project_key="my-project").run(
@@ -143,7 +143,7 @@ result = await init_evaluations(project_key="my-project").run(
 )
 ```
 
-`Scorer.fn` receives the `DatasetRow` the output was generated from plus the generated output, may be sync or async, and must return a bool or a number from 0 to 1; booleans become 1.0 or 0.0. `Judge.threshold` defaults to 0.5 and `Scorer.threshold` to 1.0 — a perfect score, which is what a boolean scorer wants — and both accept an optional `pass_rate_threshold`. Judge keys and scorer names share one `criterionType` namespace and must be unique within a run, case-insensitively, because that name is part of each result's deterministic event identity. `Judge.ground_truth_context` overrides what the judge is graded against when the dataset row's expected output is not it.
+`Scorer.fn` receives the `DatasetRow` the output was generated from plus the generated output, may be sync or async, and must return a **finite number from 0 to 1** — that is the whole contract, and nothing is coerced on your behalf. A scorer answering a yes/no question returns `1.0` or `0.0` itself; a `bool`, a `NaN`, or an out-of-range number is an `invalid_score` result. Returning one score type keeps a threshold comparison meaning the same thing for binary and graded scorers, and keeps a scorer that accidentally returns a non-score from passing silently — every non-empty value is truthy, so `return "high"` would otherwise score 1.0. `Judge.threshold` defaults to 0.5 and `Scorer.threshold` to 1.0 — a perfect score, which is what a yes/no scorer wants — and both accept an optional `pass_rate_threshold`. Judge keys and scorer names share one `criterionType` namespace and must be unique within a run, case-insensitively, because that name is part of each result's deterministic event identity. `Judge.ground_truth_context` overrides what the judge is graded against when the dataset row's expected output is not it.
 
 **The SDK reports scores and never rules on them.** LaunchDarkly derives each row's verdict at ingest by comparing the score against the criterion's stored threshold and success direction, so pass/fail policy is one server-side implementation that applies to every SDK version and to runs already recorded. A judge's direction lives on its AI Config and is injected server-side, keeping the one input a verdict turns on server-attested; a `Scorer` has no LaunchDarkly-side config to read, so it declares its own `success_direction` (default `"higher_is_better"` — set `"lower_is_better"` for a scorer that counts something unwanted, like a regex hit count).
 

packages/client/src/launchdarkly_ai_server/evaluations/criteria.py

@@ -7,7 +7,7 @@
 
 from .types import DatasetRow
 
-type ScorerFn = Callable[[DatasetRow, Any], float | bool | Awaitable[float | bool]]
+type ScorerFn = Callable[[DatasetRow, Any], float | Awaitable[float]]
 
 type SuccessDirection = Literal["higher_is_better", "lower_is_better"]
 
@@ -70,12 +70,19 @@ class Scorer:
     ``fn`` may be sync or async and receives ``(row, output)``, where ``row``
     is the :class:`~launchdarkly_ai_server.evaluations.types.DatasetRow` the
     output was generated from and ``output`` is the generated output. It must
-    return a boolean or a numeric score from 0 to 1. Boolean results are
-    converted to 1.0 or 0.0 before being emitted as evaluation events.
+    return a **finite number from 0 to 1** -- that is the whole contract.
+    Nothing is coerced on the caller's behalf, so a scorer answering a yes/no
+    question returns ``1.0`` or ``0.0`` itself; a ``bool`` is an
+    ``invalid_score`` result, as is a non-finite or out-of-range number.
+
+    Returning one score type keeps the ``threshold`` comparison below meaning
+    the same thing for binary and graded scorers. It also keeps a scorer that
+    accidentally returns a non-score from passing silently: every non-empty
+    value is truthy, so ``return "high"`` scored 1.0 under the old coercion.
 
     ``threshold`` defaults to 1.0: a row passes only on a perfect score, which
-    matches the common case of boolean scorers. Pass a lower threshold for
-    graded numeric scorers.
+    is what a scorer answering a yes/no question wants. Pass a lower threshold
+    for a graded scorer.
 
     ``success_direction`` says which way the score points, and defaults to
     higher-is-better. Unlike a judge, a scorer has no LaunchDarkly-side config

packages/client/src/launchdarkly_ai_server/evaluations/runner.py

@@ -867,6 +867,7 @@ async def _run_scorer_for_result(
         self,
         row: Mapping[str, Any],
         scorer: Scorer,
+        warned_scorers: set[str],
     ) -> dict[str, Any]:
         started = datetime.now(UTC)
         started_clock = time.perf_counter()
@@ -898,25 +899,26 @@ async def _run_scorer_for_result(
             return self._criterion_error_result(
                 base, started_clock, "scorer_raised", f"scorer fn raised: {error}"
             )
-        if isinstance(score_value, bool):
-            score: float = 1.0 if score_value else 0.0
-        else:
-            maybe_score = numeric_score(score_value)
-            if maybe_score is None:
-                return self._criterion_error_result(
-                    base,
-                    started_clock,
-                    "invalid_score",
-                    "scorer fn must return a bool or a finite number, "
-                    f"got {_describe_value(score_value)}",
+        # A bool is rejected rather than read as 1.0/0.0. numeric_score already
+        # excludes it, so the whole return contract is "a finite number in
+        # 0-1": a scorer answering a yes/no question returns 1.0 or 0.0 itself.
+        # Coercing on the caller's behalf sent two score types to ingest and
+        # made the threshold comparison mean different things for binary and
+        # graded scorers -- and silently scored `return "high"`-style bugs as a
+        # pass, since every non-empty value is truthy.
+        score = numeric_score(score_value)
+        if score is None or score < 0 or score > 1:
+            message = (
+                "scorer fn must return a finite number between 0 and 1, "
+                f"got {_describe_value(score_value)}"
+            )
+            if scorer.name not in warned_scorers:
+                warned_scorers.add(scorer.name)
+                logger.warning(
+                    "scorer %r returned an invalid score: %s", scorer.name, message
                 )
-            score = maybe_score
-        if score < 0 or score > 1:
             return self._criterion_error_result(
-                base,
-                started_clock,
-                "invalid_score",
-                f"scorer fn score must be between 0 and 1, got {_describe_value(score_value)}",
+                base, started_clock, "invalid_score", message
             )
         completed = datetime.now(UTC)
         return {
@@ -1030,14 +1032,17 @@ async def _run_criteria_for_results(
     ) -> list[dict[str, Any]]:
         """Run every (row, criterion) pair, bounded by the run's concurrency."""
         controller = ConcurrencyController(concurrency)
+        warned_scorers: set[str] = set()
 
         async def run_one(
             row: Mapping[str, Any], criterion: Criterion
         ) -> dict[str, Any]:
             await controller.acquire()
             try:
                 if isinstance(criterion, Scorer):
-                    return await self._run_scorer_for_result(row, criterion)
+                    return await self._run_scorer_for_result(
+                        row, criterion, warned_scorers
+                    )
                 return await self._run_ld_judge_for_result(
                     row,
                     criterion,

packages/client/tests/test_evaluations_run.py

@@ -13,6 +13,7 @@
 
 from launchdarkly_ai_server import NativeTool, create_handler
 from launchdarkly_ai_server.evaluations.api import HttpResponse
+from launchdarkly_ai_server.evaluations.runner import _describe_value
 from launchdarkly_ai_server.experimental.evaluations import (
     AIConfig,
     DatasetRow,
@@ -2023,11 +2024,12 @@ async def handler(*args: object) -> dict[str, Any]:
             "usage": {"input_tokens": 10, "output_tokens": 4},
         }
 
-    def check_refund(row: DatasetRow, output: Any) -> bool:
+    def check_refund(row: DatasetRow, output: Any) -> float:
         assert row.row_index == 42
         assert row.input == "Ticket A"
         assert output == "refund exists"
-        return "refund" in str(output)
+        # A yes/no scorer returns the score itself: the SDK coerces nothing.
+        return 1.0 if "refund" in str(output) else 0.0
 
     result = await evals.run(
         key="support-qa",
@@ -2366,7 +2368,7 @@ async def handler(*args: object) -> dict[str, Any]:
             generation={"provider": "OpenAI", "model": "gpt-4o", "mode": "completion"},
             criteria=[
                 Judge(key="accuracy"),
-                Scorer(name="accuracy", fn=lambda row, output: True),
+                Scorer(name="accuracy", fn=lambda row, output: 1.0),
             ],
         )
 
@@ -2395,7 +2397,7 @@ async def handler(*args: object) -> dict[str, Any]:
             generation={"provider": "OpenAI", "model": "gpt-4o", "mode": "completion"},
             criteria=[
                 Judge(key="Accuracy"),
-                Scorer(name="accuracy", fn=lambda row, output: True),
+                Scorer(name="accuracy", fn=lambda row, output: 1.0),
             ],
         )
 
@@ -2485,7 +2487,7 @@ async def handler(
             generation={"provider": "OpenAI", "model": "gpt-4o", "mode": "completion"},
             criteria=[
                 Judge(key="$ld:ai:judge:accuracy"),
-                Scorer(name="nonempty", fn=lambda row, output: bool(output)),
+                Scorer(name="nonempty", fn=lambda row, output: 1.0 if output else 0.0),
             ],
         )
 
@@ -2510,7 +2512,7 @@ def test_criteria_reject_thresholds_outside_zero_to_one(
     with pytest.raises(ValueError, match=f"{field} must be a number between 0 and 1"):
         Judge(key="$ld:ai:judge:accuracy", **{field: value})
     with pytest.raises(ValueError, match=f"{field} must be a number between 0 and 1"):
-        Scorer(name="nonempty", fn=lambda row, output: True, **{field: value})
+        Scorer(name="nonempty", fn=lambda row, output: 1.0, **{field: value})
 
 
 def judge_variation(
@@ -4894,3 +4896,180 @@ async def test_a_completion_or_list_tagged_handler_is_selected(
     )
 
     assert stub.await_count == 1
+
+
+def scorer_run_transport() -> SequencedTransport:
+    """Transport for a one-row scorer-only run."""
+    return SequencedTransport(
+        [
+            response(200, {"id": "dataset-id", "name": "golden"}),
+            response(
+                200,
+                dataset_page([{"rowIndex": 7, "input": "Question"}], total=1),
+            ),
+            response(201, {"id": "evaluation-id", "name": "support-qa", "version": 3}),
+            response(
+                201,
+                {"id": "run-id", "evaluationId": "evaluation-id", "state": "PENDING"},
+            ),
+            response(
+                200,
+                {"statusCounts": {"total": 1, "passed": 1, "error": 0, "pending": 0}},
+            ),
+        ]
+    )
+
+
+@pytest.mark.parametrize(
+    "returned",
+    [
+        pytest.param(True, id="true"),
+        pytest.param(False, id="false"),
+        pytest.param(float("nan"), id="nan"),
+        pytest.param(float("inf"), id="inf"),
+        pytest.param(3, id="above-range"),
+        pytest.param(-1, id="below-range"),
+        pytest.param("high", id="string"),
+        pytest.param(None, id="none"),
+    ],
+)
+@pytest.mark.asyncio
+async def test_scorer_must_return_a_finite_number_in_range(
+    stub_sdk_client: MagicMock,
+    returned: Any,
+    caplog: pytest.LogCaptureFixture,
+) -> None:
+    """A bool is an invalid score, not a shortcut for 1.0/0.0.
+
+    Coercing booleans sent two score types to ingest and made the threshold
+    comparison mean different things for binary and graded scorers. It also hid
+    bugs: every non-empty value is truthy, so a scorer that returned "high"
+    scored a pass.
+    """
+    transport = scorer_run_transport()
+    evals = init_evaluations(
+        project_key="proj", api_key="token", sdk_key="sdk-key", transport=transport
+    )
+
+    async def handler(*args: object) -> dict[str, Any]:
+        return {"output": "generated"}
+
+    with caplog.at_level("WARNING"):
+        result = await evals.run(
+            key="support-qa",
+            dataset="golden",
+            handler=tagged(handler),
+            generation=COMPLETION,
+            criteria=[Scorer(name="binary", fn=lambda row, output: returned)],
+        )
+
+    events = [call.args[2] for call in stub_sdk_client.track.call_args_list]
+    scorer_event = next(event for event in events if event.get("kind") == "scorer")
+    assert scorer_event["status"] == "ERROR"
+    assert scorer_event["error"]["code"] == "invalid_score"
+    # The message names the offending value, so a caller can see what came back.
+    # It is rendered the way judges see values (true, null, "high", NaN).
+    shown = _describe_value(returned)
+    assert shown in scorer_event["error"]["message"]
+    assert scorer_event["errorMessage"] == scorer_event["error"]["message"]
+    assert "score" not in scorer_event
+    # The caller's own logs name the scorer and the value, once per scorer.
+    warnings = [r for r in caplog.records if "invalid score" in r.getMessage()]
+    assert len(warnings) == 1
+    assert "'binary'" in warnings[0].getMessage()
+    assert shown in warnings[0].getMessage()
+    # A rejected score is a per-criterion ERROR, never a raised exception: the
+    # row's generation has already been paid for.
+    assert result.run_id == "run-id"
+    stub_sdk_client.flush.assert_awaited()
+
+
+@pytest.mark.parametrize(
+    ("returned", "expected"),
+    [(1.0, 1.0), (0.0, 0.0), (0, 0.0), (1, 1.0), (0.25, 0.25)],
+)
+@pytest.mark.asyncio
+async def test_scorer_accepts_the_range_boundaries_and_integers(
+    stub_sdk_client: MagicMock,
+    returned: Any,
+    expected: float,
+) -> None:
+    """0 and 1 stay valid, as ints too -- only bool is excluded."""
+    transport = scorer_run_transport()
+    evals = init_evaluations(
+        project_key="proj", api_key="token", sdk_key="sdk-key", transport=transport
+    )
+
+    async def handler(*args: object) -> dict[str, Any]:
+        return {"output": "generated"}
+
+    await evals.run(
+        key="support-qa",
+        dataset="golden",
+        handler=tagged(handler),
+        generation=COMPLETION,
+        criteria=[Scorer(name="graded", fn=lambda row, output: returned)],
+    )
+
+    events = [call.args[2] for call in stub_sdk_client.track.call_args_list]
+    scorer_event = next(event for event in events if event.get("kind") == "scorer")
+    assert scorer_event["status"] == "COMPLETE"
+    assert scorer_event["score"] == expected
+
+
+@pytest.mark.asyncio
+async def test_async_scorer_score_is_awaited_before_validation(
+    stub_sdk_client: MagicMock,
+) -> None:
+    transport = scorer_run_transport()
+    evals = init_evaluations(
+        project_key="proj", api_key="token", sdk_key="sdk-key", transport=transport
+    )
+
+    async def handler(*args: object) -> dict[str, Any]:
+        return {"output": "generated"}
+
+    async def graded(row: DatasetRow, output: Any) -> float:
+        return 0.5
+
+    await evals.run(
+        key="support-qa",
+        dataset="golden",
+        handler=tagged(handler),
+        generation=COMPLETION,
+        criteria=[Scorer(name="graded", fn=graded)],
+    )
+
+    events = [call.args[2] for call in stub_sdk_client.track.call_args_list]
+    scorer_event = next(event for event in events if event.get("kind") == "scorer")
+    assert scorer_event["status"] == "COMPLETE"
+    assert scorer_event["score"] == 0.5
+
+
+@pytest.mark.asyncio
+async def test_invalid_score_warning_repeats_on_each_run(
+    stub_sdk_client: MagicMock,
+    caplog: pytest.LogCaptureFixture,
+) -> None:
+    """One module logs a warning in each run for a scorer that returns an invalid score."""
+    transport = scorer_run_transport()
+    transport.responses = transport.responses * 2
+    evals = init_evaluations(
+        project_key="proj", api_key="token", sdk_key="sdk-key", transport=transport
+    )
+
+    async def handler(*args: object) -> dict[str, Any]:
+        return {"output": "generated"}
+
+    with caplog.at_level("WARNING"):
+        for _ in range(2):
+            await evals.run(
+                key="support-qa",
+                dataset="golden",
+                handler=tagged(handler),
+                generation=COMPLETION,
+                criteria=[Scorer(name="quality", fn=lambda row, output: "high")],
+            )
+
+    warnings = [r for r in caplog.records if "invalid score" in r.getMessage()]
+    assert len(warnings) == 2