From 312578099cb1d445a8527e2a7551fcbb64c035e6 Mon Sep 17 00:00:00 2001 From: Birfy Date: Mon, 19 Oct 2026 20:00:57 +0800 Subject: [PATCH 1/2] [Fix] Unify Choice/Noul confidence as max_k p_k / max(p, 1-p) Every backend used to pass its own confidence through validation, so Laya's normalised entropy, Jev's margin and Cua's max p were not comparable. Compute contract confidence in validation.py from the validated probabilities; keep each backend's number as backend_confidence on the answer, in Decision.raw, and on tool/browser ticks. Closes #89. --- CHANGELOG.md | 6 +++ docs/decision-models.md | 15 ++++-- s1a/browser/action_space.py | 2 + s1a/browser/decision_model.py | 5 ++ s1a/decision_models/__init__.py | 11 ++++- s1a/decision_models/baselines.py | 2 +- s1a/decision_models/types.py | 21 +++++++-- s1a/decision_models/validation.py | 61 ++++++++++++++++++++----- s1a/mcp_server.py | 2 +- s1a/probe.py | 2 +- s1a/tool/models.py | 35 +++++++------- tests/decision_model_contract.py | 23 ++++++++++ tests/test_cli.py | 2 +- tests/test_decision_models_base.py | 4 +- tests/test_decision_models_baselines.py | 8 ++-- tests/test_decision_models_laya.py | 4 +- tests/test_decision_models_omnijev.py | 5 +- tests/test_decision_models_types.py | 38 +++++++++++---- tests/test_tool_models.py | 2 +- 19 files changed, 188 insertions(+), 60 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index fa5e7ff..a4d572c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -54,6 +54,12 @@ The format follows [Keep a Changelog](https://keepachangelog.com/en/1.1.0/); ver ### Changed +- `Choice.confidence` / `Noul.confidence` now mean the same thing on every backend: `max_k p_k` for a choice and + `max(p, 1 - p)` for a noul, both computed in `validation.py` from the validated probabilities. Historical `conf=` + values from `laya` / `laya-served` (normalised entropy), `jev` / `clm` (service-reported) and `omnijev` (Jev margin) + are not comparable with new ones. Each backend's own number is kept as `backend_confidence` on the answer, in + `Decision.raw["backend_confidence"]`, and on tool/browser tick records when present. See #89. + - Review guidance (`self-review`, `review-pr`) now treats merge readiness as a gate: authors confirm the pull request reports mergeable with CI green on the recorded head before requesting review, and reviewers do not approve a conflicting branch or a run pinned to an older commit. diff --git a/docs/decision-models.md b/docs/decision-models.md index 2c2af5d..752ab52 100644 --- a/docs/decision-models.md +++ b/docs/decision-models.md @@ -22,7 +22,13 @@ Input: an `Observation(state, images=())` (a JSON object or plain text) and type `ChoiceQuestion(options, goal=, rules=, operation=)` picks one key among `options` (a string per key, or a dict for the browser's element rows); `NoulQuestion(question, criteria=)` asks whether a statement holds. Output: a `Decision(answers, latency_ms, usage, model, raw)` with one `Choice(key, probabilities, confidence)` or -`Noul(p, confidence)` per question name. +`Noul(p, confidence)` per question name. **`confidence` always means the same thing on every backend:** for a +choice it is `max_k p_k` (the peak probability); for a noul it is `max(p, 1 - p)`. Validation computes both from +the validated probabilities in `s1a/decision_models/validation.py`, so Laya's normalised entropy, Jev's normalised +margin, and OmniJev's recompute never become the reported `confidence`. Each backend's own number is kept as +`backend_confidence` on the answer and under `Decision.raw["backend_confidence"]` (a map of question name → number) +when the backend sent one in [0, 1]. These numbers are **not** calibrated unless a backend says so; do not reuse a +threshold across backends. `Usage.known` distinguishes reported zero tokens from missing or malformed provider usage. A local backend can return `Usage()` for a known zero; `Usage.from_payload` preserves tolerant counts but marks incomplete usage unknown. @@ -41,7 +47,8 @@ decision.choice("pick").key, decision.latency_ms, decision.usage.input_tokens `decide_many` is the one place validation happens. It refuses, with `MODEL_SERVICE_CONFIG_ERROR`, an empty question set, an image on a text-only model and a question type the backend lacks. It checks every answer: a choice must name an offered key, its probabilities must cover exactly the offered keys, lie in [0, 1], sum to 1 -within 0.02 and peak at the key; a noul probability lies in [0, 1] and its confidence defaults to `max(p, 1 - p)`. +within 0.02 and peak at the key; a noul probability lies in [0, 1]. After those checks, `Choice.confidence` is set +to `max_k p_k` and `Noul.confidence` to `max(p, 1 - p)` regardless of what the backend put in its `confidence` field. An unusable answer is re-asked with the same request up to `attempts` times (never on a deterministic backend, never after a transport error) and then raises `MODEL_CALL_FAILED`. `decide`, `choose` and `ask` are one-question shorthands; `warm()` and `close()` open and release the backend. @@ -56,7 +63,7 @@ shorthands; `warm()` and `close()` open and release the backend. | `laya-served` | `ServedLayaModel(client, model=, max_len=)` | `laya-served` | one `POST /v1/systemone` per request to `LAYA_SERVED_URL`, body built with `laya_question()` as for `laya`; one retry on a dropped connection, 502 or 504, and after `Retry-After` on 503, all within `LAYA_SERVED_TIMEOUT_S`; the same window check as `laya`; `model` is the served checkpoint and revision, and `raw` keeps `served_by`, `url`, `request_id` and `server_timing` (see [served-laya.md](served-laya.md)) | | `cua` | `CuaS1Model(scorer, collator, model=, context_bytes=, option_bytes=)` | `cua` | Cua-S1 Nano, one `score_elements` pass per request on a thread; choice questions only, text only, deterministic; the context is header, state and rules; the checkpoint reads its first 256 bytes, and the first overflowing request logs one warning; `from_env` reads `CUA_S1_*` (see [configuration.md](configuration.md)) | | `omnijev` | `OmniJevModel(agent, model=)` | `omnijev` | OmniJev (a Qwen3.5 vision-language decision model), one `MSO1.system_one` call per request on a thread; choice and noul questions over the observation's screenshot, deterministic; the text state, goal and rules go in front of each question (a browser state puts its recent actions first and caps the page text at 2,000 characters, so a long page cannot push the history out of the 6,000-character context), a browser element row becomes its label and value; a choice's probabilities, which OmniJev leaves `abstain` out of, are renormalised over the offered keys; an observation without an image is `MODEL_CALL_FAILED`; `from_env` reads `OMNIJEV_*` (see [configuration.md](configuration.md)); in process only: system1-omni has no OmniJev vision worker yet (its supported models are Laya, Open-Jev text and Cua-S1 screenshots), so there is no served `omnijev` path | -| `random` | `RandomModel(seed)` | `random` | uniform over the offered keys, confidence 0, one seeded stream per episode; choice questions only | +| `random` | `RandomModel(seed)` | `random` | uniform over the offered keys (contract confidence = `1/|options|`), one seeded stream per episode; choice questions only | | `rule` | `RuleModel(name, rule)` | the rule's name | one-hot, confidence 1; a key outside the menu raises `RuntimeError`, a bug in the rule | `name` lands in every tick's `source` and in `Episode.policy`; the eval table's columns take their labels from it. @@ -80,7 +87,7 @@ parameters) answer the same `choice` question in process. reads the validated decision onto a candidate. Ticks keep `decision_ms` and `input_tokens` for the artifact readers. - Rails (`DecisionModelRail`): one `check` question, noul or choice, banded by the spec's thresholds. - `decide`, `probe` and the MCP server: one `pick` question; the printed dict stays `choice`, `probabilities`, - `confidence`, `ms`. + `confidence` (= `max_k p_k`), `ms`, and `backend_confidence` when the backend sent its own number. Three oracle tests in `tests/test_decision_models_jev.py` pin the tool, rail and browser bodies to the pre-layer JSON. diff --git a/s1a/browser/action_space.py b/s1a/browser/action_space.py index b1a16fb..507896f 100644 --- a/s1a/browser/action_space.py +++ b/s1a/browser/action_space.py @@ -69,6 +69,7 @@ class Move: latency_ms: int usage: Usage model: str + backend_confidence: float | None = None def submittable(item: dict[str, Any]) -> bool: @@ -242,4 +243,5 @@ def interpret(decision: Decision, space: ActionSpace) -> Move: latency_ms=decision.latency_ms, usage=decision.usage, model=decision.model, + backend_confidence=operation.backend_confidence, ) diff --git a/s1a/browser/decision_model.py b/s1a/browser/decision_model.py index b5d785d..b924fe2 100644 --- a/s1a/browser/decision_model.py +++ b/s1a/browser/decision_model.py @@ -344,6 +344,11 @@ async def _decide_message(self, messages: Any) -> AssistantMessage: "operation": move.operation, "target": move.candidate.item.get("label") if move.candidate else None, "confidence": round(move.confidence, 3), + **( + {"backend_confidence": round(move.backend_confidence, 3)} + if move.backend_confidence is not None + else {} + ), "elements": len(space.elements), "elapsed_ms": round((time.perf_counter() - run.started_at) * 1000), "probabilities": probabilities, # the replay's bars: the settled head's top keys diff --git a/s1a/decision_models/__init__.py b/s1a/decision_models/__init__.py index 0142db5..7af66ad 100644 --- a/s1a/decision_models/__init__.py +++ b/s1a/decision_models/__init__.py @@ -29,7 +29,14 @@ Reply, Usage, ) -from s1a.decision_models.validation import choice_faults, validate_answers, validate_choice, validate_noul +from s1a.decision_models.validation import ( + choice_confidence, + choice_faults, + noul_confidence, + validate_answers, + validate_choice, + validate_noul, +) __all__ = [ "DECISION_MODEL_NAMES", @@ -57,7 +64,9 @@ "RuleModel", "Usage", "build_model", + "choice_confidence", "choice_faults", + "noul_confidence", "jev_question", "validate_answers", "validate_choice", diff --git a/s1a/decision_models/baselines.py b/s1a/decision_models/baselines.py index b823179..38455a9 100644 --- a/s1a/decision_models/baselines.py +++ b/s1a/decision_models/baselines.py @@ -32,7 +32,7 @@ async def _decide(self, observation: Observation, questions: dict[str, Question] for name, question in questions.items(): assert isinstance(question, ChoiceQuestion) ids = question.ids - # the entropy of a uniform pick is total, so its confidence is none + # backend sends 0; validation replaces Choice.confidence with max p (= 1/|ids| for uniform) answers[name] = _choice(self._rng.choice(ids), {key: 1.0 / len(ids) for key in ids}, 0.0) return Reply(answers=answers, latency_ms=0, model=self.model) diff --git a/s1a/decision_models/types.py b/s1a/decision_models/types.py index ad33a7f..d702103 100644 --- a/s1a/decision_models/types.py +++ b/s1a/decision_models/types.py @@ -94,30 +94,41 @@ def type(self) -> QuestionType: class Choice: """A validated choice answer: the key, a probability per key (sum 1 within 0.02, peak at the key), a confidence. - Only ``validation.validate_choice`` builds one from a backend's answer. + ``confidence`` is always ``max_k p_k`` (set by ``validation.validate_choice``). ``backend_confidence`` is the + number the backend itself reported, when it sent one in [0, 1]; it is a different scale for Laya (normalised + entropy) and Jev (normalised margin) and must not be compared across backends. """ key: str probabilities: dict[str, float] confidence: float + backend_confidence: float | None = None def as_dict(self) -> Json: - """The CLI and MCP output shape.""" - return {"choice": self.key, "probabilities": dict(self.probabilities), "confidence": self.confidence} + """The CLI and MCP output shape; includes ``backend_confidence`` only when the backend sent one.""" + out: Json = {"choice": self.key, "probabilities": dict(self.probabilities), "confidence": self.confidence} + if self.backend_confidence is not None: + out["backend_confidence"] = self.backend_confidence + return out @dataclass(frozen=True) class Noul: """A validated noul answer: the probability the statement holds and a confidence. - Only ``validation.validate_noul`` builds one from a backend's answer; a missing confidence becomes max(p, 1-p). + ``confidence`` is always ``max(p, 1 - p)`` (set by ``validation.validate_noul``). ``backend_confidence`` keeps + the backend's own number when it sent one in [0, 1]. """ p: float confidence: float + backend_confidence: float | None = None def as_dict(self) -> Json: - return {"noul": self.p, "confidence": self.confidence} + out: Json = {"noul": self.p, "confidence": self.confidence} + if self.backend_confidence is not None: + out["backend_confidence"] = self.backend_confidence + return out Answer = Union[Choice, Noul] diff --git a/s1a/decision_models/validation.py b/s1a/decision_models/validation.py index 0504fb4..64c007d 100644 --- a/s1a/decision_models/validation.py +++ b/s1a/decision_models/validation.py @@ -2,6 +2,9 @@ """The invariants every answer must satisfy, checked on the backend's dicts; a breach is ``MODEL_CALL_FAILED``. ``validate_choice`` and ``validate_noul`` are the only constructors of ``Choice`` and ``Noul`` from a backend's answer. +``Choice.confidence`` is always ``max_k p_k`` and ``Noul.confidence`` is always ``max(p, 1 - p)``, computed here from +the validated probabilities so every backend reports the same scale. A backend's own number is kept as +``backend_confidence`` on the answer and under ``Decision.raw["backend_confidence"]``. """ from __future__ import annotations @@ -29,18 +32,27 @@ def unit_interval(value: object) -> TypeGuard[float]: return isinstance(value, (int, float)) and not isinstance(value, bool) and math.isfinite(value) and 0 <= value <= 1 +def choice_confidence(probabilities: dict[str, float]) -> float: + """The contract confidence for a choice: the peak probability (``max_k p_k``).""" + return float(max(probabilities.values())) + + +def noul_confidence(p: float) -> float: + """The contract confidence for a noul: how far ``p`` sits from a coin flip (``max(p, 1 - p)``).""" + return max(p, 1.0 - p) + + def choice_faults(fields: Json, ids: list[str]) -> list[str]: """Why ``fields`` is not a choice answer over ``ids``: empty when it is one. ``choice`` is one of ``ids``; ``probabilities`` covers exactly ``ids`` with values in [0, 1] summing to 1 (within - 0.02) and peaks at the chosen key (ties allowed); ``confidence`` lies in [0, 1]. + 0.02) and peaks at the chosen key (ties allowed). Backend ``confidence`` is not checked here: validation + recomputes it as ``max_k p_k``. """ faults: list[str] = [] - key, distribution, confidence = fields.get("choice"), fields.get("probabilities"), fields.get("confidence") + key, distribution = fields.get("choice"), fields.get("probabilities") if not isinstance(key, str) or key not in ids: faults.append(f"choice {key!r} is not one of {ids}") - if not unit_interval(confidence): - faults.append(f"confidence {confidence!r} is not in [0, 1]") if not isinstance(distribution, dict): return faults + ["probabilities is not an object"] if set(distribution) != set(ids): @@ -55,26 +67,41 @@ def choice_faults(fields: Json, ids: list[str]) -> list[str]: def validate_choice(fields: Json, ids: list[str]) -> Choice: - """The typed choice with its numbers normalised, or ``MODEL_CALL_FAILED`` naming every fault.""" + """The typed choice with contract confidence ``max_k p_k``, or ``MODEL_CALL_FAILED`` naming every fault. + + When the backend sent a finite confidence in [0, 1], it is kept as ``backend_confidence`` (Laya's normalised + entropy, Jev's margin, OmniJev's recompute, and so on); it never becomes ``Choice.confidence``. + """ faults = choice_faults(fields, ids) if faults: raise build_error( StatusCode.MODEL_CALL_FAILED, error_msg="decisions answer is not an offered choice: " + "; ".join(faults) ) + probabilities = {str(key): float(weight) for key, weight in fields["probabilities"].items()} + native = fields.get("confidence") return Choice( key=str(fields["choice"]), - probabilities={str(key): float(weight) for key, weight in fields["probabilities"].items()}, - confidence=float(fields["confidence"]), + probabilities=probabilities, + confidence=choice_confidence(probabilities), + backend_confidence=float(native) if unit_interval(native) else None, ) def validate_noul(fields: Json) -> Noul: - """The typed noul with ``p`` in the unit interval and a confidence, ``max(p, 1 - p)`` when the backend gave none.""" - p, confidence = fields.get("noul"), fields.get("confidence") + """The typed noul with contract confidence ``max(p, 1 - p)``. + + A backend-supplied confidence in [0, 1] is kept as ``backend_confidence``; any other value is dropped. The + reported ``Noul.confidence`` is always ``max(p, 1 - p)``. + """ + p, native = fields.get("noul"), fields.get("confidence") if not unit_interval(p): raise build_error(StatusCode.MODEL_CALL_FAILED, error_msg="decisions answer is not a noul probability") p = float(p) - return Noul(p=p, confidence=float(confidence) if unit_interval(confidence) else max(p, 1.0 - p)) + return Noul( + p=p, + confidence=noul_confidence(p), + backend_confidence=float(native) if unit_interval(native) else None, + ) def validate_answer(fields: Any, question: Question) -> Answer: @@ -89,10 +116,20 @@ def validate_answer(fields: Any, question: Question) -> Answer: def validate_answers(reply: Reply, questions: dict[str, Question]) -> Decision: - """Every asked question answered and valid, as a ``Decision``; answers nobody asked for are dropped.""" + """Every asked question answered and valid, as a ``Decision``; answers nobody asked for are dropped. + + Per-question backend-native confidences are copied onto ``Decision.raw["backend_confidence"]`` (a map of + question name to number) so step records and probes can still read the original scale. + """ answers: dict[str, Answer] = {} for name, question in questions.items(): if name not in reply.answers: raise build_error(StatusCode.MODEL_CALL_FAILED, error_msg=f"no answer for question {name!r}") answers[name] = validate_answer(reply.answers[name], question) - return Decision(answers, reply.latency_ms, reply.usage, reply.model, reply.raw) + raw = dict(reply.raw) + native = { + name: answer.backend_confidence for name, answer in answers.items() if answer.backend_confidence is not None + } + if native: + raw["backend_confidence"] = native + return Decision(answers, reply.latency_ms, reply.usage, reply.model, raw) diff --git a/s1a/mcp_server.py b/s1a/mcp_server.py index 0a2c4a9..a5be356 100644 --- a/s1a/mcp_server.py +++ b/s1a/mcp_server.py @@ -128,7 +128,7 @@ async def decide( rules: str, model: Literal["jev", "clm", "laya", "laya-served", "cua"] = "jev", ) -> Decision: - """One choice question: the chosen key, probabilities, confidence and decision latency in ms. + """One choice question: the chosen key, probabilities, confidence (= max p) and decision latency in ms. Use jev over HTTP (default), laya/cua locally after installing the matching extra, or laya-served against a running Laya server (LAYA_SERVED_URL). Local model loading diff --git a/s1a/probe.py b/s1a/probe.py index 40c328e..de65c8b 100644 --- a/s1a/probe.py +++ b/s1a/probe.py @@ -72,7 +72,7 @@ async def pick( ) -> dict[str, Any]: """One ``choice`` question over ``options``: the chosen key, a probability per option, the confidence, the latency. - The dict the CLI prints and the MCP server returns: ``choice``, ``probabilities``, ``confidence``, ``ms``. + The dict the CLI prints and the MCP server returns: ``choice``, ``probabilities``, ``confidence`` (always ``max_k p_k``), ``ms``, and ``backend_confidence`` when the backend sent its own number. """ decision = await decision_model.decide_many(Observation(state), {"pick": ChoiceQuestion(options, rules=rules)}) return {**decision.choice("pick").as_dict(), "ms": decision.latency_ms} diff --git a/s1a/tool/models.py b/s1a/tool/models.py index 5f91009..ec1e77a 100644 --- a/s1a/tool/models.py +++ b/s1a/tool/models.py @@ -151,23 +151,24 @@ async def _decide(self) -> AssistantMessage: state.error = f"decision failed: {exc}" return self._stop("BLOCKED", state.error) choice = decision.choice("pick") - state.ticks.append( - { - "step": len(state.ticks) + 1, - "key": choice.key, - "confidence": round(choice.confidence, 3), - "probabilities": choice.probabilities, - "ms": round((time.perf_counter() - started) * 1000), - "input_tokens": decision.usage.input_tokens, - "output_tokens": decision.usage.output_tokens, - "usage_known": decision.usage.known, - "plan": bool(state.plan), - "blocked": sorted(state.blocked), - "source": self.name, - "model": decision.model, - **decision.provenance, - } - ) + tick = { + "step": len(state.ticks) + 1, + "key": choice.key, + "confidence": round(choice.confidence, 3), + "probabilities": choice.probabilities, + "ms": round((time.perf_counter() - started) * 1000), + "input_tokens": decision.usage.input_tokens, + "output_tokens": decision.usage.output_tokens, + "usage_known": decision.usage.known, + "plan": bool(state.plan), + "blocked": sorted(state.blocked), + "source": self.name, + "model": decision.model, + **decision.provenance, + } + if choice.backend_confidence is not None: + tick["backend_confidence"] = round(choice.backend_confidence, 3) + state.ticks.append(tick) state.blocked = set() # a block, the notices and the plan last one turn state.notices = [] state.plan = "" diff --git a/tests/decision_model_contract.py b/tests/decision_model_contract.py index 5150809..ed08f0a 100644 --- a/tests/decision_model_contract.py +++ b/tests/decision_model_contract.py @@ -62,6 +62,11 @@ async def test_confidence_and_latency_are_in_range(self) -> None: self.assertGreaterEqual(decision.latency_ms, 0) self.assertGreaterEqual(decision.usage.input_tokens, 0) + async def test_choice_confidence_is_the_peak_probability(self) -> None: + """Every backend reports ``max_k p_k`` as ``Choice.confidence`` (issue #89).""" + choice = (await self.make().decide_many(OBSERVATION, {"pick": PICK})).choice("pick") + self.assertAlmostEqual(choice.confidence, max(choice.probabilities.values()), places=6) + async def test_decide_equals_decide_many_with_one_question(self) -> None: one = await self.make().decide(OBSERVATION, PICK, name="pick") many = (await self.make().decide_many(OBSERVATION, {"pick": PICK})).answers["pick"] @@ -174,6 +179,24 @@ async def test_a_scripted_noul_comes_back_as_given(self) -> None: decision = await self.make_scripted([{"check": {"noul": 0.25}}]).decide_many(OBSERVATION, {"check": CHECK}) self.assertEqual((decision.noul("check").p, decision.noul("check").confidence), (0.25, 0.75)) + async def test_scripted_backend_confidence_is_kept_apart_from_contract_confidence(self) -> None: + """A backend-native confidence must not become ``Choice.confidence`` (issue #89). + + Some backends rewrite the number before validation (OmniJev recomputes Jev's margin); whatever lands in + the reply's ``confidence`` field is kept as ``backend_confidence``, while the contract field is always + ``max_k p_k``. + """ + if not self.scriptable: + self.skipTest("not scriptable") + # peak p = 0.9, with a Laya-like entropy confidence of 0.4 in the scripted dict + skewed = {"pick": {"choice": "stand", "probabilities": {"hit": 0.1, "stand": 0.9}, "confidence": 0.4}} + decision = await self.make_scripted([skewed]).decide_many(OBSERVATION, {"pick": PICK}) + choice = decision.choice("pick") + self.assertEqual(choice.key, "stand") + self.assertAlmostEqual(choice.confidence, 0.9, places=6) + self.assertIsNotNone(choice.backend_confidence) + self.assertEqual(decision.raw["backend_confidence"]["pick"], choice.backend_confidence) + def _seen_observations(decision_model: DecisionModel) -> list[Observation]: """Wrap the instance's ``_decide``; the returned list grows by the observation of every call.""" diff --git a/tests/test_cli.py b/tests/test_cli.py index 74d1ac7..dc267ee 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -195,7 +195,7 @@ def build(model_name: str, **kwargs: Any) -> JevModel: code, out, _err = _main(DECIDE) self.assertEqual((code, built), (0, ["jev"])) answer = json.loads(out) - self.assertEqual(sorted(answer), ["choice", "confidence", "ms", "probabilities"]) + self.assertEqual({"choice", "confidence", "ms", "probabilities"} - set(answer), set()) self.assertEqual((answer["choice"], answer["probabilities"]["stand"], answer["ms"]), ("stand", 1.0, 9)) question = transport.bodies[0]["questions"]["pick"] self.assertEqual( diff --git a/tests/test_decision_models_base.py b/tests/test_decision_models_base.py index f25f0b0..5dbf485 100644 --- a/tests/test_decision_models_base.py +++ b/tests/test_decision_models_base.py @@ -63,8 +63,8 @@ class TestScriptedModel(IsolatedAsyncioTestCase): async def test_the_knobs_script_the_answers_and_the_calls_are_recorded(self) -> None: decision_model = ScriptedModel(choose="hit", noul=[0.3], latency_ms=4, model="s-1") decision = await decision_model.decide_many(OBSERVATION, {"pick": PICK, "check": CHECK}) - self.assertEqual(decision.choice("pick"), Choice("hit", {"hit": 1.0, "stand": 0.0}, 1.0)) - self.assertEqual(decision.noul("check"), Noul(0.3, 0.7)) + self.assertEqual(decision.choice("pick"), Choice("hit", {"hit": 1.0, "stand": 0.0}, 1.0, backend_confidence=1.0)) + self.assertEqual(decision.noul("check"), Noul(0.3, 0.7)) # no backend confidence when scripted list omits it self.assertEqual((decision.latency_ms, decision.model), (4, "s-1")) self.assertEqual(decision_model.calls, [(OBSERVATION, {"pick": PICK, "check": CHECK})]) diff --git a/tests/test_decision_models_baselines.py b/tests/test_decision_models_baselines.py index da7f68c..994f05f 100644 --- a/tests/test_decision_models_baselines.py +++ b/tests/test_decision_models_baselines.py @@ -40,10 +40,12 @@ async def _picks(self, seed: int) -> list[str]: decision_model = RandomModel(seed) return [(await decision_model.decide(OBSERVATION, PICK)).key for _ in range(20)] # type: ignore[union-attr] - async def test_uniform_with_no_confidence_and_no_cost(self) -> None: + async def test_uniform_with_peak_confidence_and_no_cost(self) -> None: decision = await RandomModel(0).decide_many(OBSERVATION, {"pick": PICK}) choice = decision.choice("pick") - self.assertEqual((choice.probabilities, choice.confidence), ({"hit": 0.5, "stand": 0.5}, 0.0)) + # contract confidence is max p (= 0.5 for uniform two-way); backend sent 0.0 for provenance + self.assertEqual(choice.probabilities, {"hit": 0.5, "stand": 0.5}) + self.assertEqual((choice.confidence, choice.backend_confidence), (0.5, 0.0)) self.assertEqual((decision.latency_ms, decision.usage.input_tokens, decision.model), (0, 0, "random")) async def test_reproducible_per_seed_and_not_the_stream_of_random_seeded_with_the_integer(self) -> None: @@ -59,7 +61,7 @@ async def test_one_hot_on_the_rule_with_full_confidence_and_the_rule_name_as_sou decision_model = RuleModel("always-inc", lambda state, options: seen.append((state, options)) or "inc") options = {"inc": "add one", "noop": "do nothing"} decision = await decision_model.decide_many(Observation({"n": 1}), {"pick": ChoiceQuestion(options)}) - self.assertEqual(decision.choice("pick"), Choice("inc", {"inc": 1.0, "noop": 0.0}, 1.0)) + self.assertEqual(decision.choice("pick"), Choice("inc", {"inc": 1.0, "noop": 0.0}, 1.0, backend_confidence=1.0)) self.assertEqual( (decision_model.name, decision_model.model, decision.latency_ms), ("always-inc", "always-inc", 0) ) diff --git a/tests/test_decision_models_laya.py b/tests/test_decision_models_laya.py index 0b9b566..83ea381 100644 --- a/tests/test_decision_models_laya.py +++ b/tests/test_decision_models_laya.py @@ -122,8 +122,8 @@ async def test_a_noul_question_sends_a_string_instruction_and_native_criteria(se async def test_the_answers_come_back_typed_with_laya_usage_model_and_a_measured_latency(self) -> None: agent = FakeLayaAgent(input_tokens=77) decision = await _model(agent).decide_many(OBSERVATION, {"pick": PICK, "check": CHECK}) - self.assertEqual(decision.choice("pick"), Choice("hit", {"hit": 1.0, "stand": 0.0}, 1.0)) - self.assertEqual(decision.noul("check"), Noul(0.5, 0.5)) + self.assertEqual(decision.choice("pick"), Choice("hit", {"hit": 1.0, "stand": 0.0}, 1.0, backend_confidence=1.0)) + self.assertEqual(decision.noul("check"), Noul(0.5, 0.5, backend_confidence=0.5)) self.assertEqual((decision.usage.input_tokens, decision.usage.output_tokens), (77, 0)) self.assertEqual(decision.model, "laya-rl-agent") self.assertGreater(decision.latency_ms, 0) diff --git a/tests/test_decision_models_omnijev.py b/tests/test_decision_models_omnijev.py index 0c75d4d..2c9d089 100644 --- a/tests/test_decision_models_omnijev.py +++ b/tests/test_decision_models_omnijev.py @@ -130,7 +130,10 @@ async def test_a_choice_is_renormalised_over_the_offered_keys(self) -> None: self.assertEqual(choice.key, "stand") self.assertAlmostEqual(sum(choice.probabilities.values()), 1.0, places=6) self.assertAlmostEqual(choice.probabilities["stand"], 0.6 / 0.9, places=6) # abstain 0.1 left out - self.assertAlmostEqual(choice.confidence, (2 * 0.6 / 0.9 - 1) / 1, places=6) + peak = 0.6 / 0.9 + self.assertAlmostEqual(choice.confidence, peak, places=6) # contract: max_k p_k + # OmniJev's pre-validation Jev-margin recompute is kept as backend_confidence + self.assertAlmostEqual(choice.backend_confidence, (2 * peak - 1) / 1, places=6) async def test_abstain_stays_in_the_raw_reply(self) -> None: decision = await _model().decide_many(PICTURED, {"pick": contract.PICK}) diff --git a/tests/test_decision_models_types.py b/tests/test_decision_models_types.py index 6b88dd4..34c3fd7 100644 --- a/tests/test_decision_models_types.py +++ b/tests/test_decision_models_types.py @@ -70,6 +70,15 @@ def test_as_dict_is_the_cli_shape(self) -> None: GOOD.as_dict(), {"choice": "inc", "probabilities": {"inc": 0.7, "noop": 0.3}, "confidence": 0.7} ) self.assertEqual(Noul(0.9, 0.9).as_dict(), {"noul": 0.9, "confidence": 0.9}) + self.assertEqual( + Choice("inc", {"inc": 0.7, "noop": 0.3}, 0.7, backend_confidence=0.4).as_dict(), + { + "choice": "inc", + "probabilities": {"inc": 0.7, "noop": 0.3}, + "confidence": 0.7, + "backend_confidence": 0.4, + }, + ) def test_usage_tolerates_a_missing_or_malformed_payload(self) -> None: self.assertEqual(Usage.from_payload({"input_tokens": 315, "output_tokens": 31}), Usage(315, 31)) @@ -101,7 +110,7 @@ def test_a_decision_hands_out_answers_by_type(self) -> None: class TestValidation(TestCase): def test_a_good_choice_passes_with_its_numbers_normalised(self) -> None: choice = validate_choice(_fields("inc", {"inc": 1, "noop": 0}, 1), IDS) - self.assertEqual(choice, Choice("inc", {"inc": 1.0, "noop": 0.0}, 1.0)) + self.assertEqual(choice, Choice("inc", {"inc": 1.0, "noop": 0.0}, 1.0, backend_confidence=1.0)) self.assertEqual(choice_faults(GOOD_FIELDS, IDS), []) def test_every_fault_is_named(self) -> None: @@ -110,7 +119,6 @@ def test_every_fault_is_named(self) -> None: "cover": _fields("inc", {"inc": 1.0}, 1.0), "sum to": _fields("inc", {"inc": 0.3, "noop": 0.3}, 0.9), "not the peak": _fields("inc", {"inc": 0.3, "noop": 0.7}, 0.7), - "confidence": _fields("inc", {"inc": 1.0, "noop": 0.0}, 1.5), "not an object": _fields("inc", None, 1.0), "not in [0, 1]": _fields("inc", {"inc": 1.2, "noop": -0.2}, 1.0), "choice None": {}, @@ -126,14 +134,27 @@ def test_every_fault_is_named(self) -> None: self.assertIn("not an offered choice", str(caught.exception)) self.assertIn(fragment, str(caught.exception)) + def test_choice_confidence_is_always_the_peak_probability(self) -> None: + # Laya-like entropy 0.4 with peak 0.9 → contract confidence 0.9, backend 0.4 + choice = validate_choice(_fields("inc", {"inc": 0.9, "noop": 0.1}, 0.4), IDS) + self.assertEqual(choice.confidence, 0.9) + self.assertEqual(choice.backend_confidence, 0.4) + # missing or out-of-range backend confidence is dropped, not faulted + self.assertIsNone(validate_choice(_fields("inc", {"inc": 1.0, "noop": 0.0}, 1.5), IDS).backend_confidence) + self.assertIsNone( + validate_choice({"choice": "inc", "probabilities": {"inc": 1.0, "noop": 0.0}}, IDS).backend_confidence + ) + def test_ties_at_the_peak_and_a_sum_within_two_percent_are_accepted(self) -> None: self.assertEqual(choice_faults(_fields("inc", {"inc": 0.5, "noop": 0.5}, 0.5), IDS), []) self.assertEqual(choice_faults(_fields("inc", {"inc": 0.6, "noop": 0.39}, 0.5), IDS), []) - def test_a_noul_needs_a_probability_and_defaults_its_confidence(self) -> None: + def test_a_noul_needs_a_probability_and_uses_max_p_as_confidence(self) -> None: self.assertEqual(validate_noul({"noul": 0.2}), Noul(0.2, 0.8)) - self.assertEqual(validate_noul({"noul": 0.9, "confidence": 0.55}), Noul(0.9, 0.55)) + # backend 0.55 is kept apart; contract confidence is max(0.9, 0.1) = 0.9 + self.assertEqual(validate_noul({"noul": 0.9, "confidence": 0.55}), Noul(0.9, 0.9, backend_confidence=0.55)) self.assertEqual(validate_noul({"noul": 0.9, "confidence": 7}).confidence, 0.9) + self.assertIsNone(validate_noul({"noul": 0.9, "confidence": 7}).backend_confidence) for bad in (1.7, -0.1, None, "0.5", True): with self.assertRaises(BaseError): validate_noul({"noul": bad}) @@ -151,10 +172,11 @@ def test_validate_answers_builds_the_decision_from_every_asked_question(self) -> ) decision = validate_answers(reply, questions) self.assertEqual(sorted(decision.answers), ["check", "pick"]) - self.assertEqual((decision.choice("pick"), decision.noul("check")), (GOOD, Noul(0.2, 0.8))) - self.assertEqual( - (decision.latency_ms, decision.usage, decision.model, decision.raw), (3, Usage(5, 1), "m", reply.raw) - ) + expected_pick = Choice("inc", {"inc": 0.7, "noop": 0.3}, 0.7, backend_confidence=0.7) + self.assertEqual((decision.choice("pick"), decision.noul("check")), (expected_pick, Noul(0.2, 0.8))) + self.assertEqual((decision.latency_ms, decision.usage, decision.model), (3, Usage(5, 1), "m")) + self.assertEqual(decision.raw["answers"], "whatever came") + self.assertEqual(decision.raw["backend_confidence"], {"pick": 0.7}) with self.assertRaises(BaseError) as caught: validate_answers(Reply(answers={"pick": GOOD_FIELDS}, latency_ms=0), questions) self.assertIn("no answer for question 'check'", str(caught.exception)) diff --git a/tests/test_tool_models.py b/tests/test_tool_models.py index dd82e90..cebf377 100644 --- a/tests/test_tool_models.py +++ b/tests/test_tool_models.py @@ -338,7 +338,7 @@ async def test_random_picks_an_offered_key_with_a_uniform_distribution(self) -> self.assertIn(json.loads(message.tool_calls[0].arguments)["key"], {"inc", "noop"}) self.assertEqual((state.ticks[0]["source"], state.ticks[0]["input_tokens"]), ("random", 0)) self.assertEqual( - (state.ticks[0]["probabilities"], state.ticks[0]["confidence"]), ({"inc": 0.5, "noop": 0.5}, 0.0) + (state.ticks[0]["probabilities"], state.ticks[0]["confidence"]), ({"inc": 0.5, "noop": 0.5}, 0.5) ) async def test_random_is_reproducible_per_seed_and_not_the_stream_of_random_seeded_with_the_integer(self) -> None: From 3734ffc2a3ce0016e0dcfef5b6e06180cefedff5 Mon Sep 17 00:00:00 2001 From: Birfy Date: Wed, 21 Oct 2026 19:45:58 +0800 Subject: [PATCH 2/2] [Fix] Ruff-format two tests; keep a backend's raw backend_confidence and surface every head's on browser ticks --- CHANGELOG.md | 4 +++- s1a/browser/action_space.py | 7 +++++++ s1a/browser/decision_model.py | 5 +++++ s1a/decision_models/validation.py | 6 +++++- tests/test_browser_policy.py | 26 ++++++++++++++++++++++++++ tests/test_decision_models_base.py | 4 +++- tests/test_decision_models_laya.py | 4 +++- tests/test_decision_models_types.py | 9 +++++++++ 8 files changed, 61 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a4d572c..2fac8a0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -58,7 +58,9 @@ The format follows [Keep a Changelog](https://keepachangelog.com/en/1.1.0/); ver `max(p, 1 - p)` for a noul, both computed in `validation.py` from the validated probabilities. Historical `conf=` values from `laya` / `laya-served` (normalised entropy), `jev` / `clm` (service-reported) and `omnijev` (Jev margin) are not comparable with new ones. Each backend's own number is kept as `backend_confidence` on the answer, in - `Decision.raw["backend_confidence"]`, and on tool/browser tick records when present. See #89. + `Decision.raw["backend_confidence"]`, and on tool/browser tick records when present (browser ticks also carry + `backend_confidences`, every answered head's number by question name). A backend's own `raw["backend_confidence"]` + is kept as `raw["backend_confidence_reply"]` instead of being overwritten. See #89. - Review guidance (`self-review`, `review-pr`) now treats merge readiness as a gate: authors confirm the pull request reports mergeable with CI green on the recorded head before requesting review, and reviewers do not diff --git a/s1a/browser/action_space.py b/s1a/browser/action_space.py index 507896f..0e05dc4 100644 --- a/s1a/browser/action_space.py +++ b/s1a/browser/action_space.py @@ -70,6 +70,8 @@ class Move: usage: Usage model: str backend_confidence: float | None = None + # Every answered head's backend-native confidence (operation, target, text value), by question name. + backend_confidences: dict[str, float] = field(default_factory=dict) def submittable(item: dict[str, Any]) -> bool: @@ -244,4 +246,9 @@ def interpret(decision: Decision, space: ActionSpace) -> Move: usage=decision.usage, model=decision.model, backend_confidence=operation.backend_confidence, + backend_confidences={ + name: answer.backend_confidence + for name, answer in decision.answers.items() + if answer.backend_confidence is not None + }, ) diff --git a/s1a/browser/decision_model.py b/s1a/browser/decision_model.py index b924fe2..4632771 100644 --- a/s1a/browser/decision_model.py +++ b/s1a/browser/decision_model.py @@ -349,6 +349,11 @@ async def _decide_message(self, messages: Any) -> AssistantMessage: if move.backend_confidence is not None else {} ), + **( + {"backend_confidences": {name: round(v, 3) for name, v in move.backend_confidences.items()}} + if move.backend_confidences + else {} + ), "elements": len(space.elements), "elapsed_ms": round((time.perf_counter() - run.started_at) * 1000), "probabilities": probabilities, # the replay's bars: the settled head's top keys diff --git a/s1a/decision_models/validation.py b/s1a/decision_models/validation.py index 64c007d..8c0de2c 100644 --- a/s1a/decision_models/validation.py +++ b/s1a/decision_models/validation.py @@ -119,7 +119,9 @@ def validate_answers(reply: Reply, questions: dict[str, Question]) -> Decision: """Every asked question answered and valid, as a ``Decision``; answers nobody asked for are dropped. Per-question backend-native confidences are copied onto ``Decision.raw["backend_confidence"]`` (a map of - question name to number) so step records and probes can still read the original scale. + question name to number) so step records and probes can still read the original scale. If the backend's + own ``raw`` already carried a ``backend_confidence`` key, that value is kept as + ``raw["backend_confidence_reply"]`` rather than silently overwritten. """ answers: dict[str, Answer] = {} for name, question in questions.items(): @@ -131,5 +133,7 @@ def validate_answers(reply: Reply, questions: dict[str, Question]) -> Decision: name: answer.backend_confidence for name, answer in answers.items() if answer.backend_confidence is not None } if native: + if "backend_confidence" in raw: + raw["backend_confidence_reply"] = raw["backend_confidence"] raw["backend_confidence"] = native return Decision(answers, reply.latency_ms, reply.usage, reply.model, raw) diff --git a/tests/test_browser_policy.py b/tests/test_browser_policy.py index 753bfde..15b097f 100644 --- a/tests/test_browser_policy.py +++ b/tests/test_browser_policy.py @@ -630,6 +630,32 @@ async def test_tool_call_ids_differ_between_runs(self) -> None: self.assertNotEqual(first.tool_calls[0].id, second.tool_calls[0].id) +class TestInterpretBackendConfidences(TestCase): + def test_every_answered_heads_backend_confidence_reaches_the_move(self) -> None: + from s1a.decision_models import Choice, Decision + + candidate = action_space.Candidate("CLICK", "3", {"label": "Go"}) + space = action_space.ActionSpace(heads={"CLICK": {"3": candidate}}) + decision = Decision( + answers={ + "operation": Choice("CLICK", {"CLICK": 0.8, "WAIT": 0.2}, 0.8, backend_confidence=0.4), + "click_target": Choice("3", {"3": 1.0}, 1.0, backend_confidence=0.9), + }, + latency_ms=1, + ) + move = action_space.interpret(decision, space) + self.assertEqual(move.backend_confidence, 0.4) + self.assertEqual(move.backend_confidences, {"operation": 0.4, "click_target": 0.9}) + bare = Decision( + answers={ + "operation": Choice("CLICK", {"CLICK": 1.0}, 1.0), + "click_target": Choice("3", {"3": 1.0}, 1.0), + }, + latency_ms=1, + ) + self.assertEqual(action_space.interpret(bare, space).backend_confidences, {}) + + class TestDecisionModelAttributeIsolation(TestCase): """B2: the model must not collide with the base ``Model._client`` slot.""" diff --git a/tests/test_decision_models_base.py b/tests/test_decision_models_base.py index 5dbf485..db66dc9 100644 --- a/tests/test_decision_models_base.py +++ b/tests/test_decision_models_base.py @@ -63,7 +63,9 @@ class TestScriptedModel(IsolatedAsyncioTestCase): async def test_the_knobs_script_the_answers_and_the_calls_are_recorded(self) -> None: decision_model = ScriptedModel(choose="hit", noul=[0.3], latency_ms=4, model="s-1") decision = await decision_model.decide_many(OBSERVATION, {"pick": PICK, "check": CHECK}) - self.assertEqual(decision.choice("pick"), Choice("hit", {"hit": 1.0, "stand": 0.0}, 1.0, backend_confidence=1.0)) + self.assertEqual( + decision.choice("pick"), Choice("hit", {"hit": 1.0, "stand": 0.0}, 1.0, backend_confidence=1.0) + ) self.assertEqual(decision.noul("check"), Noul(0.3, 0.7)) # no backend confidence when scripted list omits it self.assertEqual((decision.latency_ms, decision.model), (4, "s-1")) self.assertEqual(decision_model.calls, [(OBSERVATION, {"pick": PICK, "check": CHECK})]) diff --git a/tests/test_decision_models_laya.py b/tests/test_decision_models_laya.py index 83ea381..c057cda 100644 --- a/tests/test_decision_models_laya.py +++ b/tests/test_decision_models_laya.py @@ -122,7 +122,9 @@ async def test_a_noul_question_sends_a_string_instruction_and_native_criteria(se async def test_the_answers_come_back_typed_with_laya_usage_model_and_a_measured_latency(self) -> None: agent = FakeLayaAgent(input_tokens=77) decision = await _model(agent).decide_many(OBSERVATION, {"pick": PICK, "check": CHECK}) - self.assertEqual(decision.choice("pick"), Choice("hit", {"hit": 1.0, "stand": 0.0}, 1.0, backend_confidence=1.0)) + self.assertEqual( + decision.choice("pick"), Choice("hit", {"hit": 1.0, "stand": 0.0}, 1.0, backend_confidence=1.0) + ) self.assertEqual(decision.noul("check"), Noul(0.5, 0.5, backend_confidence=0.5)) self.assertEqual((decision.usage.input_tokens, decision.usage.output_tokens), (77, 0)) self.assertEqual(decision.model, "laya-rl-agent") diff --git a/tests/test_decision_models_types.py b/tests/test_decision_models_types.py index 34c3fd7..478eae4 100644 --- a/tests/test_decision_models_types.py +++ b/tests/test_decision_models_types.py @@ -181,6 +181,15 @@ def test_validate_answers_builds_the_decision_from_every_asked_question(self) -> validate_answers(Reply(answers={"pick": GOOD_FIELDS}, latency_ms=0), questions) self.assertIn("no answer for question 'check'", str(caught.exception)) + def test_a_backend_raw_backend_confidence_key_is_kept_not_overwritten(self) -> None: + questions = {"pick": ChoiceQuestion({"inc": "", "noop": ""})} + reply = Reply(answers={"pick": GOOD_FIELDS}, latency_ms=0, raw={"backend_confidence": "native-blob"}) + decision = validate_answers(reply, questions) + self.assertEqual(decision.raw["backend_confidence"], {"pick": 0.7}) + self.assertEqual(decision.raw["backend_confidence_reply"], "native-blob") + untouched = validate_answers(Reply(answers={"pick": GOOD_FIELDS}, latency_ms=0), questions) + self.assertNotIn("backend_confidence_reply", untouched.raw) + def test_an_answer_of_the_wrong_shape_or_no_object_fails_on_its_fields(self) -> None: questions = {"pick": ChoiceQuestion({"inc": "", "noop": ""}), "check": NoulQuestion("q")} swapped = Reply(answers={"pick": {"noul": 0.5}, "check": GOOD_FIELDS}, latency_ms=0)