diff --git a/.env.example b/.env.example index 707df86..0d6faae 100644 --- a/.env.example +++ b/.env.example @@ -1,5 +1,6 @@ # Jev: a direct TypeSafe key. Leave TYPESAFE_API_URL unset with it; setting that URL selects the OpenRouter proxy below. TYPESAFE_API_KEY= +# The direct path sends the pinned jev-1.13.0; TYPESAFE_MODEL overrides it with a TypeSafe name (e.g. jev-1.14.0). # The chat model (typed values, final answers, `--model llm`, the rethink planner); every model that calls it needs MODEL_NAME. # OPENROUTER_* stands in for LLM_API_KEY and LLM_BASE_URL when those are unset. diff --git a/CHANGELOG.md b/CHANGELOG.md index fa5e7ff..10dcfc8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,14 @@ The format follows [Keep a Changelog](https://keepachangelog.com/en/1.1.0/); ver ## Unreleased +### Changed + +- The direct TypeSafe path pins `jev-1.13.0` instead of the moving `jev-latest` alias (which moved to Jev 1.14.0 + after every published number here was recorded on 1.13), and `TYPESAFE_MODEL` now overrides the model on that + path too; an OpenRouter-style `typesafe/...` id there is a config error. Jev step records (tool and browser + ticks) keep `requested_model` next to `model`, so a saved run says which Jev was asked for and which answered. + The 1.13.0 vs 1.14.0 benchmark re-run is still to do. See #93. + ### Added - Offline quality consumer for shared Omni public231 raw, retaining planned diff --git a/docs/benchmarks.md b/docs/benchmarks.md index 5ea9f07..071e203 100644 --- a/docs/benchmarks.md +++ b/docs/benchmarks.md @@ -142,7 +142,7 @@ Medians of the window: A 13.1 s, B 10.1 s, C 36.4 s over the two verified runs, ### Results, direct TypeSafe backend -Same task and Chrome, later the same day, decisions from `api.typesafe.ai` (`jev-latest`) instead of OpenRouter. Three runs per arm, back to back. +Same task and Chrome, later the same day, decisions from `api.typesafe.ai` (`jev-latest`, which pointed at Jev 1.13 on that date; the direct path now pins `jev-1.13.0`, see #93) instead of OpenRouter. Three runs per arm, back to back. | run | window s | decisions | median ms | decision s | probe s | tool s | value wait s | verified | |---|---|---|---|---|---|---|---|---| diff --git a/docs/browser-front.md b/docs/browser-front.md index f7a6de3..c72bcec 100644 --- a/docs/browser-front.md +++ b/docs/browser-front.md @@ -95,9 +95,9 @@ browser-use/jev-ultrafast (MIT), whose observe-decide-act tick this policy follo | `BATCH_ACTION_TIMEOUT_MS` | 2000 | a stamp lost to a re-render fails fast; the returned probe re-stamps | 14. Two decisions backends, separate keys. `typesafe` talks to `api.typesafe.ai/v1/systemone` with - `TYPESAFE_API_KEY` and `jev-latest`; `openrouter` talks to `/api/alpha/decisions` with - `OPENROUTER_API_KEY` and `typesafe/jev-1.13` (`TYPESAFE_API_URL` and `TYPESAFE_MODEL` override the - proxy only). Setting `TYPESAFE_API_URL` selects the proxy backend, which reads `OPENROUTER_API_KEY`; the + `TYPESAFE_API_KEY` and the pinned `jev-1.13.0`; `openrouter` talks to `/api/alpha/decisions` with + `OPENROUTER_API_KEY` and `typesafe/jev-1.13` (`TYPESAFE_MODEL` overrides the model on both, `TYPESAFE_API_URL` + the proxy URL). Each tick records `model` and `requested_model`. Setting `TYPESAFE_API_URL` selects the proxy backend, which reads `OPENROUTER_API_KEY`; the TypeSafe key is never sent to OpenRouter. Measured: 294 to 418 ms per decision direct, 450 to 510 ms through the proxy. 15. Batched actions (`--batch on`, `BrowserPolicy.batch_actions`, needs the `unsafe_dev` browser capability). Each step is one `browser_run_code_unsafe` call that performs the action and returns the next probe; one transport round diff --git a/docs/configuration.md b/docs/configuration.md index 8019b7f..2490a5e 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -10,7 +10,7 @@ Variables can be exported in your shell or placed in a `.env` file at the root o |:---|:---|:---|:---| | `TYPESAFE_API_KEY` | `jev` model | *(unset)* | API key for direct TypeSafe decisions endpoint (`https://api.typesafe.ai/v1/systemone`). | | `TYPESAFE_API_URL` | `jev` model | `https://openrouter.ai/api/alpha/decisions` | Endpoint URL for decisions; defaults to OpenRouter proxy, or can be overridden to a custom proxy URL. | -| `TYPESAFE_MODEL` | `jev` model | `typesafe/jev-1.13` | Model identifier when proxying Jev decisions through OpenRouter. | +| `TYPESAFE_MODEL` | `jev` model | `jev-1.13.0` direct, `typesafe/jev-1.13` through OpenRouter | Jev model id on either backend. Both defaults are pinned versions: no path sends the moving `jev-latest` or `jev-preview` aliases unless you set them here. On the direct path (`TYPESAFE_API_KEY` without `TYPESAFE_API_URL`) it takes a TypeSafe name such as `jev-1.14.0`; an OpenRouter id like `typesafe/jev-1.13` there is a config error. TypeSafe keeps `jev-1.13.0` available and promises 60 days' notice before deprecating it. Step records keep `model` (the id the response reports, else the one sent) and `requested_model` (the one sent). | | `S1A_DECISION_TIMEOUT_S` | `jev` model | `5` | Seconds one decision may take, retries included. Raise it for a slower System One server behind `TYPESAFE_API_URL` (e.g. a local OneJev-27B or CLM-8B: `30`). | | `OPENROUTER_API_KEY` | `jev` model (proxy), chat model fallback | *(unset)* | OpenRouter API key, used for proxying Jev decisions or as a fallback for `LLM_API_KEY`. | | `OPENROUTER_BASE_URL` | chat model fallback | `https://openrouter.ai/api/v1` | Fallback base URL for the chat model when `LLM_BASE_URL` or `OPENAI_BASE_URL` is unset. | diff --git a/docs/why.md b/docs/why.md index 570dda4..f1f79a1 100644 --- a/docs/why.md +++ b/docs/why.md @@ -22,7 +22,7 @@ what happens to speed, cost and score, on the same loop, the same tools and the ## 2. What Jev is, and where it fits -TypeSafe Jev (`jev-latest` at api.typesafe.ai, `typesafe/jev-1.13` on OpenRouter) is a decision model with +TypeSafe Jev (pinned `jev-1.13.0` at api.typesafe.ai, `typesafe/jev-1.13` on OpenRouter) is a decision model with three heads, each returning an index into options the caller enumerates: `choice` (one option key, a probability per option, a confidence), `noul` (the probability that a statement holds) and `score` (a position in an ordered rubric). It emits no free text. Input is capped at 32K tokens. Price: $0.042 per million input diff --git a/s1a/browser/decision_model.py b/s1a/browser/decision_model.py index b5d785d..a2ddec6 100644 --- a/s1a/browser/decision_model.py +++ b/s1a/browser/decision_model.py @@ -348,7 +348,10 @@ async def _decide_message(self, messages: Any) -> AssistantMessage: "elapsed_ms": round((time.perf_counter() - run.started_at) * 1000), "probabilities": probabilities, # the replay's bars: the settled head's top keys "candidates": candidates, + "model": move.model, # who answered: the response's model id, else the one the request named } + if "requested_model" in decision.provenance: + record["requested_model"] = decision.provenance["requested_model"] if screenshot_ms is not None: # an image-reading model: the capture is part of the step's cost record["screenshot_ms"] = screenshot_ms run.ticks.append(record) diff --git a/s1a/decision_models/jev.py b/s1a/decision_models/jev.py index 4d2b0d2..5fe0dff 100644 --- a/s1a/decision_models/jev.py +++ b/s1a/decision_models/jev.py @@ -58,7 +58,7 @@ async def _decide(self, observation: Observation, questions: dict[str, Question] latency_ms=ms, usage=Usage.from_payload(fields.get("usage")), model=str(fields.get("model") or self.model), - raw=fields, + raw={**fields, "requested_model": self.model}, # which Jev was asked for, next to the one that answered ) async def warm(self) -> None: diff --git a/s1a/decision_models/types.py b/s1a/decision_models/types.py index ad33a7f..759e286 100644 --- a/s1a/decision_models/types.py +++ b/s1a/decision_models/types.py @@ -11,7 +11,7 @@ Json = dict[str, Any] QuestionType = Literal["choice", "noul"] -PROVENANCE_KEYS = ("served_by", "url", "request_id", "server_timing") +PROVENANCE_KEYS = ("served_by", "url", "request_id", "server_timing", "requested_model") @dataclass(frozen=True) @@ -203,5 +203,6 @@ def noul(self, name: str) -> Noul: @property def provenance(self) -> Json: - """Where a served answer came from, for step records: the server's identity, URL and timings; empty in process.""" + """Where a served answer came from, for step records: the server's identity, URL and timings, and for Jev the + model id the request named (``model`` is the one the response reports, else that same id); empty in process.""" return {key: self.raw[key] for key in PROVENANCE_KEYS if key in self.raw} diff --git a/s1a/decision_models/wire.py b/s1a/decision_models/wire.py index eee6c52..aa396e9 100644 --- a/s1a/decision_models/wire.py +++ b/s1a/decision_models/wire.py @@ -21,7 +21,9 @@ DEFAULT_DECISIONS_URL = "https://openrouter.ai/api/alpha/decisions" DEFAULT_MODEL = "typesafe/jev-1.13" TYPESAFE_DECISIONS_URL = "https://api.typesafe.ai/v1/systemone" -TYPESAFE_DEFAULT_MODEL = "jev-latest" +# Pinned: ``jev-latest`` is a moving alias (it moved to jev-1.14.0 around 2026-10-21) and every published number +# here was recorded on 1.13. TypeSafe promises 60 days' notice before retiring jev-1.13.0. See #93. +TYPESAFE_DEFAULT_MODEL = "jev-1.13.0" DECISIONS_BACKENDS = ("typesafe", "openrouter") DECISIONS_TIMEOUT_S = 5.0 # Jev answers in 0.4 to 1.3 s through the proxy; a dead connection must not stall a step _RETRY_STATUSES = frozenset({429, 503, 529}) @@ -60,16 +62,27 @@ def decisions_backend_from_env() -> str: def client_from_env(backend: str, *, timeout_s: float) -> "JevDecisionsClient": """Build the decisions client for one backend from the environment. - ``typesafe`` talks to TypeSafe directly with ``TYPESAFE_API_KEY`` and its own model names; - ``openrouter`` talks to the proxy with ``OPENROUTER_API_KEY`` (``TYPESAFE_API_URL`` / ``TYPESAFE_MODEL`` - override the proxy URL and model). + ``typesafe`` talks to TypeSafe directly with ``TYPESAFE_API_KEY`` and its own model names (``TYPESAFE_MODEL`` + overrides the pinned ``TYPESAFE_DEFAULT_MODEL``; an OpenRouter-style ``typesafe/...`` id there is a config + error rather than a request TypeSafe would reject); ``openrouter`` talks to the proxy with + ``OPENROUTER_API_KEY`` (``TYPESAFE_API_URL`` / ``TYPESAFE_MODEL`` override the proxy URL and model). """ match backend: case "typesafe": + model = (os.getenv("TYPESAFE_MODEL") or "").strip() or TYPESAFE_DEFAULT_MODEL + if "/" in model: + raise build_error( + StatusCode.MODEL_SERVICE_CONFIG_ERROR, + error_msg=( + f"TYPESAFE_MODEL={model!r} is an OpenRouter model id, but TYPESAFE_API_KEY without " + f"TYPESAFE_API_URL talks to TypeSafe directly: use a TypeSafe name such as " + f"{TYPESAFE_DEFAULT_MODEL!r}, or set TYPESAFE_API_URL to use the proxy" + ), + ) return JevDecisionsClient( api_key=os.getenv("TYPESAFE_API_KEY") or "", url=TYPESAFE_DECISIONS_URL, - model=TYPESAFE_DEFAULT_MODEL, + model=model, timeout_s=timeout_s, ) case "openrouter": diff --git a/tests/test_browser_policy.py b/tests/test_browser_policy.py index 753bfde..59a242e 100644 --- a/tests/test_browser_policy.py +++ b/tests/test_browser_policy.py @@ -432,6 +432,7 @@ async def test_the_tick_records_the_decisions_latency_tokens_and_the_answering_m self.assertEqual((tick["operation"], tick["target"], tick["confidence"]), ("CLICK", "Search", 0.9)) self.assertEqual(tick["probabilities"], {"1": 0.9, "2": 0.1}, "the click head's probabilities, for the replay") self.assertEqual(tick["candidates"]["1"], "Search") + self.assertEqual((tick["model"], tick["requested_model"]), ("jev-9", "typesafe/jev-test"), "who answered (#93)") report = slot_model.report() self.assertEqual((report["decisions"], report["median_decision_ms"], report["jev_input_tokens"]), (1, 7, 315)) @@ -1084,6 +1085,16 @@ def test_typesafe_uses_its_own_key_url_and_model(self) -> None: self.assertEqual(client.model, wire.TYPESAFE_DEFAULT_MODEL) self.assertEqual(client._client.headers["Authorization"], "Bearer ts-key") + def test_typesafe_reads_typesafe_model_and_rejects_an_openrouter_id(self) -> None: + env = {"TYPESAFE_API_KEY": "ts-key", "TYPESAFE_API_URL": "", "TYPESAFE_MODEL": "jev-1.14.0"} + with mock.patch.dict("os.environ", env, clear=False): + self.assertEqual(wire.client_from_env("typesafe", timeout_s=5.0).model, "jev-1.14.0") + with mock.patch.dict("os.environ", {**env, "TYPESAFE_MODEL": "typesafe/jev-1.13"}, clear=False): + with self.assertRaises(BaseError) as caught: + wire.client_from_env("typesafe", timeout_s=5.0) + self.assertEqual(caught.exception.status, StatusCode.MODEL_SERVICE_CONFIG_ERROR) + self.assertIn("TYPESAFE_API_URL", str(caught.exception)) + def test_openrouter_uses_the_openrouter_key_and_honours_the_dotenv_overrides(self) -> None: env = { "TYPESAFE_API_KEY": "ts-key", diff --git a/tests/test_decision_models_jev.py b/tests/test_decision_models_jev.py index e7b945d..2bb30d6 100644 --- a/tests/test_decision_models_jev.py +++ b/tests/test_decision_models_jev.py @@ -249,6 +249,17 @@ async def test_usage_and_model_come_from_the_payload_with_zeros_when_missing(sel bare = ScriptedTransport(usage={}) self.assertEqual((await JevModel(bare).decide_many(OBSERVATION, {"pick": PICK})).usage, Usage()) + async def test_the_requested_model_is_recorded_next_to_the_answering_one(self) -> None: + class Moved(ScriptedTransport): # the server names the model it actually ran + async def decide(self, body: Json) -> tuple[Json, int]: + payload, ms = await super().decide(body) + return {**payload, "model": "jev-1.14.0"}, ms + + decision = await JevModel(Moved()).decide_many(OBSERVATION, {"pick": PICK}) + self.assertEqual((decision.model, decision.provenance["requested_model"]), ("jev-1.14.0", MODEL)) + silent = await JevModel(ScriptedTransport()).decide_many(OBSERVATION, {"pick": PICK}) + self.assertEqual((silent.model, silent.provenance["requested_model"]), (MODEL, MODEL)) + async def test_a_missing_or_malformed_answer_is_a_model_call_failure(self) -> None: for answers in ({}, {"pick": "nonsense"}, {"other": {"choice": "hit"}}): with self.subTest(answers=answers): @@ -288,6 +299,30 @@ def test_typesafe_when_its_key_is_set_and_no_proxy_url_overrides_it(self) -> Non self.assertEqual(decision_model.model, wire.TYPESAFE_DEFAULT_MODEL) self.assertEqual(decision_model._transport.url, wire.TYPESAFE_DECISIONS_URL) + def test_no_default_is_a_moving_alias(self) -> None: # #93: jev-latest moved to 1.14 under recorded numbers + for default in (wire.TYPESAFE_DEFAULT_MODEL, wire.DEFAULT_MODEL): + with self.subTest(default=default): + self.assertNotIn(default.rsplit("/", 1)[-1], ("jev-latest", "jev-preview")) + self.assertEqual(wire.TYPESAFE_DEFAULT_MODEL, "jev-1.13.0") + + def test_typesafe_model_overrides_the_pin_on_the_direct_path(self) -> None: + env = { + "TYPESAFE_API_KEY": "k", + "TYPESAFE_API_URL": "", + "OPENROUTER_API_KEY": "", + "TYPESAFE_MODEL": "jev-1.14.0", + } + with patch.dict(os.environ, env): + self.assertEqual(JevModel.from_env().model, "jev-1.14.0") + with patch.dict(os.environ, {**env, "TYPESAFE_MODEL": ""}): + self.assertEqual(JevModel.from_env().model, wire.TYPESAFE_DEFAULT_MODEL) + + def test_an_openrouter_model_id_on_the_direct_path_is_a_config_error(self) -> None: + env = {"TYPESAFE_API_KEY": "k", "TYPESAFE_API_URL": "", "TYPESAFE_MODEL": "typesafe/jev-1.13"} + with patch.dict(os.environ, env), self.assertRaises(BaseError) as caught: + JevModel.from_env() + self.assertEqual(caught.exception.status, StatusCode.MODEL_SERVICE_CONFIG_ERROR) + def test_openrouter_otherwise(self) -> None: env = {"TYPESAFE_API_KEY": "", "OPENROUTER_API_KEY": "r", "TYPESAFE_MODEL": "", "TYPESAFE_API_URL": ""} with patch.dict(os.environ, env):