From c18e9e7c634412e623cfa0f18aed25809c3d75a0 Mon Sep 17 00:00:00 2001 From: Tianyao Wu Date: Mon, 26 Oct 2026 09:35:16 +0800 Subject: [PATCH] benchmarks: record, measure and replay decision traces A trace (s1-trace/1) records what an agent sent, one record per attempt, with episode, step, timing and image hashes. record_s1a.py records System1-Agents' decisions from outside its source. metrics.py measures repeated work as the Open-Jev or Cua-S1 worker tokenizes it: sharing within a request, a cache of earlier prompts in 64-token chunks, and repeated states and images. replay.py replays a trace one client per episode with its think time, together or one episode after another. Two small traces and the first results come with it. --- benchmarks/README.md | 3 + benchmarks/traces/README.md | 97 ++++++ benchmarks/traces/metrics.py | 308 ++++++++++++++++ benchmarks/traces/record_s1a.py | 129 +++++++ benchmarks/traces/replay.py | 160 +++++++++ benchmarks/traces/results-2026-10-25.md | 63 ++++ benchmarks/traces/samples/README.md | 19 + .../traces/samples/blackjack.trace.jsonl | 41 +++ .../traces/samples/ticket_router.trace.jsonl | 32 ++ benchmarks/traces/tracefile.py | 140 ++++++++ tests/benchmarks/test_traces.py | 328 ++++++++++++++++++ 11 files changed, 1320 insertions(+) create mode 100644 benchmarks/traces/README.md create mode 100644 benchmarks/traces/metrics.py create mode 100644 benchmarks/traces/record_s1a.py create mode 100644 benchmarks/traces/replay.py create mode 100644 benchmarks/traces/results-2026-10-25.md create mode 100644 benchmarks/traces/samples/README.md create mode 100644 benchmarks/traces/samples/blackjack.trace.jsonl create mode 100644 benchmarks/traces/samples/ticket_router.trace.jsonl create mode 100644 benchmarks/traces/tracefile.py create mode 100644 tests/benchmarks/test_traces.py diff --git a/benchmarks/README.md b/benchmarks/README.md index 807cc777..d187b62a 100644 --- a/benchmarks/README.md +++ b/benchmarks/README.md @@ -5,6 +5,9 @@ First harness for #39: replay the same labelled requests against CUDA-backed required (`python -m pip install -r benchmarks/requirements.txt`). Reuse an existing benchmark environment when available. +[Decision traces](traces/README.md) record what agents actually send, measure how much of +it repeats, and replay it at the agent's pace. + The [latency leaderboard](latency-leaderboard.md) records the five-model H800 comparison and its timing rules. Its historical body-receipt clock is kept separate from this runner's body/JSON/schema-validation clock. diff --git a/benchmarks/traces/README.md b/benchmarks/traces/README.md new file mode 100644 index 00000000..35fc9387 --- /dev/null +++ b/benchmarks/traces/README.md @@ -0,0 +1,97 @@ +# Decision traces + +Tools for #150's first section: record what agents actually send, measure how much of +it repeats, and replay it at the pace the agent sent it. + +- [`tracefile.py`](tracefile.py) checks a trace and converts saved System1-Agents ticks. +- [`record_s1a.py`](record_s1a.py) records System1-Agents' decisions as a trace without + changing its source. +- [`metrics.py`](metrics.py) measures repeated work as one worker tokenizes it. +- [`replay.py`](replay.py) replays a trace against a running server, one client per + episode, keeping its order and think time. +- [`samples/`](samples/) holds two small traces whose content may be shared. +- [The first results](results-2026-10-25.md) come from traces recorded on 2026-10-25. + +They need Python 3.11+ and `httpx==0.28.1`; `metrics.py` also needs `tokenizers`. + +## Format `s1-trace/1` + +JSONL with three record types: + +- **header**, first and once: `format`, `source` (repo, commit, front, argv), `server` + (repo, commit, worker, url, model), `capture`, `clock`, `content` (provenance, + license, redaction) and `images.store`. `capture` is `client:transport` for + `record_s1a.py` and `converted` for `tracefile.py from-ticks`. +- **request**, one per attempt: `id`, `group` (the episode), `step`, `attempt`, + `retry_of`, `request` (the body as sent, key order kept, inline images replaced by + `{"$image": ""}`), `images` (`sha256`, `pixels_sha256`, size, media type), + `t_send_ms`, `t_recv_ms`, `think_ms` (time since the previous response in the + episode), `deadline_s`, `status`, `response`, and free `notes` (a failure's error). +- **episode_end**: `group`, `status`, `steps`. + +`id`, `group` and `request` match the manifest rows of System1-Agents' proposed +changed-image replay (ThinkFlowLab/system1-agents#72). Steps and attempts increase within +an episode; a retry names the id it repeats, so identical resent bytes are not counted as +a repeated state. Token ids are not stored: they depend on each worker's rendering and +tokenizer. + +## Recording System1-Agents + +`record_s1a.py` wraps `JevDecisionsClient.decide`, the HTTP client behind `--model jev`, +and the tool loop's `run_episode`. Each run of the tool loop is an episode; other fronts +take `--group`, one episode per invocation. Run it with System1-Agents' Python, from its +checkout, against a decision server: + +```sh +TYPESAFE_API_KEY= TYPESAFE_API_URL=http://127.0.0.1:8000/v1/systemone TYPESAFE_MODEL=jev-latest \ +OPENROUTER_API_KEY=local .venv/bin/python /path/to/benchmarks/traces/record_s1a.py \ + blackjack.trace.jsonl --commit "$(git rev-parse HEAD)" -- run blackjack --model jev --rethink off --episodes 20 +``` + +A failed decision keeps its HTTP status when the error names one. + +## Metrics + +`metrics.py --layout open_jev|cua_s1 --export ` renders and tokenizes each +request as that native worker does. Open-Jev has one sequence per candidate with the +state first; its prompts match the worker's contract fixture. Cua-S1 has one sequence per +Choice question with the question's goal first, built by +`src/models/cua_s1/text/contract.py`. Requests the trace records as failed, malformed +requests, prompts over the worker's token limit, and anything Cua-S1's request mapping +rejects are counted as unserved and not measured. Per request: + +- questions, sequences and their tokens; +- tokens the request-local planner would not recompute (the rule of Decider's + `src/models/decider/native/src/prefix.rs`, 64-token chunks). Cua-S1 runs no planner, so + for it this share is hypothetical; +- tokens a cache of earlier prompts would skip on top of that: each span the plan computes + once (the request prefix, each group prefix, each sequence's rest) skips whatever an + earlier sequence already covers, in whole 64-token chunks, keeping each sequence's last + token. Earlier means the previous request of the episode, the last K of the episode, or + the last K the server received from any episode; +- whether the state (its bytes and its images' pixels) was seen at the previous step or + earlier, the best token similarity of its state to the last K states, and how many of + its images were seen before. + +The summary turns these into the share of prefill tokens each kind of reuse could skip. + +## Replay + +```sh +python benchmarks/traces/replay.py trace.jsonl --url http://127.0.0.1:8000 --out out/ +``` + +Episodes start at their recorded offsets and run concurrently; `--together` starts them +at once and `--sequential` runs them one after another. Offsets come from one recorder +clock, so replay traces concatenated from several recordings with `--sequential`. `--think-scale 0` sends each step +as soon as the last returns, `--skip-retries` drops repeated attempts, and +`--deadline-s` counts misses against a deadline other than the recorded one. Latency and +misses count answered requests only. The output has `config.json` (with the trace's +checksum), `responses.jsonl` (latency, status, deadline miss, `prefix_cache`, base64 +body) and `summary.json` (median and nearest-rank p95, as `bench.py` reports them). + +## Data + +Traces of real web pages copy third-party content. Keep them outside this repository +and commit only derived counts, or traces from local fixtures, self-hosted pages or +sets whose license allows it. diff --git a/benchmarks/traces/metrics.py b/benchmarks/traces/metrics.py new file mode 100644 index 00000000..3dca9be9 --- /dev/null +++ b/benchmarks/traces/metrics.py @@ -0,0 +1,308 @@ +#!/usr/bin/env python3 +"""Measure repeated work in a decision trace, as one worker would tokenize it. + +For each request the worker would serve: questions, sequences and tokens; the tokens +Decider's request-local planner (`src/models/decider/native/src/prefix.rs`) would not +recompute; the tokens a cache of earlier prompts would skip on top of that, in 64-token +chunks; whether its state or images were seen before. The summary adds these up into the +share of prefill tokens each kind of reuse could skip. + + python benchmarks/traces/metrics.py TRACE --layout open_jev --export weights/open-jev-9b-merged --out DIR +""" + +import argparse +import difflib +import importlib.util +import json +import statistics +import sys +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parent)) +import tracefile as traces # noqa: E402 + +CHUNK = 64 +ROOT = Path(__file__).resolve().parents[2] + + +def render(value): + """Open-Jev's rendering: text as is, anything else as Python's sorted json.dumps.""" + return value if isinstance(value, str) else json.dumps(value, ensure_ascii=False, sort_keys=True) + + +def open_jev_prompts(request): + """Candidate prompts per question, as `src/models/open_jev/native/src/contract.rs` builds + them; None for a malformed request. It does not repeat all of the worker's checks.""" + try: + state = render(request["state"]) + prompts = [] + for question in request["questions"].values(): + instructions = render(question["instructions"]) + criteria = question.get("criteria") + kind = question["type"] + if kind == "noul" and criteria is not None: + instructions += f"\nYes means: {render(criteria['true'])}\nNo means: {render(criteria['false'])}" + head = f"Context:\n{state}\n\nQuestion: {instructions}\n" + if kind == "noul": + prompts.append([head + "Is the answer to this question yes? Answer Yes or No."]) + continue + if kind == "choice": + options = [name if value is None else f"{name}: {render(value)}" for name, value in criteria.items()] + elif kind == "score": + options = [render(level) for level in criteria] + else: + return None + prompts.append([ + f"{head}Proposed answer: {option}\nIs this proposed answer correct? Answer Yes or No." + for option in options + ]) + return prompts if prompts and all(prompts) else None + except (KeyError, TypeError, AttributeError): + return None + + +class OpenJev: + """The native Open-Jev worker: one sequence per candidate, from its merged export.""" + + name = "open_jev" + + def __init__(self, export): + from tokenizers import Tokenizer + + manifest = json.loads((Path(export) / "open_jev_export.json").read_text()) + self.prefix, self.suffix = manifest["chat_prefix"], manifest["chat_suffix"] + self.max_length = manifest["max_length"] + self.tokenizer = Tokenizer.from_file(str(Path(export) / "tokenizer.json")) + + def encode(self, text, special=True): + return self.tokenizer.encode(text, add_special_tokens=special).ids + + def rows(self, request): + """Token ids per question, then per candidate; None when the worker would refuse it.""" + prompts = open_jev_prompts(request) + if prompts is None: + return None + rows = [[self.encode(f"{self.prefix}{p.strip()}{self.suffix}") for p in q] for q in prompts] + return rows if all(len(r) <= self.max_length for q in rows for r in q) else None + + def state(self, request): + return self.encode(render(request["state"]), special=False) + + +def cua_s1_contract(): + """The Cua-S1 worker's request mapping and prompts (stdlib only).""" + spec = importlib.util.spec_from_file_location("cua_s1_contract", ROOT / "src/models/cua_s1/text/contract.py") + module = importlib.util.module_from_spec(spec) + sys.modules[spec.name] = module # dataclasses look the module up by name + spec.loader.exec_module(module) + return module + + +class CuaS1: + """The native Cua-S1 text worker: one sequence per Choice question, its goal before the state. + It runs no prefix planner; its "request-local" share is what the planner would skip.""" + + name = "cua_s1" + + def __init__(self, export): + from tokenizers import Tokenizer + + self.contract = cua_s1_contract() + self.tokenizer = Tokenizer.from_file(str(Path(export) / "tokenizer.json")) + + def prompts(self, request): + """The chat text `chat_text` renders per question; None when the worker would refuse it.""" + try: + state, questions = self.contract.map_request(dict(request, model=self.contract.MODEL_NAME)) + except self.contract.RequestError: + return None + out = [] + for question in questions: + system, user = (m["content"] for m in self.contract.build_messages(state, question)) + out.append(f"<|im_start|>system\n{system}<|im_end|>\n<|im_start|>user\n{user}<|im_end|>\n" + "<|im_start|>assistant\n\n") + return out + + max_tokens = 16384 # MAX_PROMPT_TOKENS in src/models/cua_s1/native/src/processing.rs + + def rows(self, request): + prompts = self.prompts(request) + if prompts is None: + return None + rows = [[self.tokenizer.encode(p, add_special_tokens=False).ids] for p in prompts] + return rows if all(len(q[0]) <= self.max_tokens for q in rows) else None + + def state(self, request): + value = request["state"] + text = value if isinstance(value, str) else json.dumps(value, ensure_ascii=False) + return self.tokenizer.encode(text, add_special_tokens=False).ids + + +def common(rows, skip): + """`prefix.rs` `common`: tokens all rows share from `skip`, keeping one per row.""" + limit = min(len(r) for r in rows) - 1 + first, n = rows[0], skip + while n < limit and all(r[n] == first[n] for r in rows): + n += 1 + return max(n - skip, 0) + + +def plan(groups): + """`PrefixPlan::new` on rows grouped by question: (request prefix, its 64-token floor, each group's + floored prefix end, saved tokens). Without a plan the floors are 0 and nothing is saved.""" + rows = [r for g in groups for r in g] + if len(rows) < 2 or any(not r for r in rows): + return 0, 0, [0] * len(groups), 0 + request = common(rows, 0) + p = request // CHUNK * CHUNK + saved = (len(rows) - 1) * p + ends = [] + for group in groups: + prefix = common(group, request) if len(group) > 1 else 0 + ends.append((request + prefix) // CHUNK * CHUNK) + saved += (len(group) - 1) * (ends[-1] - p) + if not saved: + return request, 0, [0] * len(groups), 0 + return request, p, ends, saved + + +def worth_auto(saved, total): + return saved >= 4096 and saved >= -(-total // 3) + + +def lcp(a, b): + n = 0 + for x, y in zip(a, b): + if x != y: + break + n += 1 + return n + + +def cached(groups, p, ends, earlier): + """Tokens a cache of earlier prompts, restored at 64-token boundaries, would skip on top of the plan: + each span the plan computes once (the request prefix, each group prefix, each row's rest) skips what + a cached prompt already covers. A row keeps at least its last token.""" + if not earlier: + return 0 + cover = [[max(lcp(row, e) for e in earlier) // CHUNK * CHUNK for row in group] for group in groups] + saved = min(max(c for g in cover for c in g), p) + for group, end, c in zip(groups, ends, cover): + if end > p: + saved += max(0, min(max(c), end) - p) + start = end if end else p + saved += sum(max(0, min(ci, (len(row) - 1) // CHUNK * CHUNK) - start) for row, ci in zip(group, c)) + return saved + + +def served(record): + """Whether the recorded server answered: a recorded failure is not measured.""" + return record.get("status") in (None, 200) and "error" not in (record.get("notes") or {}) + + +def measure(requests, layout, last_k=8): + """One row per request the worker would serve, in trace order.""" + if last_k < 1: + raise ValueError("last_k must be at least 1") + out, history, arrivals = [], {}, [] + for record in requests: + groups = layout.rows(record["request"]) if served(record) else None + if not groups: + continue + first = groups[0][0] + total = sum(len(r) for g in groups for r in g) + request_prefix, p, ends, saved = plan(groups) + state = traces.state_digest(record["request"], record.get("images", [])) + state_ids = layout.state(record["request"]) + images = [i["pixels_sha256"] for i in record.get("images", [])] + earlier = history.setdefault(record["group"], []) + row = { + "id": record["id"], "group": record["group"], "step": record["step"], + "attempt": record.get("attempt", 0), + "questions": len(groups), "sequences": sum(len(g) for g in groups), + "tokens": total, "request_prefix": request_prefix, "local_saved": saved, + "auto": worth_auto(saved, total), + } + if row["attempt"] == 0: # a retry resends the same bytes; it is not compared + previous = earlier[-1] if earlier else None + window = earlier[-last_k:] + near = max((difflib.SequenceMatcher(None, e["state_ids"], state_ids, autojunk=False).ratio() + for e in window), default=0.0) + row.update({ + "prev_lcp64": lcp(previous["first"], first) // CHUNK * CHUNK if previous else 0, + "cross_saved_prev": cached(groups, p, ends, previous["rows"] if previous else []), + "cross_saved_best": cached(groups, p, ends, [r for e in window for r in e["rows"]]), + # what a server-side cache of the last K requests sees, whatever episode they came from + "cross_saved_server": cached(groups, p, ends, [r for e in arrivals[-last_k:] for r in e["rows"]]), + "state_seen_prev": bool(previous and previous["state"] == state), + "state_seen_before": any(e["state"] == state for e in earlier), + "near_duplicate": round(near, 4), + "images": len(images), + "images_seen_prev": sum(i in (previous["images"] if previous else ()) for i in images), + "images_seen_before": sum(any(i in e["images"] for e in earlier) for i in images), + }) + earlier.append({"first": first, "state": state, "state_ids": state_ids, "images": images, + "rows": [r for g in groups for r in g]}) + arrivals.append(earlier[-1]) + out.append(row) + return out + + +def summarize(rows): + firsts = [r for r in rows if r["attempt"] == 0] + total = sum(r["tokens"] for r in rows) + local = sum(r["local_saved"] for r in rows) + images = sum(r["images"] for r in firsts) + + def share(n, d): + return round(n / d, 4) if d else None + + def mean(values): + return statistics.mean(values) if values else None + + return { + "requests": len(rows), "retries": len(rows) - len(firsts), + "episodes": len({r["group"] for r in rows}), + "questions_per_request": mean([r["questions"] for r in rows]), + "sequences_per_request": mean([r["sequences"] for r in rows]), + "tokens": total, + "skipped_share": { + "request_local": share(local, total), + "plus_cache_of_previous_request": share(local + sum(r["cross_saved_prev"] for r in firsts), total), + "plus_cache_of_last_k_requests": share(local + sum(r["cross_saved_best"] for r in firsts), total), + "plus_server_cache_of_last_k": share(local + sum(r["cross_saved_server"] for r in firsts), total), + }, + "auto_requests": sum(r["auto"] for r in rows), + "state_seen_prev": share(sum(r["state_seen_prev"] for r in firsts), len(firsts)), + "state_seen_before": share(sum(r["state_seen_before"] for r in firsts), len(firsts)), + "near_duplicate_0_9": share(sum(r["near_duplicate"] >= 0.9 for r in firsts), len(firsts)), + "images": images, + "images_seen_prev": share(sum(r["images_seen_prev"] for r in firsts), images), + "images_seen_before": share(sum(r["images_seen_before"] for r in firsts), images), + } + + +def main(): + parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + parser.add_argument("trace", type=Path) + parser.add_argument("--layout", choices=("open_jev", "cua_s1"), default="open_jev") + parser.add_argument("--export", type=Path, required=True, + help="the worker's export directory (its tokenizer.json; Open-Jev also its manifest)") + parser.add_argument("--last-k", type=int, default=8) + parser.add_argument("--out", type=Path, required=True) + args = parser.parse_args() + if args.last_k < 1: + parser.error("--last-k must be at least 1") + _, requests, _ = traces.read(args.trace) + layout = (OpenJev if args.layout == "open_jev" else CuaS1)(args.export) + rows = measure(requests, layout, args.last_k) + args.out.mkdir(parents=True, exist_ok=True) + (args.out / "requests.jsonl").write_text("".join(json.dumps(r) + "\n" for r in rows)) + summary = {"trace": str(args.trace), "layout": layout.name, "last_k": args.last_k, + "unserved_requests": len(requests) - len(rows), **summarize(rows)} + (args.out / "summary.json").write_text(json.dumps(summary, indent=2) + "\n") + print(json.dumps(summary, indent=2)) + + +if __name__ == "__main__": + main() diff --git a/benchmarks/traces/record_s1a.py b/benchmarks/traces/record_s1a.py new file mode 100644 index 00000000..26fd18e8 --- /dev/null +++ b/benchmarks/traces/record_s1a.py @@ -0,0 +1,129 @@ +#!/usr/bin/env python3 +"""Record System1-Agents' decision requests as an s1-trace/1 file, without changing its source. + + python record_s1a.py OUT.jsonl --commit SHA -- run blackjack --model jev --rethink off --episodes 20 + python record_s1a.py OUT.jsonl --commit SHA --group flights-1 -- run flights --model jev + python record_s1a.py OUT.jsonl --commit SHA -- -m evals.recovery.live --model jev --repeat 1 + +Wraps `JevDecisionsClient.decide`, the HTTP client behind `--model jev` (the request body as +sent, the payload, timing; retries inside one decision are not separate records), and +`s1a.tool.loop.run_episode` (episode boundaries). Run it with System1-Agents' environment +and `TYPESAFE_API_URL` pointing at the decision server, from System1-Agents' checkout with +its virtual environment's Python. Episodes must run one at a time, as `s1a run` does. +""" + +import argparse +import contextvars +import json +import os +import re +import sys +import time + +parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) +parser.add_argument("out") +parser.add_argument("--commit", required=True, help="System1-Agents commit") +parser.add_argument("--server-commit", help="system1-omni commit of the decision server") +parser.add_argument("--worker", help="decision worker, e.g. open_jev") +parser.add_argument("--group", default="run", help="episode name for requests outside the tool loop") +if "--" not in sys.argv: + parser.error("put the s1a arguments after --") +split = sys.argv.index("--") +args = parser.parse_args(sys.argv[1:split]) +s1a_args = sys.argv[split + 1:] +if not s1a_args or s1a_args == ["-m"]: + parser.error("name the s1a command, or -m and a module, after --") + +from s1a import console # noqa: E402,F401 (first, as s1a.entry does: routes logs before openjiuwen loads) +from s1a.decision_models import wire # noqa: E402 +from s1a.tool import loop # noqa: E402 + +out = open(args.out, "w", buffering=1) +clock = time.perf_counter() +episode = contextvars.ContextVar("episode", default=args.group) +steps, last_recv, started = {}, {}, set() + + +def now_ms(): + return round((time.perf_counter() - clock) * 1000, 3) + + +def emit(record): + out.write(json.dumps(record, ensure_ascii=False) + "\n") + + +emit({ + "type": "header", "format": "s1-trace/1", + "source": {"repo": "ThinkFlowLab/system1-agents", "commit": args.commit, "front": " ".join(s1a_args[:2]), + "argv": s1a_args}, + "server": {"repo": "InferMatrixLab/system1-omni", "commit": args.server_commit, "worker": args.worker, + "url": os.environ.get("TYPESAFE_API_URL"), "model": os.environ.get("TYPESAFE_MODEL")}, + "capture": "client:transport", "clock": "ms since the recorder started (perf_counter)", + "content": {}, "images": {"store": None}, +}) + +decide = wire.JevDecisionsClient.decide + + +async def recorded_decide(self, body): + group = episode.get() + step = steps.get(group, 0) + steps[group] = step + 1 + started.add(group) + t_send = now_ms() + record = { + "type": "request", "id": f"{group}:s{step}:a0", "group": group, "step": step, "attempt": 0, + "retry_of": None, "request": body, "images": [], "t_send_ms": t_send, "t_recv_ms": None, + "think_ms": round(t_send - last_recv[group], 3) if group in last_recv else None, + "deadline_s": self._timeout_s, "status": None, "response": None, "notes": {}, + } + try: + payload, latency_ms = await decide(self, body) + record.update(status=200, response=payload, notes={"client_latency_ms": latency_ms}) + return payload, latency_ms + except BaseException as e: # a timed-out episode cancels the decision in flight + record["notes"] = {"error": f"{type(e).__name__}: {str(e)[:200]}"} + status = re.search(r"HTTP (\d{3})", str(e)) + record["status"] = int(status.group(1)) if status else None + raise + finally: + record["t_recv_ms"] = last_recv[group] = now_ms() + emit(record) + + +wire.JevDecisionsClient.decide = recorded_decide +run_episode = loop.run_episode +episodes = iter(range(10**9)) + + +async def recorded_run_episode(spec, env, *, seed, **kwargs): + group = f"{spec.name}-seed{seed}-{next(episodes)}" + token = episode.set(group) + status = "error" + try: + result = await run_episode(spec, env, seed=seed, **kwargs) + status = "error" if result.error else (result.extra.get("result_type") or "done") + return result + finally: + episode.reset(token) + if group in started: + emit({"type": "episode_end", "group": group, "status": str(status), "steps": steps[group]}) + + +loop.run_episode = recorded_run_episode # before s1a.tool.series imports it by name + +try: + if s1a_args[:1] == ["-m"]: + import runpy + + sys.argv = [s1a_args[1], *s1a_args[2:]] + runpy.run_module(s1a_args[1], run_name="__main__", alter_sys=True) + else: + from s1a import entry + + sys.argv = ["s1a", *s1a_args] + entry.s1a() +finally: + if args.group in started: + emit({"type": "episode_end", "group": args.group, "status": None, "steps": steps[args.group]}) + out.close() diff --git a/benchmarks/traces/replay.py b/benchmarks/traces/replay.py new file mode 100644 index 00000000..33fa41cf --- /dev/null +++ b/benchmarks/traces/replay.py @@ -0,0 +1,160 @@ +#!/usr/bin/env python3 +"""Replay a decision trace against a running server, keeping each episode's order and pace. + +Each episode is one client: it sends its requests one at a time, in trace order, and +waits `think_ms * --think-scale` after each response before the next, as the agent did. +Episodes start at their recorded offsets (or together) and run concurrently. Unlike +`bench.py`, nothing is compared with expected answers: the output is per-request latency, +deadline misses and the worker's `prefix_cache` metadata. + + python benchmarks/traces/replay.py TRACE --url http://127.0.0.1:8000 --out DIR +""" + +import argparse +import asyncio +import base64 +import json +import math +import platform +import statistics +import sys +import time +from pathlib import Path + +import httpx + +sys.path.insert(0, str(Path(__file__).resolve().parent)) +sys.path.insert(0, str(Path(__file__).resolve().parents[2])) +import tracefile # noqa: E402 +from benchmarks.bench import RESPONSE_FORMAT, digest, write_json # noqa: E402 + + +def episodes(requests, skip_retries): + out = {} + for record in requests: + if skip_retries and record.get("attempt", 0): + continue + out.setdefault(record["group"], []).append(record) + return out + + +def offsets(groups, together): + """Seconds after the replay starts at which each episode sends its first request.""" + starts = {g: rs[0].get("t_send_ms") for g, rs in groups.items()} + if together or any(v is None for v in starts.values()): + return {g: 0.0 for g in groups} + first = min(starts.values()) + return {g: (v - first) / 1000 for g, v in starts.items()} + + +async def run_episode(client, url, model, records, delay, scale, clock, sink, deadline_s=None): + await asyncio.sleep(delay) + for i, record in enumerate(records): + if i: + await asyncio.sleep((record.get("think_ms") or 0) * scale / 1000) + body = dict(record["request"]) + if model: + body["model"] = model + sent = time.perf_counter() + error = None + try: + response = await client.post(url, json=body) + status, content = response.status_code, response.content + except httpx.HTTPError as e: + status, content, error = None, b"", f"{type(e).__name__}: {e}" + latency = (time.perf_counter() - sent) * 1000 + prefix_cache = None + if status == 200: + try: + prefix_cache = json.loads(content).get("metadata", {}).get("prefix_cache") + except (ValueError, AttributeError): + pass + deadline = deadline_s if deadline_s is not None else record.get("deadline_s") + answered = status == 200 + sink.append({ + "id": record["id"], "group": record["group"], "step": record["step"], + "attempt": record.get("attempt", 0), "sent_s": round(sent - clock, 6), + "latency_ms": round(latency, 3), "status": status, "error": error, + "deadline_s": deadline, + "missed": latency > deadline * 1000 if answered and deadline is not None else None, + "prefix_cache": prefix_cache, "body": base64.b64encode(content).decode(), + }) + + +def summarize(records, elapsed): + """Latency of answered requests, with `bench.py`'s median and nearest-rank p95.""" + ok = [r for r in records if r["status"] == 200] + latencies = sorted(r["latency_ms"] for r in ok) + with_deadline = [r for r in ok if r["missed"] is not None] + shared = [r for r in ok if (r["prefix_cache"] or {}).get("enabled")] + return { + "requests": len(records), "ok": len(ok), "failed": len(records) - len(ok), + "elapsed_s": round(elapsed, 3), + "latency_ms": { + "p50": statistics.median(latencies) if latencies else None, + "p95": latencies[math.ceil(0.95 * len(latencies)) - 1] if latencies else None, + "max": latencies[-1] if latencies else None, + "mean": round(statistics.mean(latencies), 3) if latencies else None, + }, + "deadline_misses": sum(r["missed"] for r in with_deadline), + "answered_with_deadline": len(with_deadline), + "shared_prefix_requests": len(shared), + } + + +async def replay(args): + _, requests, _ = tracefile.read(args.trace) + groups = episodes(requests, args.skip_retries) + starts = offsets(groups, args.together) + url = args.url.rstrip("/") + "/v1/systemone" + sink = [] + clock = time.perf_counter() + limits = httpx.Limits(max_connections=len(groups), max_keepalive_connections=len(groups)) + async with httpx.AsyncClient(timeout=args.timeout, trust_env=False, limits=limits) as client: + if args.sequential: + for records in groups.values(): + await run_episode(client, url, args.model, records, 0.0, args.think_scale, clock, sink, args.deadline_s) + else: + await asyncio.gather(*( + run_episode(client, url, args.model, records, starts[g], args.think_scale, clock, sink, args.deadline_s) + for g, records in groups.items() + )) + return sink, time.perf_counter() - clock + + +def main(): + parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + parser.add_argument("trace", type=Path) + parser.add_argument("--url", required=True) + parser.add_argument("--model", help="model alias added to every request") + parser.add_argument("--think-scale", type=float, default=1.0, help="0 sends each step right after the last") + parser.add_argument("--together", action="store_true", help="start every episode at once") + parser.add_argument("--sequential", action="store_true", help="run the episodes one after another") + parser.add_argument("--deadline-s", type=float, help="count misses against this deadline, not the trace's") + parser.add_argument("--skip-retries", action="store_true", help="send first attempts only") + parser.add_argument("--timeout", type=float, default=600.0) + parser.add_argument("--out", type=Path, required=True) + args = parser.parse_args() + if args.think_scale < 0: + parser.error("--think-scale must be nonnegative") + if args.sequential and args.together: + parser.error("--sequential and --together exclude each other") + args.out.mkdir(parents=True, exist_ok=True) + write_json(args.out / "config.json", { + "trace": str(args.trace), "trace_sha256": digest(args.trace), "url": args.url, + "model": args.model, "think_scale": args.think_scale, "together": args.together, + "sequential": args.sequential, "deadline_s": args.deadline_s, + "skip_retries": args.skip_retries, "timeout_s": args.timeout, + "response_format": RESPONSE_FORMAT, "python": platform.python_version(), + "httpx": httpx.__version__, + }) + records, elapsed = asyncio.run(replay(args)) + records.sort(key=lambda r: r["sent_s"]) + (args.out / "responses.jsonl").write_text("".join(json.dumps(r) + "\n" for r in records)) + summary = summarize(records, elapsed) + write_json(args.out / "summary.json", summary) + print(json.dumps(summary, indent=2)) + + +if __name__ == "__main__": + main() diff --git a/benchmarks/traces/results-2026-10-25.md b/benchmarks/traces/results-2026-10-25.md new file mode 100644 index 00000000..6b329ea9 --- /dev/null +++ b/benchmarks/traces/results-2026-10-25.md @@ -0,0 +1,63 @@ +# Repeated work in System1-Agents traces, 2026-10-25 + +Recorded on one NVIDIA RTX 6000 Ada with [`record_s1a.py`](record_s1a.py) from +System1-Agents `9b20b98`. Open-Jev-9B decided: the native worker with #179's change, no +prefix switch, the merged export's default 4,096-token limit. Fronts that need a chat +model used Qwen3-8B through vLLM 0.32 with thinking off. Flights ran against live Google +Flights (`hl=en&gl=CH`, one route per episode); the form fixtures are +`evals.recovery.live`'s three tasks with recovery off and on. Each Flights and form +episode was one invocation with its own `--group`, and the invocations were concatenated +into one trace per front; each keeps its own clock, so replay these with `--sequential`. `zurich-london` is System1-Agents' own recording of a Jev 1.13 +run (`docs/results/laya-browser-shaping/zurich-london.jsonl`), converted with +`tracefile.py from-ticks`. + +Only the [samples](samples/) are in this repository; the other traces copy third-party +pages or prompt text and stay outside it. The ticket-router row can be recomputed from +its sample. + +## Shares of prefill tokens that could be skipped + +| Trace | Episodes / measured decisions (unserved) | Open-Jev: within a request | + server cache, last 8 | Cua-S1: within a request | + server cache, last 8 | +| --- | ---: | ---: | ---: | ---: | ---: | +| Google Flights, live | 28 / 93 (5) | 89.2% | 90.8% | 4.4% | 31.8% | +| `zurich-london` | 1 / 10 (2); Cua-S1 9 (3) | 91.5% | 91.9% | 4.7% | 35.6% | +| Form fixtures | 36 / 108 | 60.5% | 61.6% | 9.5% | 49.4% | +| 2048 | 5 / 800 | 58.2% | 59.7% | 0% | 46.6% | +| Ticket router | 1 / 30 | 59.8% | 59.8% | 0% | 45.3% | +| Blackjack | 100 / 119 | 41.8% | 41.8% | 0% | 77.2% | + +Each layout is measured on its own tokens. Unserved requests are those recorded as +failed (the five Flights results pages over 4,096 tokens) or that the worker would refuse +(two `zurich-london` pages over Open-Jev-9B's limit; three with more than 26 click +targets for Cua-S1). The Cua-S1 "within a request" share is what Decider's planner would +skip; the Cua-S1 worker runs none. + +- No state came back exactly in any trace. +- Where the state comes first (Open-Jev), sharing within a request is nearly all of the + reuse; a cache of recent prompts adds at most 1.6 points. +- Where each question's goal comes first (Cua-S1), one sequence per question leaves + little to share within a request, but the system prompt and the goal repeat at every + step, so a cache of recent prompts could skip 32 to 77%. Blackjack's 77% needs a + server-side cache that ignores episode boundaries: each hand is its own short episode + (84 of 100 take one decision); within episodes it is 12%. + +## Pace + +| Trace | Think time p50 / p90 | Decision latency p50 / p90 (recorded) | +| --- | ---: | ---: | +| Google Flights | 0.67 / 6.7 s | 6.4 / 7.7 s | +| Form fixtures | 1.2 / 3.4 s | 1.07 / 1.27 s | +| 2048 | 0.67 / 0.68 s | 0.14 / 0.15 s | +| Ticket router, blackjack (no chat model) | 1 to 3 ms | 0.08 to 0.23 s | + +Agents send one request at a time; without a chat model the next request follows the +last response within milliseconds. + +## Caveats + +- Trajectories depend on the deciding model. Open-Jev-9B ends most Flights episodes at + the second decision (22 of 28), as System1-Agents' A40 runs also show, so Flights + episodes are short. +- Flights results pages can exceed the 9B export's 4,096-token default; the export can be + made with a longer `--max-length`. +- One GPU, one decision model, one chat model, one day. diff --git a/benchmarks/traces/samples/README.md b/benchmarks/traces/samples/README.md new file mode 100644 index 00000000..48ef4310 --- /dev/null +++ b/benchmarks/traces/samples/README.md @@ -0,0 +1,19 @@ +# Sample traces + +Recorded on 2026-10-25 with [`record_s1a.py`](../record_s1a.py) from +[System1-Agents](https://github.com/ThinkFlowLab/system1-agents) at `9b20b98`, against the +native Open-Jev-9B worker on one RTX 6000 Ada: + +- `blackjack.trace.jsonl`: the first 20 hands of a 100-hand + `s1a run blackjack --model jev --rethink off`. +- `ticket_router.trace.jsonl`: one run of `s1a run ticket_router --model jev --rethink off` + over System1-Agents' 30 synthetic tickets. + +The request records are as the recorder wrote them; that day's recorder marked every +finished episode `done`. The headers' `source`, `server` and `content` were filled in +afterwards, and the blackjack trace was cut to 20 episodes. + +The prompt text and tickets are System1-Agents' work under the Apache License 2.0 +([LICENSE](https://github.com/ThinkFlowLab/system1-agents/blob/9b20b989d919bc69227f81bc0ec571ca56925586/LICENSE), +[NOTICE](https://github.com/ThinkFlowLab/system1-agents/blob/9b20b989d919bc69227f81bc0ec571ca56925586/NOTICE)); +the blackjack states come from RLCard. diff --git a/benchmarks/traces/samples/blackjack.trace.jsonl b/benchmarks/traces/samples/blackjack.trace.jsonl new file mode 100644 index 00000000..cc473c95 --- /dev/null +++ b/benchmarks/traces/samples/blackjack.trace.jsonl @@ -0,0 +1,41 @@ +{"type": "header", "format": "s1-trace/1", "source": {"repo": "ThinkFlowLab/system1-agents", "commit": "9b20b989d919bc69227f81bc0ec571ca56925586", "front": "run blackjack", "argv": ["run", "blackjack", "--model", "jev", "--rethink", "off", "--episodes", "100"]}, "server": {"repo": "InferMatrixLab/system1-omni", "commit": null, "worker": "open_jev", "url": "http://127.0.0.1:8840/v1/systemone", "model": "jev-latest", "notes": "native Open-Jev-9B worker built with #179's change (a local build), no prefix switch"}, "capture": "client:transport", "clock": "ms since the recorder started (perf_counter)", "content": {"provenance": "System1-Agents blackjack agent (s1a run blackjack --model jev --rethink off), first 20 of 100 episodes", "license": "Apache-2.0 (System1-Agents prompt text); RLCard game states", "redaction": "none needed"}, "images": {"store": null}} +{"type": "request", "id": "blackjack-seed0-0:s0:a0", "group": "blackjack-seed0-0", "step": 0, "attempt": 0, "retry_of": null, "request": {"model": "jev-latest", "state": {"player_cards": ["C8", "D6"], "player_total": 14, "soft_hand": false, "dealer_showing": ["H5"]}, "questions": {"pick": {"type": "choice", "criteria": {"hit": "hit (blackjack action)", "stand": "stand (blackjack action)"}, "instructions": {"rules": "Blackjack against a dealer who hits until 17. Aim to beat the dealer's final total without exceeding 21. Cards are written suit then rank (H7 = seven of hearts, SA = ace of spades, DT = ten of diamonds). Basic strategy: stand on hard 17 or more; stand on hard 13 to 16 when the dealer shows 2 to 6, otherwise hit; stand on hard 12 only against a dealer 4 to 6; always hit hard 11 or less; hit soft 17 or less; stand on soft 19 or more; stand on soft 18 unless the dealer shows 9, ten or ace."}}}}, "images": [], "t_send_ms": 175.409, "t_recv_ms": 233.305, "think_ms": null, "deadline_s": 30.0, "status": 200, "response": {"model": "Qwen/Qwen3.5-9B", "answers": {"pick": {"type": "choice", "choice": "stand", "probabilities": {"hit": 0.054384223743739364, "stand": 0.9456157762562606}, "confidence": 0.8912315525125212}}, "usage": {"input_tokens": 460, "output_tokens": 0}, "metadata": {"method": "native_merged_lora_decision_head", "temperature": 1.8969118766347646, "candidate_sequences": 2, "inference_seconds": 0.056598375, "base_revision": "c202236235762e1c871ad0ccb60c8ee5ba337b9a", "max_length": 4096, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}}}, "notes": {"client_latency_ms": 58}} +{"type": "episode_end", "group": "blackjack-seed0-0", "status": "done", "steps": 1} +{"type": "request", "id": "blackjack-seed1-1:s0:a0", "group": "blackjack-seed1-1", "step": 0, "attempt": 0, "retry_of": null, "request": {"model": "jev-latest", "state": {"player_cards": ["S5", "SJ"], "player_total": 15, "soft_hand": false, "dealer_showing": ["H4"]}, "questions": {"pick": {"type": "choice", "criteria": {"hit": "hit (blackjack action)", "stand": "stand (blackjack action)"}, "instructions": {"rules": "Blackjack against a dealer who hits until 17. Aim to beat the dealer's final total without exceeding 21. Cards are written suit then rank (H7 = seven of hearts, SA = ace of spades, DT = ten of diamonds). Basic strategy: stand on hard 17 or more; stand on hard 13 to 16 when the dealer shows 2 to 6, otherwise hit; stand on hard 12 only against a dealer 4 to 6; always hit hard 11 or less; hit soft 17 or less; stand on soft 19 or more; stand on soft 18 unless the dealer shows 9, ten or ace."}}}}, "images": [], "t_send_ms": 384.523, "t_recv_ms": 442.28, "think_ms": null, "deadline_s": 30.0, "status": 200, "response": {"model": "Qwen/Qwen3.5-9B", "answers": {"pick": {"type": "choice", "choice": "stand", "probabilities": {"hit": 0.19952293374735547, "stand": 0.8004770662526446}, "confidence": 0.6009541325052892}}, "usage": {"input_tokens": 458, "output_tokens": 0}, "metadata": {"method": "native_merged_lora_decision_head", "temperature": 1.8969118766347646, "candidate_sequences": 2, "inference_seconds": 0.056393552, "base_revision": "c202236235762e1c871ad0ccb60c8ee5ba337b9a", "max_length": 4096, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}}}, "notes": {"client_latency_ms": 58}} +{"type": "episode_end", "group": "blackjack-seed1-1", "status": "done", "steps": 1} +{"type": "request", "id": "blackjack-seed2-2:s0:a0", "group": "blackjack-seed2-2", "step": 0, "attempt": 0, "retry_of": null, "request": {"model": "jev-latest", "state": {"player_cards": ["DJ", "CQ"], "player_total": 20, "soft_hand": false, "dealer_showing": ["S2"]}, "questions": {"pick": {"type": "choice", "criteria": {"hit": "hit (blackjack action)", "stand": "stand (blackjack action)"}, "instructions": {"rules": "Blackjack against a dealer who hits until 17. Aim to beat the dealer's final total without exceeding 21. Cards are written suit then rank (H7 = seven of hearts, SA = ace of spades, DT = ten of diamonds). Basic strategy: stand on hard 17 or more; stand on hard 13 to 16 when the dealer shows 2 to 6, otherwise hit; stand on hard 12 only against a dealer 4 to 6; always hit hard 11 or less; hit soft 17 or less; stand on soft 19 or more; stand on soft 18 unless the dealer shows 9, ten or ace."}}}}, "images": [], "t_send_ms": 450.12, "t_recv_ms": 517.106, "think_ms": null, "deadline_s": 30.0, "status": 200, "response": {"model": "Qwen/Qwen3.5-9B", "answers": {"pick": {"type": "choice", "choice": "stand", "probabilities": {"hit": 0.01232915406597062, "stand": 0.9876708459340294}, "confidence": 0.9753416918680589}}, "usage": {"input_tokens": 458, "output_tokens": 0}, "metadata": {"method": "native_merged_lora_decision_head", "temperature": 1.8969118766347646, "candidate_sequences": 2, "inference_seconds": 0.065860559, "base_revision": "c202236235762e1c871ad0ccb60c8ee5ba337b9a", "max_length": 4096, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}}}, "notes": {"client_latency_ms": 67}} +{"type": "episode_end", "group": "blackjack-seed2-2", "status": "done", "steps": 1} +{"type": "request", "id": "blackjack-seed3-3:s0:a0", "group": "blackjack-seed3-3", "step": 0, "attempt": 0, "retry_of": null, "request": {"model": "jev-latest", "state": {"player_cards": ["H2", "DK"], "player_total": 12, "soft_hand": false, "dealer_showing": ["C5"]}, "questions": {"pick": {"type": "choice", "criteria": {"hit": "hit (blackjack action)", "stand": "stand (blackjack action)"}, "instructions": {"rules": "Blackjack against a dealer who hits until 17. Aim to beat the dealer's final total without exceeding 21. Cards are written suit then rank (H7 = seven of hearts, SA = ace of spades, DT = ten of diamonds). Basic strategy: stand on hard 17 or more; stand on hard 13 to 16 when the dealer shows 2 to 6, otherwise hit; stand on hard 12 only against a dealer 4 to 6; always hit hard 11 or less; hit soft 17 or less; stand on soft 19 or more; stand on soft 18 unless the dealer shows 9, ten or ace."}}}}, "images": [], "t_send_ms": 526.995, "t_recv_ms": 599.408, "think_ms": null, "deadline_s": 30.0, "status": 200, "response": {"model": "Qwen/Qwen3.5-9B", "answers": {"pick": {"type": "choice", "choice": "stand", "probabilities": {"hit": 0.4582018171018082, "stand": 0.5417981828981918}, "confidence": 0.08359636579638363}}, "usage": {"input_tokens": 458, "output_tokens": 0}, "metadata": {"method": "native_merged_lora_decision_head", "temperature": 1.8969118766347646, "candidate_sequences": 2, "inference_seconds": 0.071211664, "base_revision": "c202236235762e1c871ad0ccb60c8ee5ba337b9a", "max_length": 4096, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}}}, "notes": {"client_latency_ms": 72}} +{"type": "episode_end", "group": "blackjack-seed3-3", "status": "done", "steps": 1} +{"type": "request", "id": "blackjack-seed4-4:s0:a0", "group": "blackjack-seed4-4", "step": 0, "attempt": 0, "retry_of": null, "request": {"model": "jev-latest", "state": {"player_cards": ["CT", "CA"], "player_total": 21, "soft_hand": true, "dealer_showing": ["C9"]}, "questions": {"pick": {"type": "choice", "criteria": {"hit": "hit (blackjack action)", "stand": "stand (blackjack action)"}, "instructions": {"rules": "Blackjack against a dealer who hits until 17. Aim to beat the dealer's final total without exceeding 21. Cards are written suit then rank (H7 = seven of hearts, SA = ace of spades, DT = ten of diamonds). Basic strategy: stand on hard 17 or more; stand on hard 13 to 16 when the dealer shows 2 to 6, otherwise hit; stand on hard 12 only against a dealer 4 to 6; always hit hard 11 or less; hit soft 17 or less; stand on soft 19 or more; stand on soft 18 unless the dealer shows 9, ten or ace."}}}}, "images": [], "t_send_ms": 606.941, "t_recv_ms": 682.342, "think_ms": null, "deadline_s": 30.0, "status": 200, "response": {"model": "Qwen/Qwen3.5-9B", "answers": {"pick": {"type": "choice", "choice": "stand", "probabilities": {"hit": 0.04240589229281261, "stand": 0.9575941077071874}, "confidence": 0.9151882154143749}}, "usage": {"input_tokens": 456, "output_tokens": 0}, "metadata": {"method": "native_merged_lora_decision_head", "temperature": 1.8969118766347646, "candidate_sequences": 2, "inference_seconds": 0.074331155, "base_revision": "c202236235762e1c871ad0ccb60c8ee5ba337b9a", "max_length": 4096, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}}}, "notes": {"client_latency_ms": 75}} +{"type": "episode_end", "group": "blackjack-seed4-4", "status": "done", "steps": 1} +{"type": "request", "id": "blackjack-seed5-5:s0:a0", "group": "blackjack-seed5-5", "step": 0, "attempt": 0, "retry_of": null, "request": {"model": "jev-latest", "state": {"player_cards": ["HJ", "D3"], "player_total": 13, "soft_hand": false, "dealer_showing": ["H2"]}, "questions": {"pick": {"type": "choice", "criteria": {"hit": "hit (blackjack action)", "stand": "stand (blackjack action)"}, "instructions": {"rules": "Blackjack against a dealer who hits until 17. Aim to beat the dealer's final total without exceeding 21. Cards are written suit then rank (H7 = seven of hearts, SA = ace of spades, DT = ten of diamonds). Basic strategy: stand on hard 17 or more; stand on hard 13 to 16 when the dealer shows 2 to 6, otherwise hit; stand on hard 12 only against a dealer 4 to 6; always hit hard 11 or less; hit soft 17 or less; stand on soft 19 or more; stand on soft 18 unless the dealer shows 9, ten or ace."}}}}, "images": [], "t_send_ms": 689.512, "t_recv_ms": 765.251, "think_ms": null, "deadline_s": 30.0, "status": 200, "response": {"model": "Qwen/Qwen3.5-9B", "answers": {"pick": {"type": "choice", "choice": "stand", "probabilities": {"hit": 0.04818642155726929, "stand": 0.9518135784427308}, "confidence": 0.9036271568854617}}, "usage": {"input_tokens": 460, "output_tokens": 0}, "metadata": {"method": "native_merged_lora_decision_head", "temperature": 1.8969118766347646, "candidate_sequences": 2, "inference_seconds": 0.074754476, "base_revision": "c202236235762e1c871ad0ccb60c8ee5ba337b9a", "max_length": 4096, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}}}, "notes": {"client_latency_ms": 76}} +{"type": "episode_end", "group": "blackjack-seed5-5", "status": "done", "steps": 1} +{"type": "request", "id": "blackjack-seed6-6:s0:a0", "group": "blackjack-seed6-6", "step": 0, "attempt": 0, "retry_of": null, "request": {"model": "jev-latest", "state": {"player_cards": ["ST", "HJ"], "player_total": 20, "soft_hand": false, "dealer_showing": ["H3"]}, "questions": {"pick": {"type": "choice", "criteria": {"hit": "hit (blackjack action)", "stand": "stand (blackjack action)"}, "instructions": {"rules": "Blackjack against a dealer who hits until 17. Aim to beat the dealer's final total without exceeding 21. Cards are written suit then rank (H7 = seven of hearts, SA = ace of spades, DT = ten of diamonds). Basic strategy: stand on hard 17 or more; stand on hard 13 to 16 when the dealer shows 2 to 6, otherwise hit; stand on hard 12 only against a dealer 4 to 6; always hit hard 11 or less; hit soft 17 or less; stand on soft 19 or more; stand on soft 18 unless the dealer shows 9, ten or ace."}}}}, "images": [], "t_send_ms": 772.51, "t_recv_ms": 848.586, "think_ms": null, "deadline_s": 30.0, "status": 200, "response": {"model": "Qwen/Qwen3.5-9B", "answers": {"pick": {"type": "choice", "choice": "stand", "probabilities": {"hit": 0.008775687289839783, "stand": 0.9912243127101603}, "confidence": 0.9824486254203206}}, "usage": {"input_tokens": 458, "output_tokens": 0}, "metadata": {"method": "native_merged_lora_decision_head", "temperature": 1.8969118766347646, "candidate_sequences": 2, "inference_seconds": 0.075019785, "base_revision": "c202236235762e1c871ad0ccb60c8ee5ba337b9a", "max_length": 4096, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}}}, "notes": {"client_latency_ms": 76}} +{"type": "episode_end", "group": "blackjack-seed6-6", "status": "done", "steps": 1} +{"type": "request", "id": "blackjack-seed7-7:s0:a0", "group": "blackjack-seed7-7", "step": 0, "attempt": 0, "retry_of": null, "request": {"model": "jev-latest", "state": {"player_cards": ["H3", "CQ"], "player_total": 13, "soft_hand": false, "dealer_showing": ["D7"]}, "questions": {"pick": {"type": "choice", "criteria": {"hit": "hit (blackjack action)", "stand": "stand (blackjack action)"}, "instructions": {"rules": "Blackjack against a dealer who hits until 17. Aim to beat the dealer's final total without exceeding 21. Cards are written suit then rank (H7 = seven of hearts, SA = ace of spades, DT = ten of diamonds). Basic strategy: stand on hard 17 or more; stand on hard 13 to 16 when the dealer shows 2 to 6, otherwise hit; stand on hard 12 only against a dealer 4 to 6; always hit hard 11 or less; hit soft 17 or less; stand on soft 19 or more; stand on soft 18 unless the dealer shows 9, ten or ace."}}}}, "images": [], "t_send_ms": 856.626, "t_recv_ms": 933.288, "think_ms": null, "deadline_s": 30.0, "status": 200, "response": {"model": "Qwen/Qwen3.5-9B", "answers": {"pick": {"type": "choice", "choice": "hit", "probabilities": {"hit": 0.508939295317313, "stand": 0.49106070468268703}, "confidence": 0.017878590634625935}}, "usage": {"input_tokens": 460, "output_tokens": 0}, "metadata": {"method": "native_merged_lora_decision_head", "temperature": 1.8969118766347646, "candidate_sequences": 2, "inference_seconds": 0.07552398, "base_revision": "c202236235762e1c871ad0ccb60c8ee5ba337b9a", "max_length": 4096, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}}}, "notes": {"client_latency_ms": 77}} +{"type": "episode_end", "group": "blackjack-seed7-7", "status": "done", "steps": 1} +{"type": "request", "id": "blackjack-seed8-8:s0:a0", "group": "blackjack-seed8-8", "step": 0, "attempt": 0, "retry_of": null, "request": {"model": "jev-latest", "state": {"player_cards": ["HA", "DK"], "player_total": 21, "soft_hand": true, "dealer_showing": ["H9"]}, "questions": {"pick": {"type": "choice", "criteria": {"hit": "hit (blackjack action)", "stand": "stand (blackjack action)"}, "instructions": {"rules": "Blackjack against a dealer who hits until 17. Aim to beat the dealer's final total without exceeding 21. Cards are written suit then rank (H7 = seven of hearts, SA = ace of spades, DT = ten of diamonds). Basic strategy: stand on hard 17 or more; stand on hard 13 to 16 when the dealer shows 2 to 6, otherwise hit; stand on hard 12 only against a dealer 4 to 6; always hit hard 11 or less; hit soft 17 or less; stand on soft 19 or more; stand on soft 18 unless the dealer shows 9, ten or ace."}}}}, "images": [], "t_send_ms": 940.422, "t_recv_ms": 1016.189, "think_ms": null, "deadline_s": 30.0, "status": 200, "response": {"model": "Qwen/Qwen3.5-9B", "answers": {"pick": {"type": "choice", "choice": "stand", "probabilities": {"hit": 0.03751723345133573, "stand": 0.9624827665486643}, "confidence": 0.9249655330973285}}, "usage": {"input_tokens": 456, "output_tokens": 0}, "metadata": {"method": "native_merged_lora_decision_head", "temperature": 1.8969118766347646, "candidate_sequences": 2, "inference_seconds": 0.074686882, "base_revision": "c202236235762e1c871ad0ccb60c8ee5ba337b9a", "max_length": 4096, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}}}, "notes": {"client_latency_ms": 76}} +{"type": "episode_end", "group": "blackjack-seed8-8", "status": "done", "steps": 1} +{"type": "request", "id": "blackjack-seed9-9:s0:a0", "group": "blackjack-seed9-9", "step": 0, "attempt": 0, "retry_of": null, "request": {"model": "jev-latest", "state": {"player_cards": ["H2", "S8"], "player_total": 10, "soft_hand": false, "dealer_showing": ["H4"]}, "questions": {"pick": {"type": "choice", "criteria": {"hit": "hit (blackjack action)", "stand": "stand (blackjack action)"}, "instructions": {"rules": "Blackjack against a dealer who hits until 17. Aim to beat the dealer's final total without exceeding 21. Cards are written suit then rank (H7 = seven of hearts, SA = ace of spades, DT = ten of diamonds). Basic strategy: stand on hard 17 or more; stand on hard 13 to 16 when the dealer shows 2 to 6, otherwise hit; stand on hard 12 only against a dealer 4 to 6; always hit hard 11 or less; hit soft 17 or less; stand on soft 19 or more; stand on soft 18 unless the dealer shows 9, ten or ace."}}}}, "images": [], "t_send_ms": 1023.083, "t_recv_ms": 1099.066, "think_ms": null, "deadline_s": 30.0, "status": 200, "response": {"model": "Qwen/Qwen3.5-9B", "answers": {"pick": {"type": "choice", "choice": "stand", "probabilities": {"hit": 0.228286969405356, "stand": 0.771713030594644}, "confidence": 0.543426061189288}}, "usage": {"input_tokens": 460, "output_tokens": 0}, "metadata": {"method": "native_merged_lora_decision_head", "temperature": 1.8969118766347646, "candidate_sequences": 2, "inference_seconds": 0.075071143, "base_revision": "c202236235762e1c871ad0ccb60c8ee5ba337b9a", "max_length": 4096, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}}}, "notes": {"client_latency_ms": 76}} +{"type": "episode_end", "group": "blackjack-seed9-9", "status": "done", "steps": 1} +{"type": "request", "id": "blackjack-seed10-10:s0:a0", "group": "blackjack-seed10-10", "step": 0, "attempt": 0, "retry_of": null, "request": {"model": "jev-latest", "state": {"player_cards": ["H9", "H7"], "player_total": 16, "soft_hand": false, "dealer_showing": ["H3"]}, "questions": {"pick": {"type": "choice", "criteria": {"hit": "hit (blackjack action)", "stand": "stand (blackjack action)"}, "instructions": {"rules": "Blackjack against a dealer who hits until 17. Aim to beat the dealer's final total without exceeding 21. Cards are written suit then rank (H7 = seven of hearts, SA = ace of spades, DT = ten of diamonds). Basic strategy: stand on hard 17 or more; stand on hard 13 to 16 when the dealer shows 2 to 6, otherwise hit; stand on hard 12 only against a dealer 4 to 6; always hit hard 11 or less; hit soft 17 or less; stand on soft 19 or more; stand on soft 18 unless the dealer shows 9, ten or ace."}}}}, "images": [], "t_send_ms": 1106.752, "t_recv_ms": 1183.225, "think_ms": null, "deadline_s": 30.0, "status": 200, "response": {"model": "Qwen/Qwen3.5-9B", "answers": {"pick": {"type": "choice", "choice": "stand", "probabilities": {"hit": 0.08132531594509443, "stand": 0.9186746840549055}, "confidence": 0.837349368109811}}, "usage": {"input_tokens": 460, "output_tokens": 0}, "metadata": {"method": "native_merged_lora_decision_head", "temperature": 1.8969118766347646, "candidate_sequences": 2, "inference_seconds": 0.075425338, "base_revision": "c202236235762e1c871ad0ccb60c8ee5ba337b9a", "max_length": 4096, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}}}, "notes": {"client_latency_ms": 76}} +{"type": "episode_end", "group": "blackjack-seed10-10", "status": "done", "steps": 1} +{"type": "request", "id": "blackjack-seed11-11:s0:a0", "group": "blackjack-seed11-11", "step": 0, "attempt": 0, "retry_of": null, "request": {"model": "jev-latest", "state": {"player_cards": ["DT", "D4"], "player_total": 14, "soft_hand": false, "dealer_showing": ["HA"]}, "questions": {"pick": {"type": "choice", "criteria": {"hit": "hit (blackjack action)", "stand": "stand (blackjack action)"}, "instructions": {"rules": "Blackjack against a dealer who hits until 17. Aim to beat the dealer's final total without exceeding 21. Cards are written suit then rank (H7 = seven of hearts, SA = ace of spades, DT = ten of diamonds). Basic strategy: stand on hard 17 or more; stand on hard 13 to 16 when the dealer shows 2 to 6, otherwise hit; stand on hard 12 only against a dealer 4 to 6; always hit hard 11 or less; hit soft 17 or less; stand on soft 19 or more; stand on soft 18 unless the dealer shows 9, ten or ace."}}}}, "images": [], "t_send_ms": 1190.401, "t_recv_ms": 1266.735, "think_ms": null, "deadline_s": 30.0, "status": 200, "response": {"model": "Qwen/Qwen3.5-9B", "answers": {"pick": {"type": "choice", "choice": "stand", "probabilities": {"hit": 0.0828799164163315, "stand": 0.9171200835836684}, "confidence": 0.8342401671673367}}, "usage": {"input_tokens": 456, "output_tokens": 0}, "metadata": {"method": "native_merged_lora_decision_head", "temperature": 1.8969118766347646, "candidate_sequences": 2, "inference_seconds": 0.075069431, "base_revision": "c202236235762e1c871ad0ccb60c8ee5ba337b9a", "max_length": 4096, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}}}, "notes": {"client_latency_ms": 76}} +{"type": "episode_end", "group": "blackjack-seed11-11", "status": "done", "steps": 1} +{"type": "request", "id": "blackjack-seed12-12:s0:a0", "group": "blackjack-seed12-12", "step": 0, "attempt": 0, "retry_of": null, "request": {"model": "jev-latest", "state": {"player_cards": ["C4", "SQ"], "player_total": 14, "soft_hand": false, "dealer_showing": ["SA"]}, "questions": {"pick": {"type": "choice", "criteria": {"hit": "hit (blackjack action)", "stand": "stand (blackjack action)"}, "instructions": {"rules": "Blackjack against a dealer who hits until 17. Aim to beat the dealer's final total without exceeding 21. Cards are written suit then rank (H7 = seven of hearts, SA = ace of spades, DT = ten of diamonds). Basic strategy: stand on hard 17 or more; stand on hard 13 to 16 when the dealer shows 2 to 6, otherwise hit; stand on hard 12 only against a dealer 4 to 6; always hit hard 11 or less; hit soft 17 or less; stand on soft 19 or more; stand on soft 18 unless the dealer shows 9, ten or ace."}}}}, "images": [], "t_send_ms": 1274.248, "t_recv_ms": 1349.598, "think_ms": null, "deadline_s": 30.0, "status": 200, "response": {"model": "Qwen/Qwen3.5-9B", "answers": {"pick": {"type": "choice", "choice": "stand", "probabilities": {"hit": 0.13597702604387754, "stand": 0.8640229739561225}, "confidence": 0.728045947912245}}, "usage": {"input_tokens": 456, "output_tokens": 0}, "metadata": {"method": "native_merged_lora_decision_head", "temperature": 1.8969118766347646, "candidate_sequences": 2, "inference_seconds": 0.07406129, "base_revision": "c202236235762e1c871ad0ccb60c8ee5ba337b9a", "max_length": 4096, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}}}, "notes": {"client_latency_ms": 75}} +{"type": "episode_end", "group": "blackjack-seed12-12", "status": "done", "steps": 1} +{"type": "request", "id": "blackjack-seed13-13:s0:a0", "group": "blackjack-seed13-13", "step": 0, "attempt": 0, "retry_of": null, "request": {"model": "jev-latest", "state": {"player_cards": ["D4", "S4"], "player_total": 8, "soft_hand": false, "dealer_showing": ["SQ"]}, "questions": {"pick": {"type": "choice", "criteria": {"hit": "hit (blackjack action)", "stand": "stand (blackjack action)"}, "instructions": {"rules": "Blackjack against a dealer who hits until 17. Aim to beat the dealer's final total without exceeding 21. Cards are written suit then rank (H7 = seven of hearts, SA = ace of spades, DT = ten of diamonds). Basic strategy: stand on hard 17 or more; stand on hard 13 to 16 when the dealer shows 2 to 6, otherwise hit; stand on hard 12 only against a dealer 4 to 6; always hit hard 11 or less; hit soft 17 or less; stand on soft 19 or more; stand on soft 18 unless the dealer shows 9, ten or ace."}}}}, "images": [], "t_send_ms": 1356.27, "t_recv_ms": 1433.047, "think_ms": null, "deadline_s": 30.0, "status": 200, "response": {"model": "Qwen/Qwen3.5-9B", "answers": {"pick": {"type": "choice", "choice": "stand", "probabilities": {"hit": 0.48623589402895434, "stand": 0.5137641059710457}, "confidence": 0.02752821194209143}}, "usage": {"input_tokens": 456, "output_tokens": 0}, "metadata": {"method": "native_merged_lora_decision_head", "temperature": 1.8969118766347646, "candidate_sequences": 2, "inference_seconds": 0.075627421, "base_revision": "c202236235762e1c871ad0ccb60c8ee5ba337b9a", "max_length": 4096, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}}}, "notes": {"client_latency_ms": 77}} +{"type": "episode_end", "group": "blackjack-seed13-13", "status": "done", "steps": 1} +{"type": "request", "id": "blackjack-seed14-14:s0:a0", "group": "blackjack-seed14-14", "step": 0, "attempt": 0, "retry_of": null, "request": {"model": "jev-latest", "state": {"player_cards": ["DA", "ST"], "player_total": 21, "soft_hand": true, "dealer_showing": ["D8"]}, "questions": {"pick": {"type": "choice", "criteria": {"hit": "hit (blackjack action)", "stand": "stand (blackjack action)"}, "instructions": {"rules": "Blackjack against a dealer who hits until 17. Aim to beat the dealer's final total without exceeding 21. Cards are written suit then rank (H7 = seven of hearts, SA = ace of spades, DT = ten of diamonds). Basic strategy: stand on hard 17 or more; stand on hard 13 to 16 when the dealer shows 2 to 6, otherwise hit; stand on hard 12 only against a dealer 4 to 6; always hit hard 11 or less; hit soft 17 or less; stand on soft 19 or more; stand on soft 18 unless the dealer shows 9, ten or ace."}}}}, "images": [], "t_send_ms": 1440.144, "t_recv_ms": 1515.155, "think_ms": null, "deadline_s": 30.0, "status": 200, "response": {"model": "Qwen/Qwen3.5-9B", "answers": {"pick": {"type": "choice", "choice": "stand", "probabilities": {"hit": 0.021236761930287246, "stand": 0.9787632380697128}, "confidence": 0.9575264761394255}}, "usage": {"input_tokens": 456, "output_tokens": 0}, "metadata": {"method": "native_merged_lora_decision_head", "temperature": 1.8969118766347646, "candidate_sequences": 2, "inference_seconds": 0.074046648, "base_revision": "c202236235762e1c871ad0ccb60c8ee5ba337b9a", "max_length": 4096, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}}}, "notes": {"client_latency_ms": 75}} +{"type": "episode_end", "group": "blackjack-seed14-14", "status": "done", "steps": 1} +{"type": "request", "id": "blackjack-seed15-15:s0:a0", "group": "blackjack-seed15-15", "step": 0, "attempt": 0, "retry_of": null, "request": {"model": "jev-latest", "state": {"player_cards": ["CK", "D4"], "player_total": 14, "soft_hand": false, "dealer_showing": ["SJ"]}, "questions": {"pick": {"type": "choice", "criteria": {"hit": "hit (blackjack action)", "stand": "stand (blackjack action)"}, "instructions": {"rules": "Blackjack against a dealer who hits until 17. Aim to beat the dealer's final total without exceeding 21. Cards are written suit then rank (H7 = seven of hearts, SA = ace of spades, DT = ten of diamonds). Basic strategy: stand on hard 17 or more; stand on hard 13 to 16 when the dealer shows 2 to 6, otherwise hit; stand on hard 12 only against a dealer 4 to 6; always hit hard 11 or less; hit soft 17 or less; stand on soft 19 or more; stand on soft 18 unless the dealer shows 9, ten or ace."}}}}, "images": [], "t_send_ms": 1522.532, "t_recv_ms": 1598.482, "think_ms": null, "deadline_s": 30.0, "status": 200, "response": {"model": "Qwen/Qwen3.5-9B", "answers": {"pick": {"type": "choice", "choice": "stand", "probabilities": {"hit": 0.09643386400835587, "stand": 0.9035661359916441}, "confidence": 0.8071322719832883}}, "usage": {"input_tokens": 456, "output_tokens": 0}, "metadata": {"method": "native_merged_lora_decision_head", "temperature": 1.8969118766347646, "candidate_sequences": 2, "inference_seconds": 0.074934604, "base_revision": "c202236235762e1c871ad0ccb60c8ee5ba337b9a", "max_length": 4096, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}}}, "notes": {"client_latency_ms": 76}} +{"type": "episode_end", "group": "blackjack-seed15-15", "status": "done", "steps": 1} +{"type": "request", "id": "blackjack-seed16-16:s0:a0", "group": "blackjack-seed16-16", "step": 0, "attempt": 0, "retry_of": null, "request": {"model": "jev-latest", "state": {"player_cards": ["H9", "C2"], "player_total": 11, "soft_hand": false, "dealer_showing": ["HQ"]}, "questions": {"pick": {"type": "choice", "criteria": {"hit": "hit (blackjack action)", "stand": "stand (blackjack action)"}, "instructions": {"rules": "Blackjack against a dealer who hits until 17. Aim to beat the dealer's final total without exceeding 21. Cards are written suit then rank (H7 = seven of hearts, SA = ace of spades, DT = ten of diamonds). Basic strategy: stand on hard 17 or more; stand on hard 13 to 16 when the dealer shows 2 to 6, otherwise hit; stand on hard 12 only against a dealer 4 to 6; always hit hard 11 or less; hit soft 17 or less; stand on soft 19 or more; stand on soft 18 unless the dealer shows 9, ten or ace."}}}}, "images": [], "t_send_ms": 1605.745, "t_recv_ms": 1683.045, "think_ms": null, "deadline_s": 30.0, "status": 200, "response": {"model": "Qwen/Qwen3.5-9B", "answers": {"pick": {"type": "choice", "choice": "stand", "probabilities": {"hit": 0.31915656389294733, "stand": 0.6808434361070527}, "confidence": 0.36168687221410534}}, "usage": {"input_tokens": 458, "output_tokens": 0}, "metadata": {"method": "native_merged_lora_decision_head", "temperature": 1.8969118766347646, "candidate_sequences": 2, "inference_seconds": 0.076239542, "base_revision": "c202236235762e1c871ad0ccb60c8ee5ba337b9a", "max_length": 4096, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}}}, "notes": {"client_latency_ms": 77}} +{"type": "episode_end", "group": "blackjack-seed16-16", "status": "done", "steps": 1} +{"type": "request", "id": "blackjack-seed17-17:s0:a0", "group": "blackjack-seed17-17", "step": 0, "attempt": 0, "retry_of": null, "request": {"model": "jev-latest", "state": {"player_cards": ["CQ", "D7"], "player_total": 17, "soft_hand": false, "dealer_showing": ["SQ"]}, "questions": {"pick": {"type": "choice", "criteria": {"hit": "hit (blackjack action)", "stand": "stand (blackjack action)"}, "instructions": {"rules": "Blackjack against a dealer who hits until 17. Aim to beat the dealer's final total without exceeding 21. Cards are written suit then rank (H7 = seven of hearts, SA = ace of spades, DT = ten of diamonds). Basic strategy: stand on hard 17 or more; stand on hard 13 to 16 when the dealer shows 2 to 6, otherwise hit; stand on hard 12 only against a dealer 4 to 6; always hit hard 11 or less; hit soft 17 or less; stand on soft 19 or more; stand on soft 18 unless the dealer shows 9, ten or ace."}}}}, "images": [], "t_send_ms": 1690.72, "t_recv_ms": 1766.329, "think_ms": null, "deadline_s": 30.0, "status": 200, "response": {"model": "Qwen/Qwen3.5-9B", "answers": {"pick": {"type": "choice", "choice": "stand", "probabilities": {"hit": 0.0014574957884183008, "stand": 0.9985425042115816}, "confidence": 0.9970850084231633}}, "usage": {"input_tokens": 458, "output_tokens": 0}, "metadata": {"method": "native_merged_lora_decision_head", "temperature": 1.8969118766347646, "candidate_sequences": 2, "inference_seconds": 0.074575652, "base_revision": "c202236235762e1c871ad0ccb60c8ee5ba337b9a", "max_length": 4096, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}}}, "notes": {"client_latency_ms": 76}} +{"type": "episode_end", "group": "blackjack-seed17-17", "status": "done", "steps": 1} +{"type": "request", "id": "blackjack-seed18-18:s0:a0", "group": "blackjack-seed18-18", "step": 0, "attempt": 0, "retry_of": null, "request": {"model": "jev-latest", "state": {"player_cards": ["C9", "H2"], "player_total": 11, "soft_hand": false, "dealer_showing": ["S8"]}, "questions": {"pick": {"type": "choice", "criteria": {"hit": "hit (blackjack action)", "stand": "stand (blackjack action)"}, "instructions": {"rules": "Blackjack against a dealer who hits until 17. Aim to beat the dealer's final total without exceeding 21. Cards are written suit then rank (H7 = seven of hearts, SA = ace of spades, DT = ten of diamonds). Basic strategy: stand on hard 17 or more; stand on hard 13 to 16 when the dealer shows 2 to 6, otherwise hit; stand on hard 12 only against a dealer 4 to 6; always hit hard 11 or less; hit soft 17 or less; stand on soft 19 or more; stand on soft 18 unless the dealer shows 9, ten or ace."}}}}, "images": [], "t_send_ms": 1774.083, "t_recv_ms": 1849.391, "think_ms": null, "deadline_s": 30.0, "status": 200, "response": {"model": "Qwen/Qwen3.5-9B", "answers": {"pick": {"type": "choice", "choice": "stand", "probabilities": {"hit": 0.22563015923151414, "stand": 0.7743698407684858}, "confidence": 0.5487396815369716}}, "usage": {"input_tokens": 460, "output_tokens": 0}, "metadata": {"method": "native_merged_lora_decision_head", "temperature": 1.8969118766347646, "candidate_sequences": 2, "inference_seconds": 0.074122505, "base_revision": "c202236235762e1c871ad0ccb60c8ee5ba337b9a", "max_length": 4096, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}}}, "notes": {"client_latency_ms": 75}} +{"type": "episode_end", "group": "blackjack-seed18-18", "status": "done", "steps": 1} +{"type": "request", "id": "blackjack-seed19-19:s0:a0", "group": "blackjack-seed19-19", "step": 0, "attempt": 0, "retry_of": null, "request": {"model": "jev-latest", "state": {"player_cards": ["C6", "DK"], "player_total": 16, "soft_hand": false, "dealer_showing": ["CT"]}, "questions": {"pick": {"type": "choice", "criteria": {"hit": "hit (blackjack action)", "stand": "stand (blackjack action)"}, "instructions": {"rules": "Blackjack against a dealer who hits until 17. Aim to beat the dealer's final total without exceeding 21. Cards are written suit then rank (H7 = seven of hearts, SA = ace of spades, DT = ten of diamonds). Basic strategy: stand on hard 17 or more; stand on hard 13 to 16 when the dealer shows 2 to 6, otherwise hit; stand on hard 12 only against a dealer 4 to 6; always hit hard 11 or less; hit soft 17 or less; stand on soft 19 or more; stand on soft 18 unless the dealer shows 9, ten or ace."}}}}, "images": [], "t_send_ms": 1856.714, "t_recv_ms": 1934.689, "think_ms": null, "deadline_s": 30.0, "status": 200, "response": {"model": "Qwen/Qwen3.5-9B", "answers": {"pick": {"type": "choice", "choice": "stand", "probabilities": {"hit": 0.059018240006057515, "stand": 0.9409817599939424}, "confidence": 0.8819635199878848}}, "usage": {"input_tokens": 456, "output_tokens": 0}, "metadata": {"method": "native_merged_lora_decision_head", "temperature": 1.8969118766347646, "candidate_sequences": 2, "inference_seconds": 0.076930275, "base_revision": "c202236235762e1c871ad0ccb60c8ee5ba337b9a", "max_length": 4096, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}}}, "notes": {"client_latency_ms": 78}} +{"type": "episode_end", "group": "blackjack-seed19-19", "status": "done", "steps": 1} diff --git a/benchmarks/traces/samples/ticket_router.trace.jsonl b/benchmarks/traces/samples/ticket_router.trace.jsonl new file mode 100644 index 00000000..e252547a --- /dev/null +++ b/benchmarks/traces/samples/ticket_router.trace.jsonl @@ -0,0 +1,32 @@ +{"type": "header", "format": "s1-trace/1", "source": {"repo": "ThinkFlowLab/system1-agents", "commit": "9b20b989d919bc69227f81bc0ec571ca56925586", "front": "run ticket_router", "argv": ["run", "ticket_router", "--model", "jev", "--rethink", "off", "--episodes", "1"]}, "server": {"repo": "InferMatrixLab/system1-omni", "commit": null, "worker": "open_jev", "url": "http://127.0.0.1:8840/v1/systemone", "model": "jev-latest", "notes": "native Open-Jev-9B worker built with #179's change (a local build), no prefix switch"}, "capture": "client:transport", "clock": "ms since the recorder started (perf_counter)", "content": {"provenance": "System1-Agents ticket_router agent (s1a run ticket_router --model jev --rethink off), its 30 local synthetic tickets", "license": "Apache-2.0 (System1-Agents tickets and prompt text)", "redaction": "none needed"}, "images": {"store": null}} +{"type": "request", "id": "ticket_router-seed0-0:s0:a0", "group": "ticket_router-seed0-0", "step": 0, "attempt": 0, "retry_of": null, "request": {"model": "jev-latest", "state": {"ticket": {"id": "t_a5a229edd810", "title": "Track the new order", "description": "The duplicate charge on my previous order has been resolved. Today I only want the current location of this parcel.", "order_status": "shipped"}, "progress": {"ticket_id": "t_a5a229edd810"}}, "questions": {"pick": {"type": "choice", "criteria": {"logistics": "Delivery tracking, delivery progress or delivery problems", "payment": "Charges, failed payments or duplicate payments", "returns": "Requests for returns, exchanges or refunds", "account": "Login or account access problems", "human": "Insufficient information, multiple independent requests or an explicit request for a human"}, "instructions": {"rules": "Route the current explicit request to exactly one queue. logistics: delivery tracking or delivery problems. payment: charges, failed payments or duplicate payments. returns: requests for returns, exchanges or refunds. account: login or account access problems. human: insufficient information, multiple independent requests that cannot be uniquely routed, or an explicit request for a human. An explicit human request takes priority. Recognize negation and resolved, quoted or hypothetical background; do not route by keywords alone. Order status is context, not the user's intent. A payment problem with an explicit request for a refund belongs to returns. If no unique queue fits, choose human."}}}}, "images": [], "t_send_ms": 170.999, "t_recv_ms": 383.125, "think_ms": null, "deadline_s": 30.0, "status": 200, "response": {"model": "Qwen/Qwen3.5-9B", "answers": {"pick": {"type": "choice", "choice": "logistics", "probabilities": {"logistics": 0.9982556714431506, "payment": 6.41629474490934e-05, "returns": 3.7564348636503654e-05, "account": 2.3086944545137816e-05, "human": 0.001619514316218535}, "confidence": 0.9978195893039382}}, "usage": {"input_tokens": 1301, "output_tokens": 0}, "metadata": {"method": "native_merged_lora_decision_head", "temperature": 1.8969118766347646, "candidate_sequences": 5, "inference_seconds": 0.210661437, "base_revision": "c202236235762e1c871ad0ccb60c8ee5ba337b9a", "max_length": 4096, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}}}, "notes": {"client_latency_ms": 212}} +{"type": "request", "id": "ticket_router-seed0-0:s1:a0", "group": "ticket_router-seed0-0", "step": 1, "attempt": 0, "retry_of": null, "request": {"model": "jev-latest", "state": {"ticket": {"id": "t_c85ca34669b1", "title": "Transaction status", "description": "My bank shows a charge, but the order is still marked unpaid. Please verify whether this transaction succeeded.", "order_status": "unpaid"}, "progress": {"ticket_id": "t_c85ca34669b1"}}, "questions": {"pick": {"type": "choice", "criteria": {"logistics": "Delivery tracking, delivery progress or delivery problems", "payment": "Charges, failed payments or duplicate payments", "returns": "Requests for returns, exchanges or refunds", "account": "Login or account access problems", "human": "Insufficient information, multiple independent requests or an explicit request for a human"}, "instructions": {"rules": "Route the current explicit request to exactly one queue. logistics: delivery tracking or delivery problems. payment: charges, failed payments or duplicate payments. returns: requests for returns, exchanges or refunds. account: login or account access problems. human: insufficient information, multiple independent requests that cannot be uniquely routed, or an explicit request for a human. An explicit human request takes priority. Recognize negation and resolved, quoted or hypothetical background; do not route by keywords alone. Order status is context, not the user's intent. A payment problem with an explicit request for a refund belongs to returns. If no unique queue fits, choose human."}}}}, "images": [], "t_send_ms": 385.447, "t_recv_ms": 609.384, "think_ms": 2.322, "deadline_s": 30.0, "status": 200, "response": {"model": "Qwen/Qwen3.5-9B", "answers": {"pick": {"type": "choice", "choice": "payment", "probabilities": {"logistics": 3.967752128717432e-05, "payment": 0.9986817680810919, "returns": 3.4894734991288265e-05, "account": 3.897657334542177e-05, "human": 0.001204683089284076}, "confidence": 0.9983522101013648}}, "usage": {"input_tokens": 1296, "output_tokens": 0}, "metadata": {"method": "native_merged_lora_decision_head", "temperature": 1.8969118766347646, "candidate_sequences": 5, "inference_seconds": 0.222773064, "base_revision": "c202236235762e1c871ad0ccb60c8ee5ba337b9a", "max_length": 4096, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}}}, "notes": {"client_latency_ms": 224}} +{"type": "request", "id": "ticket_router-seed0-0:s2:a0", "group": "ticket_router-seed0-0", "step": 2, "attempt": 0, "retry_of": null, "request": {"model": "jev-latest", "state": {"ticket": {"id": "t_55ac1dd8d045", "title": "Forgotten password", "description": "I forgot my password and cannot log in. Please help me recover my account."}, "progress": {"ticket_id": "t_55ac1dd8d045"}}, "questions": {"pick": {"type": "choice", "criteria": {"logistics": "Delivery tracking, delivery progress or delivery problems", "payment": "Charges, failed payments or duplicate payments", "returns": "Requests for returns, exchanges or refunds", "account": "Login or account access problems", "human": "Insufficient information, multiple independent requests or an explicit request for a human"}, "instructions": {"rules": "Route the current explicit request to exactly one queue. logistics: delivery tracking or delivery problems. payment: charges, failed payments or duplicate payments. returns: requests for returns, exchanges or refunds. account: login or account access problems. human: insufficient information, multiple independent requests that cannot be uniquely routed, or an explicit request for a human. An explicit human request takes priority. Recognize negation and resolved, quoted or hypothetical background; do not route by keywords alone. Order status is context, not the user's intent. A payment problem with an explicit request for a refund belongs to returns. If no unique queue fits, choose human."}}}}, "images": [], "t_send_ms": 611.157, "t_recv_ms": 810.302, "think_ms": 1.773, "deadline_s": 30.0, "status": 200, "response": {"model": "Qwen/Qwen3.5-9B", "answers": {"pick": {"type": "choice", "choice": "account", "probabilities": {"logistics": 8.036630200548002e-06, "payment": 7.062775151398434e-06, "returns": 8.952554758902793e-06, "account": 0.9998542210016718, "human": 0.00012172703821755081}, "confidence": 0.9998177762520896}}, "usage": {"input_tokens": 1236, "output_tokens": 0}, "metadata": {"method": "native_merged_lora_decision_head", "temperature": 1.8969118766347646, "candidate_sequences": 5, "inference_seconds": 0.198184141, "base_revision": "c202236235762e1c871ad0ccb60c8ee5ba337b9a", "max_length": 4096, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}}}, "notes": {"client_latency_ms": 199}} +{"type": "request", "id": "ticket_router-seed0-0:s3:a0", "group": "ticket_router-seed0-0", "step": 3, "attempt": 0, "retry_of": null, "request": {"model": "jev-latest", "state": {"ticket": {"id": "t_6892a5546a24", "title": "Do not want to keep the item", "description": "I want to send this item back to the seller and get my money back. I do not need parcel tracking.", "order_status": "delivered"}, "progress": {"ticket_id": "t_6892a5546a24"}}, "questions": {"pick": {"type": "choice", "criteria": {"logistics": "Delivery tracking, delivery progress or delivery problems", "payment": "Charges, failed payments or duplicate payments", "returns": "Requests for returns, exchanges or refunds", "account": "Login or account access problems", "human": "Insufficient information, multiple independent requests or an explicit request for a human"}, "instructions": {"rules": "Route the current explicit request to exactly one queue. logistics: delivery tracking or delivery problems. payment: charges, failed payments or duplicate payments. returns: requests for returns, exchanges or refunds. account: login or account access problems. human: insufficient information, multiple independent requests that cannot be uniquely routed, or an explicit request for a human. An explicit human request takes priority. Recognize negation and resolved, quoted or hypothetical background; do not route by keywords alone. Order status is context, not the user's intent. A payment problem with an explicit request for a refund belongs to returns. If no unique queue fits, choose human."}}}}, "images": [], "t_send_ms": 811.859, "t_recv_ms": 1053.286, "think_ms": 1.557, "deadline_s": 30.0, "status": 200, "response": {"model": "Qwen/Qwen3.5-9B", "answers": {"pick": {"type": "choice", "choice": "returns", "probabilities": {"logistics": 2.897935084509008e-05, "payment": 3.651741108031633e-05, "returns": 0.9997560520682776, "account": 8.62116431155027e-06, "human": 0.00016983000548531234}, "confidence": 0.999695065085347}}, "usage": {"input_tokens": 1351, "output_tokens": 0}, "metadata": {"method": "native_merged_lora_decision_head", "temperature": 1.8969118766347646, "candidate_sequences": 5, "inference_seconds": 0.240329008, "base_revision": "c202236235762e1c871ad0ccb60c8ee5ba337b9a", "max_length": 4096, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}}}, "notes": {"client_latency_ms": 241}} +{"type": "request", "id": "ticket_router-seed0-0:s4:a0", "group": "ticket_router-seed0-0", "step": 4, "attempt": 0, "retry_of": null, "request": {"model": "jev-latest", "state": {"ticket": {"id": "t_97415027853c", "title": "Ask for delivery now", "description": "If it still has not arrived next week, I will request a return. Right now I only want you to speed up delivery.", "order_status": "shipped"}, "progress": {"ticket_id": "t_97415027853c"}}, "questions": {"pick": {"type": "choice", "criteria": {"logistics": "Delivery tracking, delivery progress or delivery problems", "payment": "Charges, failed payments or duplicate payments", "returns": "Requests for returns, exchanges or refunds", "account": "Login or account access problems", "human": "Insufficient information, multiple independent requests or an explicit request for a human"}, "instructions": {"rules": "Route the current explicit request to exactly one queue. logistics: delivery tracking or delivery problems. payment: charges, failed payments or duplicate payments. returns: requests for returns, exchanges or refunds. account: login or account access problems. human: insufficient information, multiple independent requests that cannot be uniquely routed, or an explicit request for a human. An explicit human request takes priority. Recognize negation and resolved, quoted or hypothetical background; do not route by keywords alone. Order status is context, not the user's intent. A payment problem with an explicit request for a refund belongs to returns. If no unique queue fits, choose human."}}}}, "images": [], "t_send_ms": 1055.037, "t_recv_ms": 1299.443, "think_ms": 1.751, "deadline_s": 30.0, "status": 200, "response": {"model": "Qwen/Qwen3.5-9B", "answers": {"pick": {"type": "choice", "choice": "logistics", "probabilities": {"logistics": 0.9951557361432923, "payment": 6.435803875946026e-05, "returns": 0.001153679896970919, "account": 6.72990320259696e-05, "human": 0.003558926888951241}, "confidence": 0.9939446701791155}}, "usage": {"input_tokens": 1351, "output_tokens": 0}, "metadata": {"method": "native_merged_lora_decision_head", "temperature": 1.8969118766347646, "candidate_sequences": 5, "inference_seconds": 0.243230964, "base_revision": "c202236235762e1c871ad0ccb60c8ee5ba337b9a", "max_length": 4096, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}}}, "notes": {"client_latency_ms": 244}} +{"type": "request", "id": "ticket_router-seed0-0:s5:a0", "group": "ticket_router-seed0-0", "step": 5, "attempt": 0, "retry_of": null, "request": {"model": "jev-latest", "state": {"ticket": {"id": "t_52586863f51e", "title": "Delivery stalled", "description": "My parcel has been sitting at the same sorting center. Please investigate the delivery problem.", "order_status": "shipped"}, "progress": {"ticket_id": "t_52586863f51e"}}, "questions": {"pick": {"type": "choice", "criteria": {"logistics": "Delivery tracking, delivery progress or delivery problems", "payment": "Charges, failed payments or duplicate payments", "returns": "Requests for returns, exchanges or refunds", "account": "Login or account access problems", "human": "Insufficient information, multiple independent requests or an explicit request for a human"}, "instructions": {"rules": "Route the current explicit request to exactly one queue. logistics: delivery tracking or delivery problems. payment: charges, failed payments or duplicate payments. returns: requests for returns, exchanges or refunds. account: login or account access problems. human: insufficient information, multiple independent requests that cannot be uniquely routed, or an explicit request for a human. An explicit human request takes priority. Recognize negation and resolved, quoted or hypothetical background; do not route by keywords alone. Order status is context, not the user's intent. A payment problem with an explicit request for a refund belongs to returns. If no unique queue fits, choose human."}}}}, "images": [], "t_send_ms": 1301.354, "t_recv_ms": 1532.015, "think_ms": 1.911, "deadline_s": 30.0, "status": 200, "response": {"model": "Qwen/Qwen3.5-9B", "answers": {"pick": {"type": "choice", "choice": "logistics", "probabilities": {"logistics": 0.9973294346765323, "payment": 7.526499826558002e-05, "returns": 0.0001047055011750545, "account": 8.269447604112119e-05, "human": 0.0024079003479861335}, "confidence": 0.9966617933456653}}, "usage": {"input_tokens": 1296, "output_tokens": 0}, "metadata": {"method": "native_merged_lora_decision_head", "temperature": 1.8969118766347646, "candidate_sequences": 5, "inference_seconds": 0.229545958, "base_revision": "c202236235762e1c871ad0ccb60c8ee5ba337b9a", "max_length": 4096, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}}}, "notes": {"client_latency_ms": 231}} +{"type": "request", "id": "ticket_router-seed0-0:s6:a0", "group": "ticket_router-seed0-0", "step": 6, "attempt": 0, "retry_of": null, "request": {"model": "jev-latest", "state": {"ticket": {"id": "t_1a6847ba8b66", "title": "Payment error", "description": "After I click pay, bank card verification fails and I cannot complete the payment.", "order_status": "unpaid"}, "progress": {"ticket_id": "t_1a6847ba8b66"}}, "questions": {"pick": {"type": "choice", "criteria": {"logistics": "Delivery tracking, delivery progress or delivery problems", "payment": "Charges, failed payments or duplicate payments", "returns": "Requests for returns, exchanges or refunds", "account": "Login or account access problems", "human": "Insufficient information, multiple independent requests or an explicit request for a human"}, "instructions": {"rules": "Route the current explicit request to exactly one queue. logistics: delivery tracking or delivery problems. payment: charges, failed payments or duplicate payments. returns: requests for returns, exchanges or refunds. account: login or account access problems. human: insufficient information, multiple independent requests that cannot be uniquely routed, or an explicit request for a human. An explicit human request takes priority. Recognize negation and resolved, quoted or hypothetical background; do not route by keywords alone. Order status is context, not the user's intent. A payment problem with an explicit request for a refund belongs to returns. If no unique queue fits, choose human."}}}}, "images": [], "t_send_ms": 1533.861, "t_recv_ms": 1751.654, "think_ms": 1.846, "deadline_s": 30.0, "status": 200, "response": {"model": "Qwen/Qwen3.5-9B", "answers": {"pick": {"type": "choice", "choice": "payment", "probabilities": {"logistics": 6.667862042688407e-06, "payment": 0.9998648421423355, "returns": 1.2884392416282849e-05, "account": 1.8141873040370616e-05, "human": 9.746373016522545e-05}, "confidence": 0.9998310526779194}}, "usage": {"input_tokens": 1281, "output_tokens": 0}, "metadata": {"method": "native_merged_lora_decision_head", "temperature": 1.8969118766347646, "candidate_sequences": 5, "inference_seconds": 0.216876726, "base_revision": "c202236235762e1c871ad0ccb60c8ee5ba337b9a", "max_length": 4096, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}}}, "notes": {"client_latency_ms": 218}} +{"type": "request", "id": "ticket_router-seed0-0:s7:a0", "group": "ticket_router-seed0-0", "step": 7, "attempt": 0, "retry_of": null, "request": {"model": "jev-latest", "state": {"ticket": {"id": "t_fac102ef66cc", "title": "Unclear order issue", "description": "Something about this order looks wrong, but I do not know what it is."}, "progress": {"ticket_id": "t_fac102ef66cc"}}, "questions": {"pick": {"type": "choice", "criteria": {"logistics": "Delivery tracking, delivery progress or delivery problems", "payment": "Charges, failed payments or duplicate payments", "returns": "Requests for returns, exchanges or refunds", "account": "Login or account access problems", "human": "Insufficient information, multiple independent requests or an explicit request for a human"}, "instructions": {"rules": "Route the current explicit request to exactly one queue. logistics: delivery tracking or delivery problems. payment: charges, failed payments or duplicate payments. returns: requests for returns, exchanges or refunds. account: login or account access problems. human: insufficient information, multiple independent requests that cannot be uniquely routed, or an explicit request for a human. An explicit human request takes priority. Recognize negation and resolved, quoted or hypothetical background; do not route by keywords alone. Order status is context, not the user's intent. A payment problem with an explicit request for a refund belongs to returns. If no unique queue fits, choose human."}}}}, "images": [], "t_send_ms": 1753.423, "t_recv_ms": 1956.604, "think_ms": 1.769, "deadline_s": 30.0, "status": 200, "response": {"model": "Qwen/Qwen3.5-9B", "answers": {"pick": {"type": "choice", "choice": "human", "probabilities": {"logistics": 0.0017651713578115753, "payment": 0.0006335423098539025, "returns": 0.00035405031050790434, "account": 0.0002608970514332481, "human": 0.9969863389703935}, "confidence": 0.9962329237129919}}, "usage": {"input_tokens": 1211, "output_tokens": 0}, "metadata": {"method": "native_merged_lora_decision_head", "temperature": 1.8969118766347646, "candidate_sequences": 5, "inference_seconds": 0.202054149, "base_revision": "c202236235762e1c871ad0ccb60c8ee5ba337b9a", "max_length": 4096, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}}}, "notes": {"client_latency_ms": 203}} +{"type": "request", "id": "ticket_router-seed0-0:s8:a0", "group": "ticket_router-seed0-0", "step": 8, "attempt": 0, "retry_of": null, "request": {"model": "jev-latest", "state": {"ticket": {"id": "t_30a6dedb9828", "title": "Human help with account", "description": "I do not want to continue automated account recovery. Please transfer me to a real person."}, "progress": {"ticket_id": "t_30a6dedb9828"}}, "questions": {"pick": {"type": "choice", "criteria": {"logistics": "Delivery tracking, delivery progress or delivery problems", "payment": "Charges, failed payments or duplicate payments", "returns": "Requests for returns, exchanges or refunds", "account": "Login or account access problems", "human": "Insufficient information, multiple independent requests or an explicit request for a human"}, "instructions": {"rules": "Route the current explicit request to exactly one queue. logistics: delivery tracking or delivery problems. payment: charges, failed payments or duplicate payments. returns: requests for returns, exchanges or refunds. account: login or account access problems. human: insufficient information, multiple independent requests that cannot be uniquely routed, or an explicit request for a human. An explicit human request takes priority. Recognize negation and resolved, quoted or hypothetical background; do not route by keywords alone. Order status is context, not the user's intent. A payment problem with an explicit request for a refund belongs to returns. If no unique queue fits, choose human."}}}}, "images": [], "t_send_ms": 1958.589, "t_recv_ms": 2173.407, "think_ms": 1.985, "deadline_s": 30.0, "status": 200, "response": {"model": "Qwen/Qwen3.5-9B", "answers": {"pick": {"type": "choice", "choice": "human", "probabilities": {"logistics": 1.20638025557256e-05, "payment": 1.3912008955785837e-05, "returns": 1.649630206009708e-05, "account": 0.0004135023726556548, "human": 0.9995440255137729}, "confidence": 0.9994300318922161}}, "usage": {"input_tokens": 1251, "output_tokens": 0}, "metadata": {"method": "native_merged_lora_decision_head", "temperature": 1.8969118766347646, "candidate_sequences": 5, "inference_seconds": 0.214136861, "base_revision": "c202236235762e1c871ad0ccb60c8ee5ba337b9a", "max_length": 4096, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}}}, "notes": {"client_latency_ms": 215}} +{"type": "request", "id": "ticket_router-seed0-0:s9:a0", "group": "ticket_router-seed0-0", "step": 9, "attempt": 0, "retry_of": null, "request": {"model": "jev-latest", "state": {"ticket": {"id": "t_f21a8be4a071", "title": "Refund requested", "description": "There is no need to chase delivery anymore. I want a refund now.", "order_status": "shipped"}, "progress": {"ticket_id": "t_f21a8be4a071"}}, "questions": {"pick": {"type": "choice", "criteria": {"logistics": "Delivery tracking, delivery progress or delivery problems", "payment": "Charges, failed payments or duplicate payments", "returns": "Requests for returns, exchanges or refunds", "account": "Login or account access problems", "human": "Insufficient information, multiple independent requests or an explicit request for a human"}, "instructions": {"rules": "Route the current explicit request to exactly one queue. logistics: delivery tracking or delivery problems. payment: charges, failed payments or duplicate payments. returns: requests for returns, exchanges or refunds. account: login or account access problems. human: insufficient information, multiple independent requests that cannot be uniquely routed, or an explicit request for a human. An explicit human request takes priority. Recognize negation and resolved, quoted or hypothetical background; do not route by keywords alone. Order status is context, not the user's intent. A payment problem with an explicit request for a refund belongs to returns. If no unique queue fits, choose human."}}}}, "images": [], "t_send_ms": 2176.166, "t_recv_ms": 2401.715, "think_ms": 2.759, "deadline_s": 30.0, "status": 200, "response": {"model": "Qwen/Qwen3.5-9B", "answers": {"pick": {"type": "choice", "choice": "returns", "probabilities": {"logistics": 8.225999253901341e-05, "payment": 7.366072069514265e-05, "returns": 0.9996796860684232, "account": 1.054125854169304e-05, "human": 0.00015385195980102359}, "confidence": 0.999599607585529}}, "usage": {"input_tokens": 1271, "output_tokens": 0}, "metadata": {"method": "native_merged_lora_decision_head", "temperature": 1.8969118766347646, "candidate_sequences": 5, "inference_seconds": 0.22443778, "base_revision": "c202236235762e1c871ad0ccb60c8ee5ba337b9a", "max_length": 4096, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}}}, "notes": {"client_latency_ms": 226}} +{"type": "request", "id": "ticket_router-seed0-0:s10:a0", "group": "ticket_router-seed0-0", "step": 10, "attempt": 0, "retry_of": null, "request": {"model": "jev-latest", "state": {"ticket": {"id": "t_507d761b520a", "title": "Access restricted", "description": "This is not a payment problem. I cannot even get into my account and need access restored."}, "progress": {"ticket_id": "t_507d761b520a"}}, "questions": {"pick": {"type": "choice", "criteria": {"logistics": "Delivery tracking, delivery progress or delivery problems", "payment": "Charges, failed payments or duplicate payments", "returns": "Requests for returns, exchanges or refunds", "account": "Login or account access problems", "human": "Insufficient information, multiple independent requests or an explicit request for a human"}, "instructions": {"rules": "Route the current explicit request to exactly one queue. logistics: delivery tracking or delivery problems. payment: charges, failed payments or duplicate payments. returns: requests for returns, exchanges or refunds. account: login or account access problems. human: insufficient information, multiple independent requests that cannot be uniquely routed, or an explicit request for a human. An explicit human request takes priority. Recognize negation and resolved, quoted or hypothetical background; do not route by keywords alone. Order status is context, not the user's intent. A payment problem with an explicit request for a refund belongs to returns. If no unique queue fits, choose human."}}}}, "images": [], "t_send_ms": 2403.845, "t_recv_ms": 2616.877, "think_ms": 2.13, "deadline_s": 30.0, "status": 200, "response": {"model": "Qwen/Qwen3.5-9B", "answers": {"pick": {"type": "choice", "choice": "account", "probabilities": {"logistics": 7.593819139761939e-06, "payment": 7.964153768490927e-06, "returns": 8.516103100460499e-06, "account": 0.9999169556300482, "human": 5.897029394306403e-05}, "confidence": 0.9998961945375603}}, "usage": {"input_tokens": 1266, "output_tokens": 0}, "metadata": {"method": "native_merged_lora_decision_head", "temperature": 1.8969118766347646, "candidate_sequences": 5, "inference_seconds": 0.211897937, "base_revision": "c202236235762e1c871ad0ccb60c8ee5ba337b9a", "max_length": 4096, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}}}, "notes": {"client_latency_ms": 213}} +{"type": "request", "id": "ticket_router-seed0-0:s11:a0", "group": "ticket_router-seed0-0", "step": 11, "attempt": 0, "retry_of": null, "request": {"model": "jev-latest", "state": {"ticket": {"id": "t_d6e0fd840d60", "title": "Two independent problems", "description": "I need help because I cannot log in to my account. Separately, please check the duplicate charge on another order."}, "progress": {"ticket_id": "t_d6e0fd840d60"}}, "questions": {"pick": {"type": "choice", "criteria": {"logistics": "Delivery tracking, delivery progress or delivery problems", "payment": "Charges, failed payments or duplicate payments", "returns": "Requests for returns, exchanges or refunds", "account": "Login or account access problems", "human": "Insufficient information, multiple independent requests or an explicit request for a human"}, "instructions": {"rules": "Route the current explicit request to exactly one queue. logistics: delivery tracking or delivery problems. payment: charges, failed payments or duplicate payments. returns: requests for returns, exchanges or refunds. account: login or account access problems. human: insufficient information, multiple independent requests that cannot be uniquely routed, or an explicit request for a human. An explicit human request takes priority. Recognize negation and resolved, quoted or hypothetical background; do not route by keywords alone. Order status is context, not the user's intent. A payment problem with an explicit request for a refund belongs to returns. If no unique queue fits, choose human."}}}}, "images": [], "t_send_ms": 2618.91, "t_recv_ms": 2834.341, "think_ms": 2.033, "deadline_s": 30.0, "status": 200, "response": {"model": "Qwen/Qwen3.5-9B", "answers": {"pick": {"type": "choice", "choice": "human", "probabilities": {"logistics": 0.008533442789073413, "payment": 0.23412701171986644, "returns": 0.011313395045061198, "account": 0.2229583079499281, "human": 0.5230678424960709}, "confidence": 0.4038348031200885}}, "usage": {"input_tokens": 1276, "output_tokens": 0}, "metadata": {"method": "native_merged_lora_decision_head", "temperature": 1.8969118766347646, "candidate_sequences": 5, "inference_seconds": 0.214551582, "base_revision": "c202236235762e1c871ad0ccb60c8ee5ba337b9a", "max_length": 4096, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}}}, "notes": {"client_latency_ms": 215}} +{"type": "request", "id": "ticket_router-seed0-0:s12:a0", "group": "ticket_router-seed0-0", "step": 12, "attempt": 0, "retry_of": null, "request": {"model": "jev-latest", "state": {"ticket": {"id": "t_46f559df489a", "title": "Restore account access", "description": "The return process is finished. I now only need the login restriction removed from my account.", "order_status": "returned"}, "progress": {"ticket_id": "t_46f559df489a"}}, "questions": {"pick": {"type": "choice", "criteria": {"logistics": "Delivery tracking, delivery progress or delivery problems", "payment": "Charges, failed payments or duplicate payments", "returns": "Requests for returns, exchanges or refunds", "account": "Login or account access problems", "human": "Insufficient information, multiple independent requests or an explicit request for a human"}, "instructions": {"rules": "Route the current explicit request to exactly one queue. logistics: delivery tracking or delivery problems. payment: charges, failed payments or duplicate payments. returns: requests for returns, exchanges or refunds. account: login or account access problems. human: insufficient information, multiple independent requests that cannot be uniquely routed, or an explicit request for a human. An explicit human request takes priority. Recognize negation and resolved, quoted or hypothetical background; do not route by keywords alone. Order status is context, not the user's intent. A payment problem with an explicit request for a refund belongs to returns. If no unique queue fits, choose human."}}}}, "images": [], "t_send_ms": 2836.458, "t_recv_ms": 3065.485, "think_ms": 2.117, "deadline_s": 30.0, "status": 200, "response": {"model": "Qwen/Qwen3.5-9B", "answers": {"pick": {"type": "choice", "choice": "account", "probabilities": {"logistics": 8.88632003080227e-06, "payment": 6.599620412039255e-06, "returns": 3.3298366055662465e-05, "account": 0.9998825856974883, "human": 6.862999601317798e-05}, "confidence": 0.9998532321218603}}, "usage": {"input_tokens": 1291, "output_tokens": 0}, "metadata": {"method": "native_merged_lora_decision_head", "temperature": 1.8969118766347646, "candidate_sequences": 5, "inference_seconds": 0.227925029, "base_revision": "c202236235762e1c871ad0ccb60c8ee5ba337b9a", "max_length": 4096, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}}}, "notes": {"client_latency_ms": 229}} +{"type": "request", "id": "ticket_router-seed0-0:s13:a0", "group": "ticket_router-seed0-0", "step": 13, "attempt": 0, "retry_of": null, "request": {"model": "jev-latest", "state": {"ticket": {"id": "t_8f0ec563a91a", "title": "Only check delivery", "description": "I am not requesting a return or a refund. I only want to know when it will be delivered.", "order_status": "shipped"}, "progress": {"ticket_id": "t_8f0ec563a91a"}}, "questions": {"pick": {"type": "choice", "criteria": {"logistics": "Delivery tracking, delivery progress or delivery problems", "payment": "Charges, failed payments or duplicate payments", "returns": "Requests for returns, exchanges or refunds", "account": "Login or account access problems", "human": "Insufficient information, multiple independent requests or an explicit request for a human"}, "instructions": {"rules": "Route the current explicit request to exactly one queue. logistics: delivery tracking or delivery problems. payment: charges, failed payments or duplicate payments. returns: requests for returns, exchanges or refunds. account: login or account access problems. human: insufficient information, multiple independent requests that cannot be uniquely routed, or an explicit request for a human. An explicit human request takes priority. Recognize negation and resolved, quoted or hypothetical background; do not route by keywords alone. Order status is context, not the user's intent. A payment problem with an explicit request for a refund belongs to returns. If no unique queue fits, choose human."}}}}, "images": [], "t_send_ms": 3067.724, "t_recv_ms": 3301.679, "think_ms": 2.239, "deadline_s": 30.0, "status": 200, "response": {"model": "Qwen/Qwen3.5-9B", "answers": {"pick": {"type": "choice", "choice": "logistics", "probabilities": {"logistics": 0.9991642941173402, "payment": 2.1410567762384555e-05, "returns": 3.31956125522694e-05, "account": 2.6466309660068705e-05, "human": 0.0007546333926851138}, "confidence": 0.9989553676466753}}, "usage": {"input_tokens": 1311, "output_tokens": 0}, "metadata": {"method": "native_merged_lora_decision_head", "temperature": 1.8969118766347646, "candidate_sequences": 5, "inference_seconds": 0.232860875, "base_revision": "c202236235762e1c871ad0ccb60c8ee5ba337b9a", "max_length": 4096, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}}}, "notes": {"client_latency_ms": 234}} +{"type": "request", "id": "ticket_router-seed0-0:s14:a0", "group": "ticket_router-seed0-0", "step": 14, "attempt": 0, "retry_of": null, "request": {"model": "jev-latest", "state": {"ticket": {"id": "t_f5251390fb88", "title": "Login verification code", "description": "The login verification code never arrives, so I cannot access my account."}, "progress": {"ticket_id": "t_f5251390fb88"}}, "questions": {"pick": {"type": "choice", "criteria": {"logistics": "Delivery tracking, delivery progress or delivery problems", "payment": "Charges, failed payments or duplicate payments", "returns": "Requests for returns, exchanges or refunds", "account": "Login or account access problems", "human": "Insufficient information, multiple independent requests or an explicit request for a human"}, "instructions": {"rules": "Route the current explicit request to exactly one queue. logistics: delivery tracking or delivery problems. payment: charges, failed payments or duplicate payments. returns: requests for returns, exchanges or refunds. account: login or account access problems. human: insufficient information, multiple independent requests that cannot be uniquely routed, or an explicit request for a human. An explicit human request takes priority. Recognize negation and resolved, quoted or hypothetical background; do not route by keywords alone. Order status is context, not the user's intent. A payment problem with an explicit request for a refund belongs to returns. If no unique queue fits, choose human."}}}}, "images": [], "t_send_ms": 3304.038, "t_recv_ms": 3512.061, "think_ms": 2.359, "deadline_s": 30.0, "status": 200, "response": {"model": "Qwen/Qwen3.5-9B", "answers": {"pick": {"type": "choice", "choice": "account", "probabilities": {"logistics": 1.7038059545106095e-05, "payment": 1.3134981565688484e-05, "returns": 1.649028347448762e-05, "account": 0.9997210149080872, "human": 0.00023232176732765678}, "confidence": 0.9996512686351089}}, "usage": {"input_tokens": 1226, "output_tokens": 0}, "metadata": {"method": "native_merged_lora_decision_head", "temperature": 1.8969118766347646, "candidate_sequences": 5, "inference_seconds": 0.206927972, "base_revision": "c202236235762e1c871ad0ccb60c8ee5ba337b9a", "max_length": 4096, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}}}, "notes": {"client_latency_ms": 208}} +{"type": "request", "id": "ticket_router-seed0-0:s15:a0", "group": "ticket_router-seed0-0", "step": 15, "attempt": 0, "retry_of": null, "request": {"model": "jev-latest", "state": {"ticket": {"id": "t_e4351833c8c8", "title": "Delivery screenshot", "description": "A friend suggested a refund, but I do not want that. Please first check why my parcel is marked as delivered.", "order_status": "delivered"}, "progress": {"ticket_id": "t_e4351833c8c8"}}, "questions": {"pick": {"type": "choice", "criteria": {"logistics": "Delivery tracking, delivery progress or delivery problems", "payment": "Charges, failed payments or duplicate payments", "returns": "Requests for returns, exchanges or refunds", "account": "Login or account access problems", "human": "Insufficient information, multiple independent requests or an explicit request for a human"}, "instructions": {"rules": "Route the current explicit request to exactly one queue. logistics: delivery tracking or delivery problems. payment: charges, failed payments or duplicate payments. returns: requests for returns, exchanges or refunds. account: login or account access problems. human: insufficient information, multiple independent requests that cannot be uniquely routed, or an explicit request for a human. An explicit human request takes priority. Recognize negation and resolved, quoted or hypothetical background; do not route by keywords alone. Order status is context, not the user's intent. A payment problem with an explicit request for a refund belongs to returns. If no unique queue fits, choose human."}}}}, "images": [], "t_send_ms": 3514.429, "t_recv_ms": 3758.442, "think_ms": 2.368, "deadline_s": 30.0, "status": 200, "response": {"model": "Qwen/Qwen3.5-9B", "answers": {"pick": {"type": "choice", "choice": "logistics", "probabilities": {"logistics": 0.9660599452329868, "payment": 0.001978095354269819, "returns": 0.01874859998397356, "account": 0.0007022066365049155, "human": 0.012511152792264989}, "confidence": 0.9575749315412335}}, "usage": {"input_tokens": 1321, "output_tokens": 0}, "metadata": {"method": "native_merged_lora_decision_head", "temperature": 1.8969118766347646, "candidate_sequences": 5, "inference_seconds": 0.242904452, "base_revision": "c202236235762e1c871ad0ccb60c8ee5ba337b9a", "max_length": 4096, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}}}, "notes": {"client_latency_ms": 244}} +{"type": "request", "id": "ticket_router-seed0-0:s16:a0", "group": "ticket_router-seed0-0", "step": 16, "attempt": 0, "retry_of": null, "request": {"model": "jev-latest", "state": {"ticket": {"id": "t_5b12b66a4428", "title": "Cannot access my profile", "description": "I want to view orders in my profile, but the system keeps signing me out. Please fix my account access."}, "progress": {"ticket_id": "t_5b12b66a4428"}}, "questions": {"pick": {"type": "choice", "criteria": {"logistics": "Delivery tracking, delivery progress or delivery problems", "payment": "Charges, failed payments or duplicate payments", "returns": "Requests for returns, exchanges or refunds", "account": "Login or account access problems", "human": "Insufficient information, multiple independent requests or an explicit request for a human"}, "instructions": {"rules": "Route the current explicit request to exactly one queue. logistics: delivery tracking or delivery problems. payment: charges, failed payments or duplicate payments. returns: requests for returns, exchanges or refunds. account: login or account access problems. human: insufficient information, multiple independent requests that cannot be uniquely routed, or an explicit request for a human. An explicit human request takes priority. Recognize negation and resolved, quoted or hypothetical background; do not route by keywords alone. Order status is context, not the user's intent. A payment problem with an explicit request for a refund belongs to returns. If no unique queue fits, choose human."}}}}, "images": [], "t_send_ms": 3761.809, "t_recv_ms": 3990.716, "think_ms": 3.367, "deadline_s": 30.0, "status": 200, "response": {"model": "Qwen/Qwen3.5-9B", "answers": {"pick": {"type": "choice", "choice": "account", "probabilities": {"logistics": 1.6106916191126148e-05, "payment": 1.2079354090227312e-05, "returns": 1.3252749012451188e-05, "account": 0.9998181663161784, "human": 0.00014039466452779443}, "confidence": 0.9997727078952229}}, "usage": {"input_tokens": 1296, "output_tokens": 0}, "metadata": {"method": "native_merged_lora_decision_head", "temperature": 1.8969118766347646, "candidate_sequences": 5, "inference_seconds": 0.227551348, "base_revision": "c202236235762e1c871ad0ccb60c8ee5ba337b9a", "max_length": 4096, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}}}, "notes": {"client_latency_ms": 229}} +{"type": "request", "id": "ticket_router-seed0-0:s17:a0", "group": "ticket_router-seed0-0", "step": 17, "attempt": 0, "retry_of": null, "request": {"model": "jev-latest", "state": {"ticket": {"id": "t_07772aa5f822", "title": "Two charges", "description": "I was charged twice for the same order. Please check the payment records and explain the duplicate charge.", "order_status": "paid"}, "progress": {"ticket_id": "t_07772aa5f822"}}, "questions": {"pick": {"type": "choice", "criteria": {"logistics": "Delivery tracking, delivery progress or delivery problems", "payment": "Charges, failed payments or duplicate payments", "returns": "Requests for returns, exchanges or refunds", "account": "Login or account access problems", "human": "Insufficient information, multiple independent requests or an explicit request for a human"}, "instructions": {"rules": "Route the current explicit request to exactly one queue. logistics: delivery tracking or delivery problems. payment: charges, failed payments or duplicate payments. returns: requests for returns, exchanges or refunds. account: login or account access problems. human: insufficient information, multiple independent requests that cannot be uniquely routed, or an explicit request for a human. An explicit human request takes priority. Recognize negation and resolved, quoted or hypothetical background; do not route by keywords alone. Order status is context, not the user's intent. A payment problem with an explicit request for a refund belongs to returns. If no unique queue fits, choose human."}}}}, "images": [], "t_send_ms": 3993.494, "t_recv_ms": 4223.389, "think_ms": 2.778, "deadline_s": 30.0, "status": 200, "response": {"model": "Qwen/Qwen3.5-9B", "answers": {"pick": {"type": "choice", "choice": "payment", "probabilities": {"logistics": 1.687615418238353e-05, "payment": 0.9996667217358574, "returns": 3.0281125845150304e-05, "account": 1.3650558432942852e-05, "human": 0.00027247042568217346}, "confidence": 0.9995834021698216}}, "usage": {"input_tokens": 1296, "output_tokens": 0}, "metadata": {"method": "native_merged_lora_decision_head", "temperature": 1.8969118766347646, "candidate_sequences": 5, "inference_seconds": 0.228778376, "base_revision": "c202236235762e1c871ad0ccb60c8ee5ba337b9a", "max_length": 4096, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}}}, "notes": {"client_latency_ms": 230}} +{"type": "request", "id": "ticket_router-seed0-0:s18:a0", "group": "ticket_router-seed0-0", "step": 18, "attempt": 0, "retry_of": null, "request": {"model": "jev-latest", "state": {"ticket": {"id": "t_ace2461b953a", "title": "Wallet charged twice", "description": "The delivery inquiry is finished. This time I only want to know why my wallet was charged twice.", "order_status": "delivered"}, "progress": {"ticket_id": "t_ace2461b953a"}}, "questions": {"pick": {"type": "choice", "criteria": {"logistics": "Delivery tracking, delivery progress or delivery problems", "payment": "Charges, failed payments or duplicate payments", "returns": "Requests for returns, exchanges or refunds", "account": "Login or account access problems", "human": "Insufficient information, multiple independent requests or an explicit request for a human"}, "instructions": {"rules": "Route the current explicit request to exactly one queue. logistics: delivery tracking or delivery problems. payment: charges, failed payments or duplicate payments. returns: requests for returns, exchanges or refunds. account: login or account access problems. human: insufficient information, multiple independent requests that cannot be uniquely routed, or an explicit request for a human. An explicit human request takes priority. Recognize negation and resolved, quoted or hypothetical background; do not route by keywords alone. Order status is context, not the user's intent. A payment problem with an explicit request for a refund belongs to returns. If no unique queue fits, choose human."}}}}, "images": [], "t_send_ms": 4225.933, "t_recv_ms": 4458.438, "think_ms": 2.544, "deadline_s": 30.0, "status": 200, "response": {"model": "Qwen/Qwen3.5-9B", "answers": {"pick": {"type": "choice", "choice": "payment", "probabilities": {"logistics": 1.890610679988126e-05, "payment": 0.9996225835132857, "returns": 4.0511447477002445e-05, "account": 1.4542963747080139e-05, "human": 0.00030345596869023133}, "confidence": 0.9995282293916071}}, "usage": {"input_tokens": 1296, "output_tokens": 0}, "metadata": {"method": "native_merged_lora_decision_head", "temperature": 1.8969118766347646, "candidate_sequences": 5, "inference_seconds": 0.231420811, "base_revision": "c202236235762e1c871ad0ccb60c8ee5ba337b9a", "max_length": 4096, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}}}, "notes": {"client_latency_ms": 232}} +{"type": "request", "id": "ticket_router-seed0-0:s19:a0", "group": "ticket_router-seed0-0", "step": 19, "attempt": 0, "retry_of": null, "request": {"model": "jev-latest", "state": {"ticket": {"id": "t_9f4c2e9e9fc4", "title": "Login on a new phone", "description": "My old phone no longer works. The new device keeps reporting account verification failure. Please restore my login."}, "progress": {"ticket_id": "t_9f4c2e9e9fc4"}}, "questions": {"pick": {"type": "choice", "criteria": {"logistics": "Delivery tracking, delivery progress or delivery problems", "payment": "Charges, failed payments or duplicate payments", "returns": "Requests for returns, exchanges or refunds", "account": "Login or account access problems", "human": "Insufficient information, multiple independent requests or an explicit request for a human"}, "instructions": {"rules": "Route the current explicit request to exactly one queue. logistics: delivery tracking or delivery problems. payment: charges, failed payments or duplicate payments. returns: requests for returns, exchanges or refunds. account: login or account access problems. human: insufficient information, multiple independent requests that cannot be uniquely routed, or an explicit request for a human. An explicit human request takes priority. Recognize negation and resolved, quoted or hypothetical background; do not route by keywords alone. Order status is context, not the user's intent. A payment problem with an explicit request for a refund belongs to returns. If no unique queue fits, choose human."}}}}, "images": [], "t_send_ms": 4461.127, "t_recv_ms": 4683.126, "think_ms": 2.689, "deadline_s": 30.0, "status": 200, "response": {"model": "Qwen/Qwen3.5-9B", "answers": {"pick": {"type": "choice", "choice": "account", "probabilities": {"logistics": 1.4762556155548501e-05, "payment": 0.00010825429812026079, "returns": 2.2386233835687076e-05, "account": 0.9989706675423696, "human": 0.000883929369518872}, "confidence": 0.998713334427962}}, "usage": {"input_tokens": 1281, "output_tokens": 0}, "metadata": {"method": "native_merged_lora_decision_head", "temperature": 1.8969118766347646, "candidate_sequences": 5, "inference_seconds": 0.22098671, "base_revision": "c202236235762e1c871ad0ccb60c8ee5ba337b9a", "max_length": 4096, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}}}, "notes": {"client_latency_ms": 222}} +{"type": "request", "id": "ticket_router-seed0-0:s20:a0", "group": "ticket_router-seed0-0", "step": 20, "attempt": 0, "retry_of": null, "request": {"model": "jev-latest", "state": {"ticket": {"id": "t_c8fd45b69015", "title": "Payment needs checking", "description": "I can log in to my account again, but checkout now keeps saying the charge failed. Please check the payment.", "order_status": "unpaid"}, "progress": {"ticket_id": "t_c8fd45b69015"}}, "questions": {"pick": {"type": "choice", "criteria": {"logistics": "Delivery tracking, delivery progress or delivery problems", "payment": "Charges, failed payments or duplicate payments", "returns": "Requests for returns, exchanges or refunds", "account": "Login or account access problems", "human": "Insufficient information, multiple independent requests or an explicit request for a human"}, "instructions": {"rules": "Route the current explicit request to exactly one queue. logistics: delivery tracking or delivery problems. payment: charges, failed payments or duplicate payments. returns: requests for returns, exchanges or refunds. account: login or account access problems. human: insufficient information, multiple independent requests that cannot be uniquely routed, or an explicit request for a human. An explicit human request takes priority. Recognize negation and resolved, quoted or hypothetical background; do not route by keywords alone. Order status is context, not the user's intent. A payment problem with an explicit request for a refund belongs to returns. If no unique queue fits, choose human."}}}}, "images": [], "t_send_ms": 4685.685, "t_recv_ms": 4922.326, "think_ms": 2.559, "deadline_s": 30.0, "status": 200, "response": {"model": "Qwen/Qwen3.5-9B", "answers": {"pick": {"type": "choice", "choice": "payment", "probabilities": {"logistics": 1.0470740136835071e-05, "payment": 0.9996623466438508, "returns": 9.167119967718128e-06, "account": 5.020938720728247e-05, "human": 0.0002678061088375178}, "confidence": 0.9995779333048135}}, "usage": {"input_tokens": 1311, "output_tokens": 0}, "metadata": {"method": "native_merged_lora_decision_head", "temperature": 1.8969118766347646, "candidate_sequences": 5, "inference_seconds": 0.235535016, "base_revision": "c202236235762e1c871ad0ccb60c8ee5ba337b9a", "max_length": 4096, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}}}, "notes": {"client_latency_ms": 237}} +{"type": "request", "id": "ticket_router-seed0-0:s21:a0", "group": "ticket_router-seed0-0", "step": 21, "attempt": 0, "retry_of": null, "request": {"model": "jev-latest", "state": {"ticket": {"id": "t_c72f978e4f83", "title": "Human help with payment", "description": "The payment failed again. This time I want a human support agent to handle it directly.", "order_status": "unpaid"}, "progress": {"ticket_id": "t_c72f978e4f83"}}, "questions": {"pick": {"type": "choice", "criteria": {"logistics": "Delivery tracking, delivery progress or delivery problems", "payment": "Charges, failed payments or duplicate payments", "returns": "Requests for returns, exchanges or refunds", "account": "Login or account access problems", "human": "Insufficient information, multiple independent requests or an explicit request for a human"}, "instructions": {"rules": "Route the current explicit request to exactly one queue. logistics: delivery tracking or delivery problems. payment: charges, failed payments or duplicate payments. returns: requests for returns, exchanges or refunds. account: login or account access problems. human: insufficient information, multiple independent requests that cannot be uniquely routed, or an explicit request for a human. An explicit human request takes priority. Recognize negation and resolved, quoted or hypothetical background; do not route by keywords alone. Order status is context, not the user's intent. A payment problem with an explicit request for a refund belongs to returns. If no unique queue fits, choose human."}}}}, "images": [], "t_send_ms": 4925.029, "t_recv_ms": 5158.952, "think_ms": 2.703, "deadline_s": 30.0, "status": 200, "response": {"model": "Qwen/Qwen3.5-9B", "answers": {"pick": {"type": "choice", "choice": "human", "probabilities": {"logistics": 8.125402634321574e-06, "payment": 0.0021753519401005927, "returns": 1.085569090283665e-05, "account": 6.9857745686196725e-06, "human": 0.9977986811917936}, "confidence": 0.997248351489742}}, "usage": {"input_tokens": 1301, "output_tokens": 0}, "metadata": {"method": "native_merged_lora_decision_head", "temperature": 1.8969118766347646, "candidate_sequences": 5, "inference_seconds": 0.232999214, "base_revision": "c202236235762e1c871ad0ccb60c8ee5ba337b9a", "max_length": 4096, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}}}, "notes": {"client_latency_ms": 234}} +{"type": "request", "id": "ticket_router-seed0-0:s22:a0", "group": "ticket_router-seed0-0", "step": 22, "attempt": 0, "retry_of": null, "request": {"model": "jev-latest", "state": {"ticket": {"id": "t_2ff652a83e3b", "title": "Return after payment", "description": "Payment succeeded and I can log in to my account, but the item arrived damaged and I want to return it.", "order_status": "delivered"}, "progress": {"ticket_id": "t_2ff652a83e3b"}}, "questions": {"pick": {"type": "choice", "criteria": {"logistics": "Delivery tracking, delivery progress or delivery problems", "payment": "Charges, failed payments or duplicate payments", "returns": "Requests for returns, exchanges or refunds", "account": "Login or account access problems", "human": "Insufficient information, multiple independent requests or an explicit request for a human"}, "instructions": {"rules": "Route the current explicit request to exactly one queue. logistics: delivery tracking or delivery problems. payment: charges, failed payments or duplicate payments. returns: requests for returns, exchanges or refunds. account: login or account access problems. human: insufficient information, multiple independent requests that cannot be uniquely routed, or an explicit request for a human. An explicit human request takes priority. Recognize negation and resolved, quoted or hypothetical background; do not route by keywords alone. Order status is context, not the user's intent. A payment problem with an explicit request for a refund belongs to returns. If no unique queue fits, choose human."}}}}, "images": [], "t_send_ms": 5161.694, "t_recv_ms": 5398.202, "think_ms": 2.742, "deadline_s": 30.0, "status": 200, "response": {"model": "Qwen/Qwen3.5-9B", "answers": {"pick": {"type": "choice", "choice": "returns", "probabilities": {"logistics": 0.0001300197626183939, "payment": 8.58976837813581e-06, "returns": 0.999620440502762, "account": 1.694796589782436e-05, "human": 0.0002240020003437183}, "confidence": 0.9995255506284525}}, "usage": {"input_tokens": 1321, "output_tokens": 0}, "metadata": {"method": "native_merged_lora_decision_head", "temperature": 1.8969118766347646, "candidate_sequences": 5, "inference_seconds": 0.235447071, "base_revision": "c202236235762e1c871ad0ccb60c8ee5ba337b9a", "max_length": 4096, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}}}, "notes": {"client_latency_ms": 237}} +{"type": "request", "id": "ticket_router-seed0-0:s23:a0", "group": "ticket_router-seed0-0", "step": 23, "attempt": 0, "retry_of": null, "request": {"model": "jev-latest", "state": {"ticket": {"id": "t_ec027c12e4da", "title": "Refund the duplicate charge", "description": "I was charged twice for one order. I explicitly want a refund of the extra charge.", "order_status": "paid"}, "progress": {"ticket_id": "t_ec027c12e4da"}}, "questions": {"pick": {"type": "choice", "criteria": {"logistics": "Delivery tracking, delivery progress or delivery problems", "payment": "Charges, failed payments or duplicate payments", "returns": "Requests for returns, exchanges or refunds", "account": "Login or account access problems", "human": "Insufficient information, multiple independent requests or an explicit request for a human"}, "instructions": {"rules": "Route the current explicit request to exactly one queue. logistics: delivery tracking or delivery problems. payment: charges, failed payments or duplicate payments. returns: requests for returns, exchanges or refunds. account: login or account access problems. human: insufficient information, multiple independent requests that cannot be uniquely routed, or an explicit request for a human. An explicit human request takes priority. Recognize negation and resolved, quoted or hypothetical background; do not route by keywords alone. Order status is context, not the user's intent. A payment problem with an explicit request for a refund belongs to returns. If no unique queue fits, choose human."}}}}, "images": [], "t_send_ms": 5400.912, "t_recv_ms": 5626.645, "think_ms": 2.71, "deadline_s": 30.0, "status": 200, "response": {"model": "Qwen/Qwen3.5-9B", "answers": {"pick": {"type": "choice", "choice": "payment", "probabilities": {"logistics": 5.8896465207716194e-05, "payment": 0.9392633780412096, "returns": 0.06019086938364967, "account": 3.189592198132583e-05, "human": 0.0004549601879515318}, "confidence": 0.9240792225515121}}, "usage": {"input_tokens": 1281, "output_tokens": 0}, "metadata": {"method": "native_merged_lora_decision_head", "temperature": 1.8969118766347646, "candidate_sequences": 5, "inference_seconds": 0.224772908, "base_revision": "c202236235762e1c871ad0ccb60c8ee5ba337b9a", "max_length": 4096, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}}}, "notes": {"client_latency_ms": 226}} +{"type": "request", "id": "ticket_router-seed0-0:s24:a0", "group": "ticket_router-seed0-0", "step": 24, "attempt": 0, "retry_of": null, "request": {"model": "jev-latest", "state": {"ticket": {"id": "t_609c2a02ab84", "title": "Check the payment", "description": "I am not asking for a refund or a return. I need to find out why the payment failed.", "order_status": "unpaid"}, "progress": {"ticket_id": "t_609c2a02ab84"}}, "questions": {"pick": {"type": "choice", "criteria": {"logistics": "Delivery tracking, delivery progress or delivery problems", "payment": "Charges, failed payments or duplicate payments", "returns": "Requests for returns, exchanges or refunds", "account": "Login or account access problems", "human": "Insufficient information, multiple independent requests or an explicit request for a human"}, "instructions": {"rules": "Route the current explicit request to exactly one queue. logistics: delivery tracking or delivery problems. payment: charges, failed payments or duplicate payments. returns: requests for returns, exchanges or refunds. account: login or account access problems. human: insufficient information, multiple independent requests that cannot be uniquely routed, or an explicit request for a human. An explicit human request takes priority. Recognize negation and resolved, quoted or hypothetical background; do not route by keywords alone. Order status is context, not the user's intent. A payment problem with an explicit request for a refund belongs to returns. If no unique queue fits, choose human."}}}}, "images": [], "t_send_ms": 5629.43, "t_recv_ms": 5866.603, "think_ms": 2.785, "deadline_s": 30.0, "status": 200, "response": {"model": "Qwen/Qwen3.5-9B", "answers": {"pick": {"type": "choice", "choice": "payment", "probabilities": {"logistics": 6.178634680370836e-06, "payment": 0.9998844602651847, "returns": 4.92768563628418e-06, "account": 1.4396330414934275e-05, "human": 9.003708408346635e-05}, "confidence": 0.9998555753314808}}, "usage": {"input_tokens": 1311, "output_tokens": 0}, "metadata": {"method": "native_merged_lora_decision_head", "temperature": 1.8969118766347646, "candidate_sequences": 5, "inference_seconds": 0.236049231, "base_revision": "c202236235762e1c871ad0ccb60c8ee5ba337b9a", "max_length": 4096, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}}}, "notes": {"client_latency_ms": 237}} +{"type": "request", "id": "ticket_router-seed0-0:s25:a0", "group": "ticket_router-seed0-0", "step": 25, "attempt": 0, "retry_of": null, "request": {"model": "jev-latest", "state": {"ticket": {"id": "t_6cb98186dde0", "title": "Change the delivery address", "description": "I have moved. Please contact the courier to redirect this order to my new address.", "order_status": "shipped"}, "progress": {"ticket_id": "t_6cb98186dde0"}}, "questions": {"pick": {"type": "choice", "criteria": {"logistics": "Delivery tracking, delivery progress or delivery problems", "payment": "Charges, failed payments or duplicate payments", "returns": "Requests for returns, exchanges or refunds", "account": "Login or account access problems", "human": "Insufficient information, multiple independent requests or an explicit request for a human"}, "instructions": {"rules": "Route the current explicit request to exactly one queue. logistics: delivery tracking or delivery problems. payment: charges, failed payments or duplicate payments. returns: requests for returns, exchanges or refunds. account: login or account access problems. human: insufficient information, multiple independent requests that cannot be uniquely routed, or an explicit request for a human. An explicit human request takes priority. Recognize negation and resolved, quoted or hypothetical background; do not route by keywords alone. Order status is context, not the user's intent. A payment problem with an explicit request for a refund belongs to returns. If no unique queue fits, choose human."}}}}, "images": [], "t_send_ms": 5869.644, "t_recv_ms": 6089.421, "think_ms": 3.041, "deadline_s": 30.0, "status": 200, "response": {"model": "Qwen/Qwen3.5-9B", "answers": {"pick": {"type": "choice", "choice": "logistics", "probabilities": {"logistics": 0.9902800892207427, "payment": 0.0001577608021503771, "returns": 0.00025682166318490176, "account": 0.0001774427090323029, "human": 0.009127885604889632}, "confidence": 0.9878501115259283}}, "usage": {"input_tokens": 1276, "output_tokens": 0}, "metadata": {"method": "native_merged_lora_decision_head", "temperature": 1.8969118766347646, "candidate_sequences": 5, "inference_seconds": 0.218644569, "base_revision": "c202236235762e1c871ad0ccb60c8ee5ba337b9a", "max_length": 4096, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}}}, "notes": {"client_latency_ms": 220}} +{"type": "request", "id": "ticket_router-seed0-0:s26:a0", "group": "ticket_router-seed0-0", "step": 26, "attempt": 0, "retry_of": null, "request": {"model": "jev-latest", "state": {"ticket": {"id": "t_273060d164f5", "title": "Cancel the purchase", "description": "I no longer need this item. I want to cancel the purchase and get my money back.", "order_status": "paid"}, "progress": {"ticket_id": "t_273060d164f5"}}, "questions": {"pick": {"type": "choice", "criteria": {"logistics": "Delivery tracking, delivery progress or delivery problems", "payment": "Charges, failed payments or duplicate payments", "returns": "Requests for returns, exchanges or refunds", "account": "Login or account access problems", "human": "Insufficient information, multiple independent requests or an explicit request for a human"}, "instructions": {"rules": "Route the current explicit request to exactly one queue. logistics: delivery tracking or delivery problems. payment: charges, failed payments or duplicate payments. returns: requests for returns, exchanges or refunds. account: login or account access problems. human: insufficient information, multiple independent requests that cannot be uniquely routed, or an explicit request for a human. An explicit human request takes priority. Recognize negation and resolved, quoted or hypothetical background; do not route by keywords alone. Order status is context, not the user's intent. A payment problem with an explicit request for a refund belongs to returns. If no unique queue fits, choose human."}}}}, "images": [], "t_send_ms": 6092.902, "t_recv_ms": 6327.255, "think_ms": 3.481, "deadline_s": 30.0, "status": 200, "response": {"model": "Qwen/Qwen3.5-9B", "answers": {"pick": {"type": "choice", "choice": "returns", "probabilities": {"logistics": 1.9850737418014602e-05, "payment": 0.00012015377993516422, "returns": 0.9996511170627909, "account": 8.209506243171576e-06, "human": 0.00020066891361284947}, "confidence": 0.9995638963284886}}, "usage": {"input_tokens": 1306, "output_tokens": 0}, "metadata": {"method": "native_merged_lora_decision_head", "temperature": 1.8969118766347646, "candidate_sequences": 5, "inference_seconds": 0.233359452, "base_revision": "c202236235762e1c871ad0ccb60c8ee5ba337b9a", "max_length": 4096, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}}}, "notes": {"client_latency_ms": 234}} +{"type": "request", "id": "ticket_router-seed0-0:s27:a0", "group": "ticket_router-seed0-0", "step": 27, "attempt": 0, "retry_of": null, "request": {"model": "jev-latest", "state": {"ticket": {"id": "t_892cab266e67", "title": "Problems with two orders", "description": "Please check delivery progress for order one and request a refund for order two. I need both done."}, "progress": {"ticket_id": "t_892cab266e67"}}, "questions": {"pick": {"type": "choice", "criteria": {"logistics": "Delivery tracking, delivery progress or delivery problems", "payment": "Charges, failed payments or duplicate payments", "returns": "Requests for returns, exchanges or refunds", "account": "Login or account access problems", "human": "Insufficient information, multiple independent requests or an explicit request for a human"}, "instructions": {"rules": "Route the current explicit request to exactly one queue. logistics: delivery tracking or delivery problems. payment: charges, failed payments or duplicate payments. returns: requests for returns, exchanges or refunds. account: login or account access problems. human: insufficient information, multiple independent requests that cannot be uniquely routed, or an explicit request for a human. An explicit human request takes priority. Recognize negation and resolved, quoted or hypothetical background; do not route by keywords alone. Order status is context, not the user's intent. A payment problem with an explicit request for a refund belongs to returns. If no unique queue fits, choose human."}}}}, "images": [], "t_send_ms": 6330.301, "t_recv_ms": 6549.256, "think_ms": 3.046, "deadline_s": 30.0, "status": 200, "response": {"model": "Qwen/Qwen3.5-9B", "answers": {"pick": {"type": "choice", "choice": "human", "probabilities": {"logistics": 0.001824690610258077, "payment": 0.0010998180157114504, "returns": 0.004607264823952921, "account": 4.2609915804913144e-05, "human": 0.9924256166342726}, "confidence": 0.9905320207928409}}, "usage": {"input_tokens": 1266, "output_tokens": 0}, "metadata": {"method": "native_merged_lora_decision_head", "temperature": 1.8969118766347646, "candidate_sequences": 5, "inference_seconds": 0.218365221, "base_revision": "c202236235762e1c871ad0ccb60c8ee5ba337b9a", "max_length": 4096, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}}}, "notes": {"client_latency_ms": 219}} +{"type": "request", "id": "ticket_router-seed0-0:s28:a0", "group": "ticket_router-seed0-0", "step": 28, "attempt": 0, "retry_of": null, "request": {"model": "jev-latest", "state": {"ticket": {"id": "t_afe6633569aa", "title": "Exchange the size", "description": "The shoes I received are too small. Please exchange them for a larger size.", "order_status": "delivered"}, "progress": {"ticket_id": "t_afe6633569aa"}}, "questions": {"pick": {"type": "choice", "criteria": {"logistics": "Delivery tracking, delivery progress or delivery problems", "payment": "Charges, failed payments or duplicate payments", "returns": "Requests for returns, exchanges or refunds", "account": "Login or account access problems", "human": "Insufficient information, multiple independent requests or an explicit request for a human"}, "instructions": {"rules": "Route the current explicit request to exactly one queue. logistics: delivery tracking or delivery problems. payment: charges, failed payments or duplicate payments. returns: requests for returns, exchanges or refunds. account: login or account access problems. human: insufficient information, multiple independent requests that cannot be uniquely routed, or an explicit request for a human. An explicit human request takes priority. Recognize negation and resolved, quoted or hypothetical background; do not route by keywords alone. Order status is context, not the user's intent. A payment problem with an explicit request for a refund belongs to returns. If no unique queue fits, choose human."}}}}, "images": [], "t_send_ms": 6552.119, "t_recv_ms": 6775.38, "think_ms": 2.863, "deadline_s": 30.0, "status": 200, "response": {"model": "Qwen/Qwen3.5-9B", "answers": {"pick": {"type": "choice", "choice": "returns", "probabilities": {"logistics": 6.35327579808697e-05, "payment": 7.136534904658197e-06, "returns": 0.9998340608555417, "account": 6.9482480272629526e-06, "human": 8.832160354558104e-05}, "confidence": 0.9997925760694271}}, "usage": {"input_tokens": 1266, "output_tokens": 0}, "metadata": {"method": "native_merged_lora_decision_head", "temperature": 1.8969118766347646, "candidate_sequences": 5, "inference_seconds": 0.222680873, "base_revision": "c202236235762e1c871ad0ccb60c8ee5ba337b9a", "max_length": 4096, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}}}, "notes": {"client_latency_ms": 223}} +{"type": "request", "id": "ticket_router-seed0-0:s29:a0", "group": "ticket_router-seed0-0", "step": 29, "attempt": 0, "retry_of": null, "request": {"model": "jev-latest", "state": {"ticket": {"id": "t_c5314933eb53", "title": "Missing details", "description": "Please deal with the matter I mentioned yesterday."}, "progress": {"ticket_id": "t_c5314933eb53"}}, "questions": {"pick": {"type": "choice", "criteria": {"logistics": "Delivery tracking, delivery progress or delivery problems", "payment": "Charges, failed payments or duplicate payments", "returns": "Requests for returns, exchanges or refunds", "account": "Login or account access problems", "human": "Insufficient information, multiple independent requests or an explicit request for a human"}, "instructions": {"rules": "Route the current explicit request to exactly one queue. logistics: delivery tracking or delivery problems. payment: charges, failed payments or duplicate payments. returns: requests for returns, exchanges or refunds. account: login or account access problems. human: insufficient information, multiple independent requests that cannot be uniquely routed, or an explicit request for a human. An explicit human request takes priority. Recognize negation and resolved, quoted or hypothetical background; do not route by keywords alone. Order status is context, not the user's intent. A payment problem with an explicit request for a refund belongs to returns. If no unique queue fits, choose human."}}}}, "images": [], "t_send_ms": 6778.354, "t_recv_ms": 6981.161, "think_ms": 2.974, "deadline_s": 30.0, "status": 200, "response": {"model": "Qwen/Qwen3.5-9B", "answers": {"pick": {"type": "choice", "choice": "human", "probabilities": {"logistics": 0.008172840935415475, "payment": 0.005364502276936421, "returns": 0.00396555664511958, "account": 0.005441077740662394, "human": 0.9770560224018661}, "confidence": 0.9713200280023326}}, "usage": {"input_tokens": 1196, "output_tokens": 0}, "metadata": {"method": "native_merged_lora_decision_head", "temperature": 1.8969118766347646, "candidate_sequences": 5, "inference_seconds": 0.202192567, "base_revision": "c202236235762e1c871ad0ccb60c8ee5ba337b9a", "max_length": 4096, "prefix_cache": {"enabled": false, "mode": "independent_candidates"}}}, "notes": {"client_latency_ms": 203}} +{"type": "episode_end", "group": "ticket_router-seed0-0", "status": "done", "steps": 30} diff --git a/benchmarks/traces/tracefile.py b/benchmarks/traces/tracefile.py new file mode 100644 index 00000000..53c83c36 --- /dev/null +++ b/benchmarks/traces/tracefile.py @@ -0,0 +1,140 @@ +#!/usr/bin/env python3 +"""Read, check and write decision traces (`s1-trace/1`), and convert recorded requests. + +A trace is JSONL: one `header`, then `request` records (one per attempt) and optional +`episode_end` records. See README.md in this directory. +""" + +import argparse +import hashlib +import json +from pathlib import Path + +FORMAT = "s1-trace/1" +CAPTURES = {"client:transport", "converted"} # record_s1a.py; from_ticks + + +def canonical(value): + """Bytes of a JSON value with its key order kept, as the request was sent.""" + return json.dumps(value, ensure_ascii=False, separators=(",", ":")).encode() + + +def state_digest(request, images=()): + """Identity of what a request asks about: its state bytes and its images' pixels.""" + h = hashlib.sha256(canonical(request.get("state"))) + for image in images: + h.update(image["pixels_sha256"].encode()) + return h.hexdigest() + + +def check(records): + """Return (header, requests, ends) or raise ValueError naming the first problem.""" + if not records or records[0].get("type") != "header": + raise ValueError("a trace starts with one header record") + header = records[0] + if header.get("format") != FORMAT: + raise ValueError(f"header format must be {FORMAT}") + if header.get("capture") not in CAPTURES: + raise ValueError(f"header capture must be one of {sorted(CAPTURES)}") + requests, ends, ids, last = [], {}, set(), {} + for number, record in enumerate(records[1:], start=2): + kind = record.get("type") + where = f"record {number}" + if kind == "request": + for key in ("id", "group"): + if not isinstance(record.get(key), str) or not record[key]: + raise ValueError(f"{where}: {key} must be a nonempty string") + if record["id"] in ids: + raise ValueError(f"{where}: duplicate id {record['id']}") + if record["group"] in ends: + raise ValueError(f"{where}: episode {record['group']} already ended") + step, attempt = record.get("step"), record.get("attempt", 0) + if not isinstance(step, int) or step < 0 or not isinstance(attempt, int) or attempt < 0: + raise ValueError(f"{where}: step and attempt must be nonnegative integers") + if (step, attempt) <= last.get(record["group"], (-1, -1)): + raise ValueError(f"{where}: steps and attempts must increase within an episode") + retry_of = record.get("retry_of") + if (attempt == 0) != (retry_of is None) or (retry_of is not None and retry_of not in ids): + raise ValueError(f"{where}: retries name an earlier id, first attempts none") + request = record.get("request") + if not isinstance(request, dict) or "state" not in request or "questions" not in request: + raise ValueError(f"{where}: request must hold state and questions") + for image in record.get("images", []): + if not all(isinstance(image.get(k), str) for k in ("sha256", "pixels_sha256")): + raise ValueError(f"{where}: images need sha256 and pixels_sha256") + for key in ("t_send_ms", "t_recv_ms", "think_ms"): + value = record.get(key) + if value is not None and (not isinstance(value, (int, float)) or value < 0): + raise ValueError(f"{where}: {key} must be a nonnegative number or null") + ids.add(record["id"]) + last[record["group"]] = (step, attempt) + requests.append(record) + elif kind == "episode_end": + group = record.get("group") + if group not in last or group in ends: + raise ValueError(f"{where}: episode_end needs an open episode") + ends[group] = record + else: + raise ValueError(f"{where}: unknown record type {kind!r}") + if not requests: + raise ValueError("a trace needs at least one request") + return header, requests, ends + + +def read(path): + records = [json.loads(line) for line in Path(path).read_text().splitlines() if line.strip()] + return check(records) + + +def write(path, records): + check(records) + Path(path).write_text("".join(json.dumps(r, ensure_ascii=False) + "\n" for r in records)) + + +def from_ticks(lines, episode, source, model=None): + """Records from System1-Agents' saved browser ticks: one JSON object per decision with + `state`, `questions` and optionally `answers` and `model`. Ticks carry no timing.""" + records = [{ + "type": "header", "format": FORMAT, "source": source, + "server": {"model": model}, "capture": "converted", + "clock": None, "content": {}, "images": {"store": None}, + }] + for step, line in enumerate(lines): + tick = json.loads(line) + request = {"state": tick["state"], "questions": tick["questions"]} + records.append({ + "type": "request", "id": f"{episode}:s{step}:a0", "group": episode, + "step": step, "attempt": 0, "retry_of": None, "request": request, + "images": [], "t_send_ms": None, "t_recv_ms": None, "think_ms": None, + "deadline_s": None, "status": None, + "response": {"answers": tick["answers"]} if "answers" in tick else None, + "notes": {"model": tick["model"]} if "model" in tick else {}, + }) + records.append({"type": "episode_end", "group": episode, "status": None, "steps": len(lines)}) + return records + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + commands = parser.add_subparsers(dest="command", required=True) + checking = commands.add_parser("check", help="check a trace and print its counts") + checking.add_argument("trace", type=Path) + converting = commands.add_parser("from-ticks", help="convert saved System1-Agents ticks") + converting.add_argument("ticks", type=Path) + converting.add_argument("out", type=Path) + converting.add_argument("--episode", required=True) + converting.add_argument("--source", required=True, help="JSON object naming repo, commit and front") + converting.add_argument("--model", help="the decision model that answered the ticks") + args = parser.parse_args() + if args.command == "check": + header, requests, ends = read(args.trace) + groups = {r["group"] for r in requests} + print(json.dumps({"requests": len(requests), "episodes": len(groups), "ended": len(ends)})) + else: + lines = [line for line in args.ticks.read_text().splitlines() if line.strip()] + write(args.out, from_ticks(lines, args.episode, json.loads(args.source), args.model)) + print(f"wrote {len(lines)} requests to {args.out}") + + +if __name__ == "__main__": + main() diff --git a/tests/benchmarks/test_traces.py b/tests/benchmarks/test_traces.py new file mode 100644 index 00000000..f57dc9eb --- /dev/null +++ b/tests/benchmarks/test_traces.py @@ -0,0 +1,328 @@ +"""Decision traces: format checks, metrics on synthetic token ids, and a paced local replay.""" + +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +import json +from pathlib import Path +import subprocess +import sys +import tempfile +import threading +import time +import unittest + +TRACES = Path(__file__).resolve().parents[2] / "benchmarks/traces" +sys.path.insert(0, str(TRACES)) +import metrics # noqa: E402 +import replay # noqa: E402 +import tracefile # noqa: E402 + +HEADER = {"type": "header", "format": tracefile.FORMAT, "capture": "converted"} + + +def request(group, step, state, attempt=0, retry_of=None, **extra): + return {"type": "request", "id": f"{group}:s{step}:a{attempt}", "group": group, "step": step, + "attempt": attempt, "retry_of": retry_of, + "request": {"state": state, "questions": {"q": {"type": "noul", "instructions": "Done?"}}}, + **extra} + + +class FormatTests(unittest.TestCase): + def test_a_valid_trace_reads_back(self): + records = [HEADER, request("e", 0, "a"), request("e", 0, "a", 1, "e:s0:a0"), + request("e", 1, "b"), {"type": "episode_end", "group": "e", "steps": 2}] + with tempfile.TemporaryDirectory() as root: + path = Path(root) / "t.jsonl" + tracefile.write(path, records) + header, requests, ends = tracefile.read(path) + self.assertEqual([r["id"] for r in requests], ["e:s0:a0", "e:s0:a1", "e:s1:a0"]) + self.assertEqual(list(ends), ["e"]) + + def test_malformed_traces_name_the_problem(self): + cases = { + "starts with one header": [request("e", 0, "a")], + "duplicate id": [HEADER, request("e", 0, "a"), request("e", 0, "a")], + "must increase": [HEADER, request("e", 1, "a"), request("e", 0, "b")], + "retries name": [HEADER, request("e", 0, "a"), request("e", 0, "a", 1)], + "already ended": [HEADER, request("e", 0, "a"), {"type": "episode_end", "group": "e"}, + request("e", 1, "b")], + "state and questions": [HEADER, {**request("e", 0, "a"), "request": {"state": "a"}}], + "nonnegative number": [HEADER, request("e", 0, "a", think_ms=-1)], + } + for message, records in cases.items(): + with self.assertRaisesRegex(ValueError, message): + tracefile.check(records) + + def test_ticks_convert_to_one_episode_without_timing(self): + ticks = [json.dumps({"state": {"page": i}, "questions": {"q": {"type": "noul", "instructions": "x"}}, + "answers": {"q": "true"}, "model": "m"}) for i in range(3)] + records = tracefile.from_ticks(ticks, "ep", {"repo": "r"}) + _, requests, ends = tracefile.check(records) + self.assertEqual([r["step"] for r in requests], [0, 1, 2]) + self.assertTrue(all(r["think_ms"] is None for r in requests)) + self.assertEqual(ends["ep"]["steps"], 3) + + +class Layout: + """Token ids straight from the request: the state's list, then one row per candidate.""" + + def rows(self, request): + return [[request["state"] + c for c in q["candidates"]] for q in request["questions"].values()] + + def state(self, request): + return request["state"] + + +def tokens_request(group, step, state, candidates, attempt=0, retry_of=None, images=(), **extra): + record = request(group, step, state, attempt, retry_of, images=list(images), **extra) + record["request"]["questions"] = {"q": {"candidates": candidates}} + return record + + +class MetricTests(unittest.TestCase): + def test_the_planner_follows_prefix_rs(self): + state = list(range(200)) + groups = [[state + [900] * 100 + [1, 2], state + [900] * 100 + [1, 3], state + [900] * 100 + [4]], + [state + [901] * 10 + [5]]] + # the shared 200 tokens floor to 192; the first question's 100 more reach 256 + self.assertEqual(metrics.plan(groups), (200, 192, [256, 192], 3 * 192 + 2 * (256 - 192))) + self.assertEqual(metrics.plan([[state]]), (0, 0, [0], 0)) + self.assertEqual(metrics.plan([[[1], [1]]]), (0, 0, [0], 0)) + + def test_cross_request_reuse_counts_chunks_once_per_request(self): + a, b = list(range(300)), list(range(130)) + [-1] * 170 + image = {"sha256": "x", "pixels_sha256": "p"} + requests = [ + tokens_request("e", 0, a, [[1], [2]], images=[image]), + tokens_request("e", 1, b, [[1], [2]], images=[image]), + tokens_request("e", 1, b, [[1], [2]], 1, "e:s1:a0", images=[image]), + tokens_request("e", 2, a, [[1], [2]], images=[image]), + ] + rows = metrics.measure(requests, Layout(), last_k=8) + self.assertEqual([r["local_saved"] for r in rows], [256, 256, 256, 256]) + # step 1 shares 130 tokens with step 0: two chunks + self.assertEqual((rows[1]["prev_lcp64"], rows[1]["cross_saved_prev"]), (128, 128)) + self.assertTrue(rows[1]["images_seen_prev"] and not rows[1]["state_seen_before"]) + self.assertEqual(rows[1]["near_duplicate"], round(2 * 130 / 600, 4)) + # the retry is not compared; step 2 repeats step 0's state + self.assertNotIn("prev_lcp64", rows[2]) + self.assertEqual((rows[3]["prev_lcp64"], rows[3]["cross_saved_best"]), (128, 256)) + self.assertTrue(rows[3]["state_seen_before"] and not rows[3]["state_seen_prev"]) + self.assertEqual(rows[3]["near_duplicate"], 1.0) + summary = metrics.summarize(rows) + self.assertEqual((summary["requests"], summary["retries"]), (4, 1)) + self.assertEqual((summary["images"], summary["images_seen_prev"]), (3, 0.6667)) + + def test_refused_and_failed_requests_are_not_measured(self): + a = list(range(100)) + requests = [ + tokens_request("e", 0, a, [[1], [2]], status=422), + tokens_request("e", 1, a, [[1], [2]], notes={"error": "CancelledError: "}), + tokens_request("e", 2, a, [[1], [2]], status=200), + tokens_request("e", 3, a, [[1], [2]]), # converted ticks carry no status + ] + self.assertEqual([r["step"] for r in metrics.measure(requests, Layout())], [2, 3]) + self.assertEqual(metrics.summarize([])["questions_per_request"], None) + with self.assertRaises(ValueError): + metrics.measure(requests, Layout(), last_k=0) + + def test_open_jev_prompts_follow_the_worker_fixture(self): + # golden prompts from Open-Jev @ 3308a15's request compiler, as the Rust worker tests use them + cases = json.loads((TRACES.parents[1] / "tests/open_jev/data/contract.json").read_text()) + for case in cases: + self.assertEqual(metrics.open_jev_prompts(case["request"]), case["prompts"]) + self.assertIsNone(metrics.open_jev_prompts({"state": "x", "questions": {"q": {"type": "choice"}}})) + self.assertIsNone(metrics.open_jev_prompts({"state": "x", "questions": {}})) + + def test_cua_s1_prompts_follow_the_worker(self): + layout = metrics.CuaS1.__new__(metrics.CuaS1) + layout.contract = metrics.cua_s1_contract() + choice = {"type": "choice", "instructions": "go", "criteria": {"a": 'Say "hi"\n', "b": None}} + prompt = layout.prompts({"state": {"a": [1.5, "é"]}, "questions": {"q": choice}})[0] + self.assertTrue(prompt.startswith(f"<|im_start|>system\n{layout.contract.SYSTEM_PROMPT}<|im_end|>\n")) + self.assertIn("<|im_start|>user\nGoal: go\n\nApp: Cua Driver\n", prompt) + self.assertIn('Accessibility tree:\n{"a": [1.5, "é"]}\n', prompt) + self.assertTrue(prompt.endswith( + 'Options:\nA. Decision "Say \\"hi\\"\\n" -> select\nB. Decision "b" -> select\n\n' + "Answer with a single letter.<|im_end|>\n<|im_start|>assistant\n\n")) + # an empty goal leaves the line out; a request with a Noul is refused whole + empty = layout.prompts({"state": "s", "questions": {"q": dict(choice, instructions="")}})[0] + self.assertIn("<|im_start|>user\nApp: Cua Driver\n", empty) + self.assertIsNone(layout.prompts({"state": "s", "questions": { + "q": choice, "n": {"type": "noul", "instructions": "x"}}})) + + +class CacheTests(unittest.TestCase): + def test_a_cache_skips_what_earlier_prompts_cover_once_per_span(self): + a = list(range(300)) + # one row, no plan: a cached earlier prompt covers 256 of its tokens + self.assertEqual(metrics.cached([[a + [1]]], 0, [0], [a + [2]]), 256) + # two questions sharing 300 tokens: the request prefix (256) is computed once and fully cached; + # the rows' rests start at 256 and the cache covers nothing more + groups = [[a + [1]], [a + [2]]] + _, p, ends, saved = metrics.plan(groups) + self.assertEqual((p, saved), (256, 256)) + self.assertEqual(metrics.cached(groups, p, ends, [a + [3]]), 256) + self.assertEqual(metrics.cached(groups, p, ends, []), 0) + # a row keeps its last token, in whole chunks: a repeated 129-token row reuses 128, + # a repeated 128-token row 64 + self.assertEqual(metrics.cached([[a[:129]]], 0, [0], [a]), 128) + self.assertEqual(metrics.cached([[a[:128]]], 0, [0], [a[:128]]), 64) + + +class Server(BaseHTTPRequestHandler): + arrivals = [] + slow = set() + + def do_POST(self): + body = json.loads(self.rfile.read(int(self.headers["Content-Length"]))) + Server.arrivals.append((time.perf_counter(), body["state"])) + if body["state"] in Server.slow: + time.sleep(0.5) + raw = json.dumps({"answers": {}, "metadata": {"prefix_cache": {"enabled": True}}}).encode() + self.send_response(200) + self.send_header("Content-Type", "application/json") + self.send_header("Content-Length", str(len(raw))) + self.end_headers() + self.wfile.write(raw) + + def log_message(self, *args): + pass + + +class ReplayTests(unittest.TestCase): + def test_episodes_keep_order_and_think_time(self): + Server.arrivals, Server.slow = [], {"b1"} + server = ThreadingHTTPServer(("127.0.0.1", 0), Server) + threading.Thread(target=server.serve_forever, daemon=True).start() + self.addCleanup(server.server_close) + self.addCleanup(server.shutdown) + records = [HEADER] + for group in "ab": + for step in range(3): + records.append(request(group, step, f"{group}{step}", think_ms=80, deadline_s=0.25)) + with tempfile.TemporaryDirectory() as root: + trace, out = Path(root) / "t.jsonl", Path(root) / "out" + tracefile.write(trace, records) + subprocess.run([sys.executable, str(TRACES / "replay.py"), str(trace), "--url", + f"http://127.0.0.1:{server.server_port}", "--out", str(out)], + check=True, capture_output=True) + summary = json.loads((out / "summary.json").read_text()) + responses = [json.loads(line) for line in (out / "responses.jsonl").read_text().splitlines()] + self.assertEqual((summary["requests"], summary["ok"], summary["shared_prefix_requests"]), (6, 6, 6)) + self.assertEqual(summary["deadline_misses"], 1) + self.assertEqual([r["id"] for r in responses if r["missed"]], ["b:s1:a0"]) + for group in "ab": + times = [t for t, state in Server.arrivals if state.startswith(group)] + states = [state for _, state in Server.arrivals if state.startswith(group)] + self.assertEqual(states, [f"{group}{i}" for i in range(3)]) + self.assertTrue(all(later - earlier >= 0.075 for earlier, later in zip(times, times[1:]))) + + def test_episodes_start_at_their_recorded_offsets_and_retries_can_be_dropped(self): + records = [request("a", 0, "a0", t_send_ms=1000.0), request("a", 0, "a0", 1, "a:s0:a0", t_send_ms=1200.0), + request("b", 0, "b0", t_send_ms=2500.0)] + groups = replay.episodes(records, skip_retries=True) + self.assertEqual({g: [r["id"] for r in rs] for g, rs in groups.items()}, {"a": ["a:s0:a0"], "b": ["b:s0:a0"]}) + self.assertEqual(replay.offsets(groups, together=False), {"a": 0.0, "b": 1.5}) + self.assertEqual(replay.offsets(groups, together=True), {"a": 0.0, "b": 0.0}) + self.assertEqual(len(replay.episodes(records, skip_retries=False)["a"]), 2) + + def test_sequential_episodes_run_one_after_another(self): + Server.arrivals, Server.slow = [], set() + server = ThreadingHTTPServer(("127.0.0.1", 0), Server) + threading.Thread(target=server.serve_forever, daemon=True).start() + self.addCleanup(server.server_close) + self.addCleanup(server.shutdown) + records = [HEADER] + [request(g, s, f"{g}{s}", think_ms=0) for g in "ab" for s in range(2)] + with tempfile.TemporaryDirectory() as root: + trace, out = Path(root) / "t.jsonl", Path(root) / "out" + tracefile.write(trace, records) + subprocess.run([sys.executable, str(TRACES / "replay.py"), str(trace), "--url", + f"http://127.0.0.1:{server.server_port}", "--sequential", "--deadline-s", "60", + "--out", str(out)], check=True, capture_output=True) + summary = json.loads((out / "summary.json").read_text()) + self.assertEqual([state for _, state in Server.arrivals], ["a0", "a1", "b0", "b1"]) + self.assertEqual((summary["answered_with_deadline"], summary["deadline_misses"]), (4, 0)) + + +FAKE_S1A = { + "s1a/__init__.py": "", + "s1a/console.py": "", + "s1a/decision_models/__init__.py": "", + "s1a/decision_models/wire.py": """ +class JevDecisionsClient: + _timeout_s = 5.0 + + async def decide(self, body): + if body["state"] == "fail": + raise RuntimeError("decisions endpoint returned HTTP 422: too long") + return {"answers": {"q": {"type": "noul", "noul": 0.5}}}, 3 +""", + "s1a/tool/__init__.py": "", + "s1a/tool/loop.py": """ +import types +from s1a.decision_models.wire import JevDecisionsClient + + +async def run_episode(spec, env, *, seed, model_name="jev"): + client = JevDecisionsClient() + for step in range(env): + await client.decide({"state": f"{seed}-{step}", "questions": {"q": {"type": "noul"}}}) + return types.SimpleNamespace(extra={"result_type": "timeout"} if seed else {}, error=None) +""", + "s1a/entry.py": """ +import asyncio, sys, types + + +def s1a(): + from s1a.tool.loop import run_episode # bound after the recorder's patch, as s1a.tool.series does + from s1a.decision_models.wire import JevDecisionsClient + spec = types.SimpleNamespace(name=sys.argv[2]) + + async def main(): + for seed in range(int(sys.argv[4])): + await run_episode(spec, 2, seed=seed) + try: + await JevDecisionsClient().decide({"state": "fail", "questions": {}}) + except RuntimeError: + pass + + asyncio.run(main()) + sys.exit(0) +""", +} + + +class RecorderTests(unittest.TestCase): + def test_the_recorder_wraps_decisions_and_episodes_from_outside(self): + with tempfile.TemporaryDirectory() as root: + for name, text in FAKE_S1A.items(): + path = Path(root) / name + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(text) + out = Path(root) / "trace.jsonl" + subprocess.run([sys.executable, str(TRACES / "record_s1a.py"), str(out), "--commit", "abc", + "--group", "outside", "--", "run", "fake", "--episodes", "2"], + check=True, capture_output=True, env={"PYTHONPATH": root, "PATH": ""}) + header, requests, ends = tracefile.read(out) + self.assertEqual(header["source"]["front"], "run fake") + self.assertEqual([r["id"] for r in requests], + ["fake-seed0-0:s0:a0", "fake-seed0-0:s1:a0", "fake-seed1-1:s0:a0", "fake-seed1-1:s1:a0", + "outside:s0:a0"]) + self.assertEqual([r["request"]["state"] for r in requests[:4]], ["0-0", "0-1", "1-0", "1-1"]) + self.assertIsNone(requests[0]["think_ms"]) + self.assertGreaterEqual(requests[1]["think_ms"], 0) + self.assertEqual((requests[1]["status"], requests[1]["deadline_s"]), (200, 5.0)) + self.assertEqual((requests[4]["status"], requests[4]["notes"]["error"]), + (422, "RuntimeError: decisions endpoint returned HTTP 422: too long")) + self.assertEqual({g: e["status"] for g, e in ends.items()}, + {"fake-seed0-0": "done", "fake-seed1-1": "timeout", "outside": None}) + + def test_the_samples_are_valid_traces(self): + for path in (TRACES / "samples").glob("*.trace.jsonl"): + header, requests, _ = tracefile.read(path) + self.assertTrue(header["content"]["license"], path) + self.assertTrue(all(r["status"] == 200 for r in requests), path) + + +if __name__ == "__main__": + unittest.main()