diff --git a/CHANGELOG.md b/CHANGELOG.md index d642492..0b279a8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,6 +14,10 @@ The format follows [Keep a Changelog](https://keepachangelog.com/en/1.1.0/); ver - Public231 decision-model benchmark with raw HTTP evidence, pinned upstream scoring, complete-denominator quality metrics and per-round service latency. +- Sokoban `--visual`: the same levels, loop and scoring with the board rendered by Valen's own renderer — one + PNG per decision, the theme and tile size each level carries, and Valen's instruction and direction texts in + the level's language (`en`/`zh`). The ASCII board never rides with the picture and stays out of the episode + records; the `visual` extra (Pillow) is needed, text-board runs stay dependency-free. - Optional local Cua-S1 4B inference through the `cua-four-b` extra and `CUA_S1_VARIANT=4b`, supporting text and multimodal input. Desktop `--pixel-target` offers named screenshot points; clicks remain bound to the observed window and capture. diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index e65bca3..364abd0 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -250,13 +250,14 @@ Everything outside `openjiuwen` is an extra. An agent whose extra is missing say | `alfworld` | alfworld, textworld | `s1a run alfworld`; also `ALFWORLD_DATA` and Python 3.11, see `evals/README.md` | | `alfworld-visual` | the `alfworld` extra, ai2thor 2.1.0, torch | `evals/replay/thor_replay.py`, the AI2-THOR scene behind an ALFWorld trial in the replay page; the 400 MB Unity build downloads on first use | | `report` | pillow, playwright | `python -m evals.replay`, the showcase pages and GIFs; `--gif` also needs `uv run playwright install chromium` | +| `visual` | pillow | `s1a run sokoban --visual`: the board renderer; text-board Sokoban stays dependency-free | | `laya` | laya (torch, transformers) | `--model laya` on every agent and on `decide` and `probe`: Laya in process, no Jev key; the checkpoint downloads into the Hugging Face cache (`HF_HOME`) on first use | | `cua` | cua-s1 (torch), huggingface-hub | `--model cua` on tool and browser agents and on `decide` and `probe`: Cua-S1 Nano in process; the 3 MB checkpoint downloads into the Hugging Face cache (`HF_HOME`) on first use | | `omnijev` | torch, torchvision, transformers, peft, accelerate, safetensors | `--model omnijev` on the browser agents: OmniJev in process, no Jev key; the model code is a clone of the OmniJev repository named by `OMNIJEV_REPO`, the checkpoint and base model by `OMNIJEV_CHECKPOINT` and `OMNIJEV_BASE` (`docs/configuration.md`) | | `dev` | pytest, pytest-asyncio, ruff, ty, jsonschema, pyyaml, referencing | the test suite, `scripts/smoke.sh` and the lint and type checks; the last three check the served-Laya fixtures against `docs/api/` | | `cua-four-b` | the `cua` extra, transformers 5, peft, torchvision, pillow | optional local Cua-S1 4B; set `CUA_S1_VARIANT=4b`, and `CUA_S1_MODALITY=text` or `multimodal` | -`uv sync --all-extras` installs all eight. The CLI runs from a checkout; a wheel install (`uv tool install`, +`uv sync --all-extras` installs all nine. The CLI runs from a checkout; a wheel install (`uv tool install`, `pip install`) is unsupported, because the data folders (`evals/2048`, `evals/millionaire`, `evals/labelled`) sit next to the package; the command refuses to start outside a checkout with one line on stderr. Runs, logs and results go under the checkout (`runs/`, `evals/results`), or under `S1A_HOME` when that variable names another diff --git a/NOTICE b/NOTICE index 22256e2..93c0fa9 100644 --- a/NOTICE +++ b/NOTICE @@ -12,3 +12,8 @@ This product includes software developed by third parties: evals/2048/style/fonts are Clear Sans by Intel Corporation (Apache-2.0) - evals/millionaire/questions.json, fetched at first run: Open Trivia Database (https://opentdb.com), CC BY-SA 4.0 +- s1a/agents/_sokoban.py: adapted Valen Sokoban simulator (Liuziyu77/Valen), Apache-2.0. + s1a/agents/_sokoban_render.py and the visual-mode instruction texts: adapted Valen Sokoban + renderer and dataset texts (Liuziyu77/Valen), Apache-2.0. + Sokoban levels and test references: Valen-Team/Valen-Eval-Game, Apache-2.0. + Pinned sources and modifications: THIRD_PARTY_LICENSES.md diff --git a/README.md b/README.md index ce1490d..d3bfbac 100644 --- a/README.md +++ b/README.md @@ -148,7 +148,9 @@ The gates and the templates: [docs/skills.md](docs/skills.md#build-a-system-1-ag - `desktop`: computer use, any Windows or macOS app window through [Cua Driver](https://cua.ai/docs/cua-driver). - `ticket_router`: 30 labelled support tickets to five queues. - `alfworld`: household tasks in text, with the AI2-THOR scene in the replays. -- `game2048`, `millionaire`, `blackjack`: games with a score per episode. +- `game2048`, `millionaire`, `blackjack`, `sokoban`: games with a score per episode. + Sokoban plays Valen’s selected evaluation levels as text boards or, with `--visual`, as Valen’s rendered + boards; see [the protocol](evals/sokoban/README.md). - `injection_guard`: a rail that answers one question at a hook of a running agent and fails closed. These use cases are [application candidates](CONTRIBUTING.md#application-candidates) for the required diff --git a/THIRD_PARTY_LICENSES.md b/THIRD_PARTY_LICENSES.md index d294043..524eccb 100644 --- a/THIRD_PARTY_LICENSES.md +++ b/THIRD_PARTY_LICENSES.md @@ -39,3 +39,22 @@ Fonts under `evals/2048/style/fonts` are Clear Sans by Intel Corporation, Apache `evals/millionaire/questions.json`, fetched on the first Millionaire run, holds questions from the [Open Trivia Database](https://opentdb.com), Creative Commons Attribution-ShareAlike 4.0 International. + +## Valen Sokoban + +`s1a/agents/_sokoban.py` is adapted from +[Liuziyu77/Valen](https://github.com/Liuziyu77/Valen/blob/c96c4f736c6928d2cb1166bc105a24cebdfcdc40/evaluation/sokoban/env.py), +commit `c96c4f736c6928d2cb1166bc105a24cebdfcdc40`, Apache-2.0. +Changes: removed unused hash, inverse-action and clone helpers; applied repository formatting. + +`s1a/agents/_sokoban_render.py`, and the visual-mode rules and direction names in +`s1a/agents/sokoban.py`, are adapted from the same commit's +[render.py](https://github.com/Liuziyu77/Valen/blob/c96c4f736c6928d2cb1166bc105a24cebdfcdc40/evaluation/sokoban/render.py) +and [dataset.py](https://github.com/Liuziyu77/Valen/blob/c96c4f736c6928d2cb1166bc105a24cebdfcdc40/evaluation/sokoban/dataset.py), +Apache-2.0. The renderer's drawing logic is unchanged, formatted; the rules and direction texts are verbatim. + +`s1a/agents/_data/sokoban_levels.jsonl` and `tests/fixtures/sokoban_reference.jsonl` are unchanged copies of +`eval_sokoban/levels.jsonl` and `eval_sokoban/reference.jsonl` from +[Valen-Team/Valen-Eval-Game](https://huggingface.co/datasets/Valen-Team/Valen-Eval-Game/tree/4f25038cd30d9a96aa377eea176a4d2e1680a35c), +revision `4f25038cd30d9a96aa377eea176a4d2e1680a35c`, declared Apache-2.0. +The full Apache-2.0 license text is this repository's `LICENSE`. diff --git a/docs/agents.md b/docs/agents.md index 6c28fd5..40c6dee 100644 --- a/docs/agents.md +++ b/docs/agents.md @@ -14,6 +14,7 @@ hook of a running agent. The injection guard rail fails closed: a decision error | `game2048` | tool | score and largest tile at a move cap | `s1a run game2048 --model jev --rethink on --episodes 10` | | `millionaire` | tool | winnings on a 15-question quiz ladder | `s1a run millionaire --model jev --rethink off --episodes 5` | | `blackjack` | tool | payoff per hand (RLCard) | `s1a run blackjack --model jev --rethink off --episodes 100` | +| `sokoban` | tool | solved fraction on 100 selected Valen levels: text boards over `jev`; `--visual` renders for an image-reading model | `s1a run sokoban --model jev --rethink off --episodes 100` | | `injection_guard` | rail | precision and recall on a labelled injection set | `s1a run injection_guard` | Every tool agent takes `--model jev|clm|laya|laya-served|cua|llm|random|rule`, `--rethink on|off`, `--episodes N`, `--seed S`, @@ -87,3 +88,19 @@ shipped set is 30 tickets in `s1a/agents/_data/ticket_router_eval.jsonl`, each w order status; the label is read by the scorer alone. `--dataset` points the agent at a JSONL of your own with the same fields, and `--batch-size` caps the tickets per episode. The rules text the model reads is `RULES` in `s1a/agents/ticket_router.py`. + +## Sokoban + +`uv run s1a run sokoban --model jev --rethink off --episodes 100` plays the bundled 100 Valen levels as text +boards. No extra is needed. `--seed` is the zero-based level offset; the selected range must fit within 100 +levels. The score is 1 for solved and 0 otherwise. Use `--model random` for an offline smoke run; there is no +rule baseline. Rethink must be off. See [protocol and provenance](../evals/sokoban/README.md). + +`--visual` plays the same levels over Valen's own rendering — one PNG per decision, theme and tile size from +the level, rules and direction labels in the level's language — with the ASCII board left out. Text-only +backends (`jev`, `clm`, `laya`, `laya-served`, text-modality `cua`, `llm`) are refused before the first +decision instead of running blind. `cua`'s multimodal 4B takes the picture, but admission is capability, +not a result: no visual Sokoban run is recorded for it, so the `jev` command above stays the reference +text-mode run. (`omnijev` is image-capable too, but it is a browser-front choice; `s1a run` cannot +select it.) `--model random --visual` smokes the rendering offline (`uv run --extra visual`); +the served Valen backend this observation was made for ships as its own change. diff --git a/docs/decision-models.md b/docs/decision-models.md index 2c2af5d..6eb79f9 100644 --- a/docs/decision-models.md +++ b/docs/decision-models.md @@ -39,12 +39,13 @@ decision.choice("pick").key, decision.latency_ms, decision.usage.input_tokens ``` `decide_many` is the one place validation happens. It refuses, with `MODEL_SERVICE_CONFIG_ERROR`, an empty -question set, an image on a text-only model and a question type the backend lacks. It checks every answer: -a choice must name an offered key, its probabilities must cover exactly the offered keys, lie in [0, 1], sum to 1 -within 0.02 and peak at the key; a noul probability lies in [0, 1] and its confidence defaults to `max(p, 1 - p)`. -An unusable answer is re-asked with the same request up to `attempts` times (never on a deterministic backend, never -after a transport error) and then raises `MODEL_CALL_FAILED`. `decide`, `choose` and `ask` are one-question -shorthands; `warm()` and `close()` open and release the backend. +question set and a question type the backend lacks; a text-only backend instead reads the observation without its +images and warns once per instance. It checks every answer: a choice must name an offered key, its probabilities +must cover exactly the offered keys, lie in [0, 1], sum to 1 within 0.02 and peak at the key; a noul probability +lies in [0, 1] and its confidence defaults to `max(p, 1 - p)`. An unusable answer is re-asked with the same request +up to `attempts` times (never on a deterministic backend, never after a transport error) and then raises +`MODEL_CALL_FAILED`. `decide`, `choose` and `ask` are one-question shorthands; `warm()` and `close()` open +and release the backend. ## Backends diff --git a/evals/README.md b/evals/README.md index 69adce7..7b84e16 100644 --- a/evals/README.md +++ b/evals/README.md @@ -6,6 +6,7 @@ folder per episode). `uv run python -m evals.table evals/results` aggregates the | eval | measures | run | |---|---|---| +| Sokoban (selected Valen levels) | solved fraction; the command below is the text board — `--visual` is the upstream image observation and takes an image-capable backend (multimodal `cua`; the image-capable `omnijev` is browser-front only, not selectable by `s1a run`); no visual run recorded | `s1a run sokoban --model jev --rethink off --episodes 100` | | Blackjack (RLCard) | payoff per hand | `s1a run blackjack --model jev --rethink off --episodes 100` | | 2048 (the MIT game, self-hosted) | score and largest tile at a move cap | `s1a run game2048 --model jev --rethink on --episodes 10` | | Millionaire (self-hosted quiz, Open Trivia DB) | winnings; the 50:50 lifeline is a candidate the model may pick; six ladders: `--seed` plus `--episodes` stays at or below 6 | `s1a run millionaire --model jev --rethink off --episodes 5` | diff --git a/evals/sokoban/README.md b/evals/sokoban/README.md new file mode 100644 index 0000000..bdecb4a --- /dev/null +++ b/evals/sokoban/README.md @@ -0,0 +1,66 @@ +# Sokoban (text boards and, with --visual, Valen's rendered boards) + +```bash +uv run s1a run sokoban --model random --rethink off --episodes 1 --max-steps 10 +uv run s1a run sokoban --model jev --rethink off --episodes 100 +``` + +No additional dependency, browser, dataset download or GPU is needed for the environment. +The model receives an ASCII board and remaining step budget, with the legend in its rules. +The four directions remain available even when blocked; blocked moves consume a step. +There is no solver, legal-action mask, undo, rule baseline or rethink assistance. + +`--seed` selects the zero-based level offset, with consecutive levels per episode; the +range must stay within the bundled 100 levels. The episode ends on success or after at +most 200 actions (`--max-steps` can lower this); `--timeout` also caps wall time. +Score is 1 for a solved board and 0 otherwise, so `mean_score` is the solved fraction +among non-error episodes. Always report errors separately. Episode metadata records the +level ID, text observation mode, pushes and whether the environment step limit was reached. +The standard episode records include the final board, steps, elapsed time and decisions. + +These are Valen's **selected** 100 evaluation levels (35 easy, 65 medium), not an unbiased +sample. They retained successful 2B RLCD cases before filling the set. Text-mode results use +symbolic boards and must not be compared as equivalent to Valen's image-only evaluation. +No learned-model performance has been measured by this import. + +## Visual mode + +```bash +uv run --extra visual s1a run sokoban --model random --rethink off --episodes 1 --max-steps 10 --visual +``` + +The `visual` extra (Pillow) rides on the command; `uv sync --extra visual` alone would prune +the other extras, so add it to your usual sync set instead (`uv sync --extra dev --extra report --extra visual`). + +`--visual` keeps the levels, episode loop, budgets and scoring untouched and swaps the +observation: each decision is shown Valen's own rendering of the board, one PNG per move, +drawn with the `theme` and `tile_size` the level carries, alongside the remaining step +budget. The ASCII board never accompanies the picture — the preview checkpoint read images — +and the pictures stay out of the episode records (`observation_mode: visual` marks the mode). +The rules and the four direction labels follow each level's `language` (`en`/`zh`) and are +Valen's own instruction texts, verbatim. + +A text-only backend — `jev`, `clm`, `laya`, `laya-served`, `cua` in its text modalities, and `llm` (the chat +model reads tool text only) — is refused before the first decision rather than run blind over a board it +cannot see; **the jev command at the top of this page is the text-mode run**. The guard admits what can +actually see the board — `cua`'s multimodal 4B — but admission is capability, not a Sokoban +result: no visual run is recorded. (`omnijev` is image-capable too, but it is a browser-front choice; +`s1a run` cannot select it.) `--model random` is the other exception, the offline smoke for +the rendering; the served Valen worker behind `/v1/systemone` is the backend this observation was made for +and ships as its own change. Its +protocol takes one PNG/JPEG data URL as `state.image`, capped at an 8 MiB body, a 4 MiB image +and 2048 pixels per side — a bundled level at tile 44 renders far below all of these. + +This is the observation Valen's preview checkpoint was trained and validated on; the text +mode above remains the off-distribution control. Measured results belong to the runs recorded +under `evals/results/`, not to this import. + +The simulator and the renderer both come from Valen, not vllm-jev's demo assets. The level +definitions are bundled without changes; evaluator-only reference solutions live under +`tests/fixtures` and are never loaded by the agent. Tests replay reference solutions over +both observation modes to validate transitions, level compatibility and rendering; this is +not model accuracy. + +See [third-party provenance](../../THIRD_PARTY_LICENSES.md#valen-sokoban) for pinned +code/data revisions and [the upstream dataset card](https://huggingface.co/datasets/Valen-Team/Valen-Eval-Game) +for the original protocol and selection details. diff --git a/pyproject.toml b/pyproject.toml index 7f9ed70..732aec5 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -41,6 +41,7 @@ alfworld-visual = [ "torchvision>=0.15", ] report = ["pillow>=10", "playwright>=1.45"] # evals.replay: pages, GIFs; never imported by a runner +visual = ["pillow>=10"] # s1a run sokoban --visual: the board renderer; text mode stays dependency-free laya = ["laya>=0.3.9"] # the in-process decision model behind --model laya; pulls torch and transformers cua = [ # Cua-S1 Nano behind --model cua; pinned to the Cua PR that ships the checkpoints (trycua/cua#4023), pulls torch "cua-s1 @ git+https://github.com/trycua/cua.git@aea61b6eb97e2d8c0f6f71eb804e5769fe910af4#subdirectory=libs/cua-s1/python", diff --git a/s1a/agents/_data/sokoban_levels.jsonl b/s1a/agents/_data/sokoban_levels.jsonl new file mode 100644 index 0000000..bb92e92 --- /dev/null +++ b/s1a/agents/_data/sokoban_levels.jsonl @@ -0,0 +1,100 @@ +{"level_id": "sokoban-test-000", "group_id": "sokoban:f0b2345ae2c14ca6ee9c04b3c9b1f7c83bf58c4d60798143969d8e0d2c947fc4", "split": "test", "board": {"height": 8, "width": 6, "floors": [7, 8, 9, 13, 14, 15, 19, 20, 21, 22, 25, 26, 27, 28, 31, 32, 33, 34, 37, 38, 39, 40], "goals": [19, 20, 32]}, "initial_state": {"player": 22, "boxes": [21, 31, 33]}, "generator_seed": 20260922, "rules_version": "sokoban-v1", "render": {"theme": "warm", "tile_size": 44}, "language": "zh", "max_steps": 200} +{"level_id": "sokoban-test-001", "group_id": "sokoban:8d1af5ee21efeef245d8e492d864d43b29226ea6e181309d25b369667a2c84ef", "split": "test", "board": {"height": 8, "width": 7, "floors": [8, 9, 10, 11, 12, 15, 17, 18, 19, 24, 25, 26, 29, 31, 32, 33, 36, 37, 38, 39, 40, 46, 47], "goals": [26, 37]}, "initial_state": {"player": 9, "boxes": [18, 37]}, "generator_seed": 20260923, "rules_version": "sokoban-v1", "render": {"theme": "classic", "tile_size": 44}, "language": "en", "max_steps": 200} +{"level_id": "sokoban-test-002", "group_id": "sokoban:96210f5b172297f122e38f2f5d553ec2c3fc71405f0b2667483dab043df8ba9b", "split": "test", "board": {"height": 6, "width": 7, "floors": [8, 9, 10, 11, 12, 15, 16, 17, 18, 19, 22, 23, 24, 25, 26, 29, 30, 31, 32, 33], "goals": [25, 26]}, "initial_state": {"player": 9, "boxes": [16, 23]}, "generator_seed": 20260924, "rules_version": "sokoban-v1", "render": {"theme": "classic", "tile_size": 44}, "language": "zh", "max_steps": 200} +{"level_id": "sokoban-test-005", "group_id": "sokoban:b2f589f57c6243bb4fb0a28ef83006a3fc41a0e783e5ed5d8679084a1cb20ec0", "split": "test", "board": {"height": 8, "width": 6, "floors": [8, 9, 10, 13, 15, 16, 19, 20, 22, 25, 26, 27, 28, 31, 32, 33, 34, 38, 40], "goals": [28]}, "initial_state": {"player": 8, "boxes": [26]}, "generator_seed": 20260928, "rules_version": "sokoban-v1", "render": {"theme": "classic", "tile_size": 44}, "language": "en", "max_steps": 200} +{"level_id": "sokoban-test-006", "group_id": "sokoban:2b987d29762978e0ab75cfff9eaeceffe3b03b1fc9160d14470b8d2478ed8017", "split": "test", "board": {"height": 9, "width": 8, "floors": [19, 20, 27, 28, 34, 35, 36, 37, 42, 43, 44, 45, 49, 50, 51, 52, 53, 57, 58, 59, 60, 61], "goals": [35, 60]}, "initial_state": {"player": 27, "boxes": [28, 58]}, "generator_seed": 20260929, "rules_version": "sokoban-v1", "render": {"theme": "classic", "tile_size": 44}, "language": "zh", "max_steps": 200} +{"level_id": "sokoban-test-010", "group_id": "sokoban:a343d11ed129af0b4e117066fe2c1ffd48556a3250d4b393b39bc4eba6e5fabd", "split": "test", "board": {"height": 8, "width": 8, "floors": [17, 20, 25, 26, 28, 33, 34, 35, 36, 38, 41, 42, 43, 44, 45, 46, 49, 50, 51, 52, 53, 54], "goals": [41, 43]}, "initial_state": {"player": 46, "boxes": [34, 44]}, "generator_seed": 20260949, "rules_version": "sokoban-v1", "render": {"theme": "warm", "tile_size": 44}, "language": "zh", "max_steps": 200} +{"level_id": "sokoban-test-011", "group_id": "sokoban:6740c44bfb0650af9a3388a8c9c41b11b6f6b165928ad0d140161649b6dd0df8", "split": "test", "board": {"height": 9, "width": 6, "floors": [7, 8, 9, 10, 13, 14, 15, 16, 19, 20, 21, 22, 25, 26, 27, 28, 31, 32, 33, 34, 37, 38, 39, 40, 43, 44, 45, 46], "goals": [14, 19, 20]}, "initial_state": {"player": 43, "boxes": [20, 21, 37]}, "generator_seed": 20260950, "rules_version": "sokoban-v1", "render": {"theme": "classic", "tile_size": 44}, "language": "en", "max_steps": 200} +{"level_id": "sokoban-test-012", "group_id": "sokoban:c610761b33e613424d415d50493a6ef41c05d8d999eb6f096e8eeef6a5251bcf", "split": "test", "board": {"height": 7, "width": 6, "floors": [7, 8, 9, 10, 13, 14, 15, 16, 19, 20, 21, 22, 25, 26, 27, 28, 31, 32, 33, 34], "goals": [8, 28]}, "initial_state": {"player": 25, "boxes": [9, 15]}, "generator_seed": 20260951, "rules_version": "sokoban-v1", "render": {"theme": "warm", "tile_size": 44}, "language": "zh", "max_steps": 200} +{"level_id": "sokoban-test-014", "group_id": "sokoban:0adb030a2731f7fbcfc42255506385cd323baf862405e48b56cb7f87e6d00a46", "split": "test", "board": {"height": 9, "width": 9, "floors": [10, 14, 19, 20, 21, 22, 23, 28, 29, 30, 31, 32, 33, 37, 38, 39, 40, 41, 42, 43, 46, 47, 48, 49, 55, 56, 57, 64], "goals": [21, 56]}, "initial_state": {"player": 38, "boxes": [41, 56]}, "generator_seed": 20260953, "rules_version": "sokoban-v1", "render": {"theme": "warm", "tile_size": 44}, "language": "zh", "max_steps": 200} +{"level_id": "sokoban-test-016", "group_id": "sokoban:da92fe09ceaeb89a8340480c1cedad98eca53567a48e4842e9d320cf996aaeb2", "split": "test", "board": {"height": 7, "width": 8, "floors": [9, 10, 11, 12, 13, 14, 17, 18, 19, 20, 21, 22, 25, 26, 27, 28, 29, 30, 33, 34, 35, 36, 37, 38, 41, 42, 43, 44, 45, 46], "goals": [13, 36, 38]}, "initial_state": {"player": 43, "boxes": [19, 27, 36]}, "generator_seed": 20260955, "rules_version": "sokoban-v1", "render": {"theme": "classic", "tile_size": 44}, "language": "zh", "max_steps": 200} +{"level_id": "sokoban-test-020", "group_id": "sokoban:e5ecd8a4232ad6d214753678f5c43736366c2793ee8f115bb1641502be71a6cc", "split": "test", "board": {"height": 9, "width": 6, "floors": [7, 8, 9, 10, 13, 14, 15, 16, 19, 20, 21, 22, 26, 27, 28, 31, 32, 33, 34, 37, 38, 39, 40, 43, 44, 45, 46], "goals": [33, 40]}, "initial_state": {"player": 37, "boxes": [22, 32]}, "generator_seed": 20260976, "rules_version": "sokoban-v1", "render": {"theme": "classic", "tile_size": 44}, "language": "zh", "max_steps": 200} +{"level_id": "sokoban-test-021", "group_id": "sokoban:f412d99b8fcb71f53de31d9cd817bd173888c2886a949cf3af51975c28a1e144", "split": "test", "board": {"height": 6, "width": 8, "floors": [9, 10, 11, 12, 13, 14, 17, 18, 19, 20, 21, 22, 25, 26, 27, 28, 29, 30, 33, 34, 35, 36, 37, 38], "goals": [22, 25]}, "initial_state": {"player": 28, "boxes": [19, 25]}, "generator_seed": 20260977, "rules_version": "sokoban-v1", "render": {"theme": "classic", "tile_size": 44}, "language": "en", "max_steps": 200} +{"level_id": "sokoban-test-023", "group_id": "sokoban:2c5c1b462300a27346ca13293c5e80d7226944184ad3374181ce09e27bfd52b4", "split": "test", "board": {"height": 6, "width": 9, "floors": [10, 11, 12, 13, 15, 19, 20, 21, 22, 23, 24, 25, 28, 29, 30, 31, 32, 37, 38, 39, 40, 41, 42, 43], "goals": [23, 29]}, "initial_state": {"player": 19, "boxes": [29, 30]}, "generator_seed": 20260979, "rules_version": "sokoban-v1", "render": {"theme": "classic", "tile_size": 44}, "language": "en", "max_steps": 200} +{"level_id": "sokoban-test-024", "group_id": "sokoban:a38c5b369c7e80f1aa1a740a3c659799d02c33c55d7718a8f45d669e84c04968", "split": "test", "board": {"height": 7, "width": 8, "floors": [13, 18, 20, 21, 22, 26, 27, 28, 29, 30, 33, 34, 35, 36, 37, 41, 42, 43, 44, 45, 46], "goals": [27, 34]}, "initial_state": {"player": 13, "boxes": [29, 37]}, "generator_seed": 20260980, "rules_version": "sokoban-v1", "render": {"theme": "classic", "tile_size": 44}, "language": "zh", "max_steps": 200} +{"level_id": "sokoban-test-025", "group_id": "sokoban:8015bbddb33c870e384addf53bfad215a9457f433d189d2d791b34a9bd2398ce", "split": "test", "board": {"height": 9, "width": 7, "floors": [8, 9, 10, 11, 12, 15, 17, 18, 19, 22, 23, 24, 25, 26, 29, 30, 32, 33, 37, 40, 45, 46, 47, 52, 53, 54], "goals": [18, 19]}, "initial_state": {"player": 17, "boxes": [18, 23]}, "generator_seed": 20260981, "rules_version": "sokoban-v1", "render": {"theme": "warm", "tile_size": 44}, "language": "en", "max_steps": 200} +{"level_id": "sokoban-test-026", "group_id": "sokoban:7e3fc4da56e80223563ffdf76b9d4fd9b5a2a5c76daf8084baeefc41a75c7350", "split": "test", "board": {"height": 9, "width": 9, "floors": [33, 40, 41, 42, 43, 48, 49, 50, 51, 52, 55, 56, 57, 58, 59, 60, 61, 66, 69], "goals": [56]}, "initial_state": {"player": 66, "boxes": [50]}, "generator_seed": 20260982, "rules_version": "sokoban-v1", "render": {"theme": "warm", "tile_size": 44}, "language": "zh", "max_steps": 200} +{"level_id": "sokoban-test-030", "group_id": "sokoban:a60bf2b0e98260f5b8e60f5e88e04e941c6be5568deb437aa7dfb30c10de1f4b", "split": "test", "board": {"height": 7, "width": 9, "floors": [11, 12, 13, 14, 15, 16, 20, 21, 22, 23, 24, 25, 30, 31, 32, 33, 34, 37, 38, 39, 40, 41, 42, 43, 48, 49, 50, 52], "goals": [13, 40]}, "initial_state": {"player": 21, "boxes": [14, 31]}, "generator_seed": 20260997, "rules_version": "sokoban-v1", "render": {"theme": "classic", "tile_size": 44}, "language": "zh", "max_steps": 200} +{"level_id": "sokoban-test-031", "group_id": "sokoban:d8c5b79c85a3fc8422613421521e0ce37ffa0ea44a55590bff34ba7e1b010c13", "split": "test", "board": {"height": 6, "width": 7, "floors": [8, 9, 10, 11, 12, 15, 16, 17, 18, 19, 22, 23, 24, 25, 26, 29, 30, 31, 32, 33], "goals": [11, 25, 30]}, "initial_state": {"player": 15, "boxes": [10, 25, 31]}, "generator_seed": 20260998, "rules_version": "sokoban-v1", "render": {"theme": "classic", "tile_size": 44}, "language": "en", "max_steps": 200} +{"level_id": "sokoban-test-033", "group_id": "sokoban:b0f1fb463f9c41b129aa6efa8ef3a67eb4cfa631aca7caa0b9806df5f746cf90", "split": "test", "board": {"height": 7, "width": 7, "floors": [8, 9, 10, 11, 12, 15, 16, 17, 18, 19, 22, 23, 24, 25, 26, 29, 30, 31, 32, 33, 36, 37, 38, 39, 40], "goals": [16, 24, 32]}, "initial_state": {"player": 40, "boxes": [16, 17, 31]}, "generator_seed": 20261000, "rules_version": "sokoban-v1", "render": {"theme": "classic", "tile_size": 44}, "language": "en", "max_steps": 200} +{"level_id": "sokoban-test-034", "group_id": "sokoban:46709ecae29a29f0c56e886e73d45fb0c717e0ab21abd8b6d73122ffc6d30711", "split": "test", "board": {"height": 7, "width": 8, "floors": [9, 10, 11, 12, 13, 14, 17, 18, 19, 20, 21, 22, 25, 26, 27, 28, 29, 30, 33, 34, 35, 36, 37, 41, 42, 43, 44, 45], "goals": [21, 33]}, "initial_state": {"player": 43, "boxes": [18, 19]}, "generator_seed": 20261001, "rules_version": "sokoban-v1", "render": {"theme": "classic", "tile_size": 44}, "language": "zh", "max_steps": 200} +{"level_id": "sokoban-test-036", "group_id": "sokoban:b69b5b8995ebe87fe38ccb891a792860b63e228f2b2c4d5f4c2949ce2aef6ff4", "split": "test", "board": {"height": 7, "width": 6, "floors": [7, 8, 9, 10, 13, 14, 15, 16, 19, 20, 21, 22, 25, 26, 27, 28, 31, 32, 33, 34], "goals": [15, 19, 32]}, "initial_state": {"player": 31, "boxes": [14, 15, 19]}, "generator_seed": 20261004, "rules_version": "sokoban-v1", "render": {"theme": "warm", "tile_size": 44}, "language": "zh", "max_steps": 200} +{"level_id": "sokoban-test-041", "group_id": "sokoban:76aa9a45773544cf9cd81abac6841a19e40a2da97cc72af3d56db75a7bc685da", "split": "test", "board": {"height": 7, "width": 6, "floors": [7, 8, 9, 10, 13, 14, 15, 16, 19, 20, 21, 22, 25, 26, 27, 28, 31, 32, 33, 34], "goals": [13, 28]}, "initial_state": {"player": 31, "boxes": [20, 27]}, "generator_seed": 20261024, "rules_version": "sokoban-v1", "render": {"theme": "classic", "tile_size": 44}, "language": "en", "max_steps": 200} +{"level_id": "sokoban-test-042", "group_id": "sokoban:3438048303fe4a4f1dbcc782882310b1065a7096cbee913bcc85dabd529ce7c7", "split": "test", "board": {"height": 9, "width": 9, "floors": [10, 11, 12, 20, 29, 30, 37, 38, 47, 48, 49, 50, 51, 52, 55, 56, 57, 58, 59, 60, 64, 65, 66, 67, 68], "goals": [38, 50, 65]}, "initial_state": {"player": 51, "boxes": [47, 49, 58]}, "generator_seed": 20261025, "rules_version": "sokoban-v1", "render": {"theme": "classic", "tile_size": 44}, "language": "zh", "max_steps": 200} +{"level_id": "sokoban-test-045", "group_id": "sokoban:89f147dab500b520aa88f693029bbccd09acadd547f17dd78852c84f5c2ab281", "split": "test", "board": {"height": 6, "width": 7, "floors": [8, 9, 10, 11, 12, 15, 16, 17, 18, 19, 22, 23, 24, 25, 26, 29, 30, 31, 32, 33], "goals": [11]}, "initial_state": {"player": 11, "boxes": [23]}, "generator_seed": 20261028, "rules_version": "sokoban-v1", "render": {"theme": "warm", "tile_size": 44}, "language": "en", "max_steps": 200} +{"level_id": "sokoban-test-050", "group_id": "sokoban:e1f6a67efd6b02d427983403c5156d4ee1da715cc2882cd0e2e46ef04a7eb1db", "split": "test", "board": {"height": 6, "width": 8, "floors": [9, 10, 11, 13, 14, 17, 18, 19, 20, 21, 22, 25, 26, 27, 28, 29, 30, 33, 34, 35, 36, 37, 38], "goals": [35, 36]}, "initial_state": {"player": 30, "boxes": [29, 35]}, "generator_seed": 20261034, "rules_version": "sokoban-v1", "render": {"theme": "classic", "tile_size": 44}, "language": "zh", "max_steps": 200} +{"level_id": "sokoban-test-051", "group_id": "sokoban:b43e2140885812d16611ef9e849924e19fc9d92d5a307c4878f320f0579d564c", "split": "test", "board": {"height": 8, "width": 7, "floors": [8, 9, 10, 11, 15, 16, 17, 18, 22, 23, 24, 25, 26, 30, 31, 32, 36, 37, 43, 44, 45, 46], "goals": [16, 17, 25]}, "initial_state": {"player": 37, "boxes": [17, 23, 24]}, "generator_seed": 20261035, "rules_version": "sokoban-v1", "render": {"theme": "warm", "tile_size": 44}, "language": "en", "max_steps": 200} +{"level_id": "sokoban-test-053", "group_id": "sokoban:4aaefb6a1d5010377af8e8335485eb48e8a4433f85891862838f2b8efebc13fe", "split": "test", "board": {"height": 9, "width": 6, "floors": [7, 8, 9, 10, 13, 14, 15, 16, 19, 20, 21, 22, 25, 26, 27, 28, 31, 32, 33, 34, 37, 38, 39, 40, 43, 44, 46], "goals": [33, 40]}, "initial_state": {"player": 13, "boxes": [16, 21]}, "generator_seed": 20261037, "rules_version": "sokoban-v1", "render": {"theme": "classic", "tile_size": 44}, "language": "en", "max_steps": 200} +{"level_id": "sokoban-test-055", "group_id": "sokoban:42c0d7ddd581f852828e654577b59b26cfe0e2484a22b1a5aa3f64a62d1e7cdd", "split": "test", "board": {"height": 7, "width": 8, "floors": [10, 12, 13, 14, 18, 19, 20, 21, 22, 25, 26, 27, 28, 29, 30, 33, 34, 36, 37, 38, 41, 42, 43, 44, 45, 46], "goals": [20, 29, 36]}, "initial_state": {"player": 42, "boxes": [21, 27, 29]}, "generator_seed": 20261039, "rules_version": "sokoban-v1", "render": {"theme": "warm", "tile_size": 44}, "language": "en", "max_steps": 200} +{"level_id": "sokoban-test-060", "group_id": "sokoban:20cdcb3b3ad476ccd821196e41316c92ee1a61b82c065763600d5561e3b99563", "split": "test", "board": {"height": 8, "width": 6, "floors": [7, 8, 9, 10, 13, 14, 15, 16, 19, 20, 21, 22, 25, 26, 27, 28, 31, 32, 33, 34, 37, 38, 39, 40], "goals": [21, 28, 32]}, "initial_state": {"player": 13, "boxes": [28, 32, 33]}, "generator_seed": 20261076, "rules_version": "sokoban-v1", "render": {"theme": "classic", "tile_size": 44}, "language": "zh", "max_steps": 200} +{"level_id": "sokoban-test-061", "group_id": "sokoban:23993dcb0f895d46df161e8aad64ea7037ac89800557f652591ad4ecdfbdd108", "split": "test", "board": {"height": 6, "width": 9, "floors": [10, 11, 12, 14, 15, 16, 20, 21, 22, 23, 24, 25, 29, 30, 31, 32, 33, 34, 37, 38, 39, 40, 41, 42, 43], "goals": [11, 31, 32]}, "initial_state": {"player": 43, "boxes": [11, 23, 32]}, "generator_seed": 20261077, "rules_version": "sokoban-v1", "render": {"theme": "warm", "tile_size": 44}, "language": "en", "max_steps": 200} +{"level_id": "sokoban-test-064", "group_id": "sokoban:5245c24526d948a6f7c2212fc7b756c0ac6c56bba3d6df42b34bb453c1ebe0ba", "split": "test", "board": {"height": 7, "width": 6, "floors": [7, 8, 9, 10, 13, 14, 15, 16, 19, 20, 21, 22, 25, 26, 27, 28, 31, 32, 33, 34], "goals": [16, 25, 32]}, "initial_state": {"player": 19, "boxes": [14, 21, 33]}, "generator_seed": 20261080, "rules_version": "sokoban-v1", "render": {"theme": "classic", "tile_size": 44}, "language": "zh", "max_steps": 200} +{"level_id": "sokoban-test-065", "group_id": "sokoban:463b1dcae20fbb97a01af3d57534c21996b7f86359adaf4df516dfb3ba3c6f30", "split": "test", "board": {"height": 9, "width": 7, "floors": [8, 9, 10, 11, 12, 15, 16, 17, 18, 19, 22, 23, 24, 25, 26, 29, 30, 31, 32, 38, 39, 40, 44, 45, 51], "goals": [10, 17, 39]}, "initial_state": {"player": 8, "boxes": [10, 17, 18]}, "generator_seed": 20261081, "rules_version": "sokoban-v1", "render": {"theme": "classic", "tile_size": 44}, "language": "en", "max_steps": 200} +{"level_id": "sokoban-test-066", "group_id": "sokoban:c0fe29853eb09f011cf61e899dfb17d48587a6a001871385215ccec7012dae8c", "split": "test", "board": {"height": 6, "width": 9, "floors": [10, 11, 12, 13, 14, 15, 16, 19, 20, 21, 22, 23, 24, 25, 28, 29, 30, 31, 32, 33, 34, 37, 38, 39, 40, 41, 42, 43], "goals": [22, 33, 41]}, "initial_state": {"player": 24, "boxes": [32, 33, 41]}, "generator_seed": 20261082, "rules_version": "sokoban-v1", "render": {"theme": "warm", "tile_size": 44}, "language": "zh", "max_steps": 200} +{"level_id": "sokoban-test-070", "group_id": "sokoban:b25d105032c87959f9503f8aa77c23075f1159bbf2f4758613b0c6b291353120", "split": "test", "board": {"height": 7, "width": 7, "floors": [8, 9, 10, 11, 12, 15, 16, 17, 18, 19, 22, 23, 24, 25, 26, 29, 30, 31, 32, 33, 36, 37, 38, 39, 40], "goals": [17, 26, 33]}, "initial_state": {"player": 38, "boxes": [23, 26, 32]}, "generator_seed": 20261105, "rules_version": "sokoban-v1", "render": {"theme": "classic", "tile_size": 44}, "language": "zh", "max_steps": 200} +{"level_id": "sokoban-test-071", "group_id": "sokoban:015799c1c52d28ef48fd124681e2b9d948c47208caeee3b6cc45b510d82be839", "split": "test", "board": {"height": 6, "width": 6, "floors": [7, 8, 9, 10, 13, 14, 15, 16, 19, 20, 21, 22, 25, 26, 27, 28], "goals": [14]}, "initial_state": {"player": 26, "boxes": [15]}, "generator_seed": 20261106, "rules_version": "sokoban-v1", "render": {"theme": "classic", "tile_size": 44}, "language": "en", "max_steps": 200} +{"level_id": "sokoban-test-073", "group_id": "sokoban:6e34b4592cc77a6214fcf81192b61e2f4846e93cf6c15d2f2b6d7cc637d0dd1c", "split": "test", "board": {"height": 7, "width": 6, "floors": [7, 8, 9, 10, 13, 14, 15, 16, 19, 20, 21, 22, 25, 26, 27, 28, 31, 32, 33, 34], "goals": [13, 14]}, "initial_state": {"player": 14, "boxes": [20, 21]}, "generator_seed": 20261109, "rules_version": "sokoban-v1", "render": {"theme": "classic", "tile_size": 44}, "language": "en", "max_steps": 200} +{"level_id": "sokoban-test-074", "group_id": "sokoban:bd0d3b065326d5b4042c062d3a5a2c4c1d68f4981de00ac1973c06305a4c67f3", "split": "test", "board": {"height": 8, "width": 7, "floors": [9, 10, 12, 15, 17, 18, 19, 22, 23, 24, 25, 26, 29, 30, 31, 32, 33, 36, 37, 38, 39, 40, 43, 44, 45, 46, 47], "goals": [24, 25]}, "initial_state": {"player": 39, "boxes": [24, 37]}, "generator_seed": 20261110, "rules_version": "sokoban-v1", "render": {"theme": "warm", "tile_size": 44}, "language": "zh", "max_steps": 200} +{"level_id": "sokoban-test-076", "group_id": "sokoban:b47d8f6f649a9bde91199bcf5dcf194d552e1eecd6934390782f7f046ec7ffde", "split": "test", "board": {"height": 7, "width": 7, "floors": [8, 9, 10, 11, 15, 16, 17, 18, 19, 23, 24, 25, 26, 29, 30, 31, 32, 33, 36, 37, 38, 39, 40], "goals": [9, 10]}, "initial_state": {"player": 15, "boxes": [25, 31]}, "generator_seed": 20261112, "rules_version": "sokoban-v1", "render": {"theme": "warm", "tile_size": 44}, "language": "zh", "max_steps": 200} +{"level_id": "sokoban-test-080", "group_id": "sokoban:7bedb7ec5cdbece010492af7016edb906d6a9302b63346c86eb807d12737fd53", "split": "test", "board": {"height": 6, "width": 6, "floors": [7, 8, 9, 10, 13, 14, 15, 16, 19, 20, 21, 22, 25, 26, 27, 28], "goals": [20, 22]}, "initial_state": {"player": 22, "boxes": [16, 20]}, "generator_seed": 20261137, "rules_version": "sokoban-v1", "render": {"theme": "warm", "tile_size": 44}, "language": "zh", "max_steps": 200} +{"level_id": "sokoban-test-081", "group_id": "sokoban:fc896d18f38b2716e54421b896ea6d3c8f940d3b464c1d40564d3752b2dacaf7", "split": "test", "board": {"height": 8, "width": 7, "floors": [8, 9, 10, 11, 12, 15, 16, 17, 18, 19, 22, 23, 24, 25, 26, 29, 30, 31, 32, 33, 36, 37, 38, 39, 40, 43, 44, 45, 46, 47], "goals": [16, 45, 46]}, "initial_state": {"player": 43, "boxes": [23, 30, 46]}, "generator_seed": 20261138, "rules_version": "sokoban-v1", "render": {"theme": "warm", "tile_size": 44}, "language": "en", "max_steps": 200} +{"level_id": "sokoban-test-082", "group_id": "sokoban:acb9ac962b5417b4421f469527e7492363104e0c3496418cec626c67dda43a4f", "split": "test", "board": {"height": 6, "width": 6, "floors": [7, 8, 9, 10, 13, 14, 15, 16, 19, 20, 21, 22, 25, 26, 27, 28], "goals": [19, 26]}, "initial_state": {"player": 10, "boxes": [14, 15]}, "generator_seed": 20261140, "rules_version": "sokoban-v1", "render": {"theme": "warm", "tile_size": 44}, "language": "zh", "max_steps": 200} +{"level_id": "sokoban-test-083", "group_id": "sokoban:564ba915b18ce19c9eccd461374d8c477e4eb0f7d4971ec631e1c5a0d4ff491f", "split": "test", "board": {"height": 8, "width": 7, "floors": [8, 9, 10, 11, 12, 15, 16, 17, 18, 19, 22, 24, 25, 26, 29, 30, 31, 33, 36, 38, 39, 43], "goals": [9, 10]}, "initial_state": {"player": 39, "boxes": [17, 18]}, "generator_seed": 20261141, "rules_version": "sokoban-v1", "render": {"theme": "classic", "tile_size": 44}, "language": "en", "max_steps": 200} +{"level_id": "sokoban-test-084", "group_id": "sokoban:1996a693e23c88973e4863e09382d34f4b71f0fa1411e45758ee83d3a8f48a49", "split": "test", "board": {"height": 7, "width": 9, "floors": [12, 13, 15, 16, 21, 22, 23, 24, 25, 28, 29, 31, 32, 33, 34, 37, 38, 39, 40, 41, 42, 43, 46, 47, 48, 49, 50, 51, 52], "goals": [24, 37, 43]}, "initial_state": {"player": 25, "boxes": [31, 32, 40]}, "generator_seed": 20261142, "rules_version": "sokoban-v1", "render": {"theme": "warm", "tile_size": 44}, "language": "zh", "max_steps": 200} +{"level_id": "sokoban-test-085", "group_id": "sokoban:0ab42fe198151eaf95788914af2cc241633f3c569d6628438064e412240d3310", "split": "test", "board": {"height": 9, "width": 8, "floors": [18, 20, 25, 26, 27, 28, 29, 33, 35, 36, 37, 38, 41, 43, 44, 45, 46, 49, 50, 51, 52, 53, 54, 57, 58, 59, 60, 61, 62], "goals": [41, 43, 44]}, "initial_state": {"player": 25, "boxes": [36, 52, 53]}, "generator_seed": 20261143, "rules_version": "sokoban-v1", "render": {"theme": "warm", "tile_size": 44}, "language": "en", "max_steps": 200} +{"level_id": "sokoban-test-086", "group_id": "sokoban:62911120f8feee2023beb6da85b077797d6b2015864ed889ffac2ebb2af99048", "split": "test", "board": {"height": 8, "width": 9, "floors": [13, 14, 15, 16, 21, 22, 23, 24, 25, 31, 32, 33, 34, 40, 41, 42, 43, 50, 51, 57, 58, 59, 60, 61], "goals": [22, 59]}, "initial_state": {"player": 22, "boxes": [24, 31]}, "generator_seed": 20261144, "rules_version": "sokoban-v1", "render": {"theme": "warm", "tile_size": 44}, "language": "zh", "max_steps": 200} +{"level_id": "sokoban-test-090", "group_id": "sokoban:5ae88afab3fe474f368e89bed7e2e2d248348f85698c697fd84e0c712c975b10", "split": "test", "board": {"height": 8, "width": 6, "floors": [7, 8, 9, 10, 13, 14, 15, 16, 19, 20, 21, 22, 25, 26, 27, 28, 31, 32, 33, 38, 39, 40], "goals": [22, 39]}, "initial_state": {"player": 26, "boxes": [16, 33]}, "generator_seed": 20261188, "rules_version": "sokoban-v1", "render": {"theme": "warm", "tile_size": 44}, "language": "zh", "max_steps": 200} +{"level_id": "sokoban-test-093", "group_id": "sokoban:aa463d6e45c3bc993979cea0da349f0a8716e4d98a8c55d991cbf428d432cf43", "split": "test", "board": {"height": 7, "width": 7, "floors": [8, 9, 10, 11, 12, 15, 16, 17, 18, 19, 22, 23, 24, 25, 26, 29, 30, 31, 32, 33, 36, 37, 38, 39, 40], "goals": [23, 37]}, "initial_state": {"player": 26, "boxes": [17, 32]}, "generator_seed": 20261191, "rules_version": "sokoban-v1", "render": {"theme": "classic", "tile_size": 44}, "language": "en", "max_steps": 200} +{"level_id": "sokoban-test-094", "group_id": "sokoban:b66052a3c9ec940354c2561603e0c2cda4e25db44e4d20b35fd193095d075f91", "split": "test", "board": {"height": 7, "width": 7, "floors": [8, 9, 10, 11, 12, 15, 16, 17, 18, 19, 22, 23, 24, 25, 26, 30, 31, 32, 33, 38, 39, 40], "goals": [10]}, "initial_state": {"player": 9, "boxes": [32]}, "generator_seed": 20261193, "rules_version": "sokoban-v1", "render": {"theme": "classic", "tile_size": 44}, "language": "zh", "max_steps": 200} +{"level_id": "sokoban-test-095", "group_id": "sokoban:1342d472171124b23d5ad670865cc10d301478b04f0a405f3cb242f2b9cd3135", "split": "test", "board": {"height": 9, "width": 7, "floors": [8, 9, 10, 11, 12, 15, 16, 17, 18, 19, 22, 23, 24, 25, 26, 29, 30, 31, 32, 33, 36, 37, 38, 39, 40, 43, 44, 45, 46, 51, 53], "goals": [22, 23, 24]}, "initial_state": {"player": 53, "boxes": [16, 17, 36]}, "generator_seed": 20261194, "rules_version": "sokoban-v1", "render": {"theme": "classic", "tile_size": 44}, "language": "en", "max_steps": 200} +{"level_id": "sokoban-test-096", "group_id": "sokoban:858ed7c13df240fbf9df1a9d3a0b61427931bac87d9a36cf7d627f9b4fc426fe", "split": "test", "board": {"height": 8, "width": 9, "floors": [11, 12, 13, 19, 20, 21, 28, 29, 30, 32, 37, 38, 39, 40, 41, 42, 43, 46, 47, 48, 49, 50, 51, 52, 55, 56, 58, 59, 60, 61], "goals": [12, 21, 52]}, "initial_state": {"player": 48, "boxes": [20, 49, 51]}, "generator_seed": 20261195, "rules_version": "sokoban-v1", "render": {"theme": "classic", "tile_size": 44}, "language": "zh", "max_steps": 200} +{"level_id": "sokoban-test-100", "group_id": "sokoban:42bbcd317e81fa995996772dbaa395976f5be8373bb3a322ceda8fada9d242cf", "split": "test", "board": {"height": 6, "width": 8, "floors": [9, 10, 11, 12, 13, 14, 17, 18, 19, 20, 21, 22, 25, 26, 27, 28, 29, 30, 34, 37], "goals": [13, 28]}, "initial_state": {"player": 26, "boxes": [10, 28]}, "generator_seed": 20261214, "rules_version": "sokoban-v1", "render": {"theme": "warm", "tile_size": 44}, "language": "zh", "max_steps": 200} +{"level_id": "sokoban-test-101", "group_id": "sokoban:2cd39c7c305198688869204a2251c1cc5353c6ba3c0ab24f2d5ec28138ea7138", "split": "test", "board": {"height": 8, "width": 6, "floors": [7, 8, 9, 10, 13, 14, 15, 16, 19, 20, 21, 22, 25, 26, 27, 28, 31, 32, 33, 34, 37, 38, 39, 40], "goals": [13, 33]}, "initial_state": {"player": 37, "boxes": [31, 33]}, "generator_seed": 20261215, "rules_version": "sokoban-v1", "render": {"theme": "warm", "tile_size": 44}, "language": "en", "max_steps": 200} +{"level_id": "sokoban-test-102", "group_id": "sokoban:e3d8caf11671e2d908629bf81aade5c75fdd01b526b6ffb506649a89664a917b", "split": "test", "board": {"height": 6, "width": 9, "floors": [11, 12, 13, 14, 15, 19, 20, 21, 22, 23, 24, 25, 28, 30, 31, 32, 33, 34, 37, 38, 39, 40, 41, 42, 43], "goals": [21]}, "initial_state": {"player": 20, "boxes": [33]}, "generator_seed": 20261216, "rules_version": "sokoban-v1", "render": {"theme": "classic", "tile_size": 44}, "language": "zh", "max_steps": 200} +{"level_id": "sokoban-test-103", "group_id": "sokoban:47935a5e7fe0306c1d0fc746690fbc821b5fc323c592c0d2247fb9aec77e76d1", "split": "test", "board": {"height": 6, "width": 8, "floors": [9, 10, 11, 12, 13, 18, 19, 20, 21, 22, 25, 26, 27, 28, 29, 30, 35, 36, 37, 38], "goals": [18, 37]}, "initial_state": {"player": 12, "boxes": [27, 36]}, "generator_seed": 20261217, "rules_version": "sokoban-v1", "render": {"theme": "warm", "tile_size": 44}, "language": "en", "max_steps": 200} +{"level_id": "sokoban-test-104", "group_id": "sokoban:18eb4bac9b8ae20e7e1c8abd185357919ed780742154eefd73bebba3cb329e8f", "split": "test", "board": {"height": 9, "width": 6, "floors": [7, 8, 9, 10, 13, 14, 15, 16, 19, 20, 21, 22, 25, 26, 27, 28, 31, 32, 33, 34, 37, 38, 39, 40, 44, 45], "goals": [9, 15, 33]}, "initial_state": {"player": 26, "boxes": [8, 21, 39]}, "generator_seed": 20261218, "rules_version": "sokoban-v1", "render": {"theme": "classic", "tile_size": 44}, "language": "zh", "max_steps": 200} +{"level_id": "sokoban-test-110", "group_id": "sokoban:63e9eb6d52aaa7c24df7d24f2ebdcbd0933867c6614b2bb36750fa5e02bcf4c2", "split": "test", "board": {"height": 6, "width": 7, "floors": [8, 9, 10, 11, 12, 15, 16, 17, 18, 19, 22, 23, 24, 25, 26, 29, 30, 31, 32, 33], "goals": [9, 22, 31]}, "initial_state": {"player": 32, "boxes": [22, 24, 25]}, "generator_seed": 20261241, "rules_version": "sokoban-v1", "render": {"theme": "classic", "tile_size": 44}, "language": "zh", "max_steps": 200} +{"level_id": "sokoban-test-111", "group_id": "sokoban:80dc0542c4b4564e1a7dd74abb4ed77392ebdde69ba4fb10ad67104029ff8587", "split": "test", "board": {"height": 6, "width": 9, "floors": [10, 11, 12, 13, 14, 15, 16, 19, 20, 21, 22, 23, 24, 25, 28, 29, 30, 31, 32, 33, 34, 37, 38, 39, 40, 41, 42, 43], "goals": [12, 21, 25]}, "initial_state": {"player": 43, "boxes": [22, 24, 32]}, "generator_seed": 20261242, "rules_version": "sokoban-v1", "render": {"theme": "warm", "tile_size": 44}, "language": "en", "max_steps": 200} +{"level_id": "sokoban-test-113", "group_id": "sokoban:0dedbe5c724ac3cc4d1052da4d304d7a433a5cec1826bb4731e1269b3d8b92b7", "split": "test", "board": {"height": 8, "width": 7, "floors": [9, 10, 11, 12, 16, 17, 18, 19, 22, 23, 24, 25, 26, 29, 30, 31, 32, 33, 36, 37, 38, 39, 40, 43, 44, 45, 46], "goals": [38, 39]}, "initial_state": {"player": 9, "boxes": [17, 18]}, "generator_seed": 20261244, "rules_version": "sokoban-v1", "render": {"theme": "classic", "tile_size": 44}, "language": "en", "max_steps": 200} +{"level_id": "sokoban-test-114", "group_id": "sokoban:7a127d219b8539c5e2d34d3235069c37911fea08052675f9014a12e18dfbc635", "split": "test", "board": {"height": 9, "width": 8, "floors": [9, 10, 11, 12, 13, 14, 17, 18, 19, 20, 21, 22, 26, 28, 29, 30, 33, 34, 35, 42, 49, 50], "goals": [10, 26]}, "initial_state": {"player": 50, "boxes": [11, 19]}, "generator_seed": 20261245, "rules_version": "sokoban-v1", "render": {"theme": "classic", "tile_size": 44}, "language": "zh", "max_steps": 200} +{"level_id": "sokoban-test-121", "group_id": "sokoban:4dc426985c7b3e491b8ce0380fa3f39f1e6dd070f80fe9b13169fdc049f4496b", "split": "test", "board": {"height": 7, "width": 6, "floors": [7, 8, 9, 10, 13, 14, 15, 16, 19, 20, 21, 22, 25, 26, 27, 28, 31, 32, 33, 34], "goals": [16, 22, 25]}, "initial_state": {"player": 8, "boxes": [16, 20, 28]}, "generator_seed": 20261257, "rules_version": "sokoban-v1", "render": {"theme": "warm", "tile_size": 44}, "language": "en", "max_steps": 200} +{"level_id": "sokoban-test-122", "group_id": "sokoban:50841741e49e03ace8deaeb1b84736226c183ca5dc6a58e9760c31a200ca832c", "split": "test", "board": {"height": 9, "width": 9, "floors": [20, 29, 30, 32, 33, 37, 38, 39, 40, 41, 42, 46, 47, 48, 49, 55, 56, 57, 58, 59, 64, 65, 66, 67], "goals": [38, 46]}, "initial_state": {"player": 32, "boxes": [56, 57]}, "generator_seed": 20261258, "rules_version": "sokoban-v1", "render": {"theme": "warm", "tile_size": 44}, "language": "zh", "max_steps": 200} +{"level_id": "sokoban-test-123", "group_id": "sokoban:761db96ddb2d1c32dd6dca4bb271c44505048b6ac10a7dd1ed523cb7ced3840b", "split": "test", "board": {"height": 8, "width": 8, "floors": [10, 11, 12, 13, 14, 17, 18, 19, 20, 21, 22, 25, 26, 27, 28, 29, 30, 33, 37, 38, 46], "goals": [18, 21]}, "initial_state": {"player": 18, "boxes": [21, 29]}, "generator_seed": 20261259, "rules_version": "sokoban-v1", "render": {"theme": "classic", "tile_size": 44}, "language": "en", "max_steps": 200} +{"level_id": "sokoban-test-124", "group_id": "sokoban:e07312af79bb2aeff73968808b78a3487bc0acd6a8be1c5748bb2335e3900ad6", "split": "test", "board": {"height": 7, "width": 8, "floors": [9, 10, 11, 12, 13, 14, 17, 18, 19, 20, 21, 22, 25, 26, 27, 29, 33, 34, 35, 36, 37, 38, 41, 42, 43, 44, 45, 46], "goals": [20, 26]}, "initial_state": {"player": 11, "boxes": [26, 34]}, "generator_seed": 20261260, "rules_version": "sokoban-v1", "render": {"theme": "classic", "tile_size": 44}, "language": "zh", "max_steps": 200} +{"level_id": "sokoban-test-125", "group_id": "sokoban:e22a26e38667a4b56e0e13c9895be3cd2aa6fa4e2cbd177af5178570d7c36bd2", "split": "test", "board": {"height": 6, "width": 9, "floors": [10, 11, 12, 13, 14, 15, 16, 19, 20, 21, 22, 23, 24, 25, 28, 29, 30, 31, 32, 33, 34, 37, 38, 39, 40, 41, 42, 43], "goals": [12, 40]}, "initial_state": {"player": 19, "boxes": [11, 42]}, "generator_seed": 20261262, "rules_version": "sokoban-v1", "render": {"theme": "classic", "tile_size": 44}, "language": "en", "max_steps": 200} +{"level_id": "sokoban-test-126", "group_id": "sokoban:20015296b0af89754092f5836a33b5dd65ca2281a4109c5aed073c3738fbd9d4", "split": "test", "board": {"height": 7, "width": 7, "floors": [8, 9, 10, 11, 12, 15, 16, 17, 18, 19, 22, 23, 24, 25, 26, 29, 30, 31, 32, 33, 36, 37, 38, 39, 40], "goals": [22, 29]}, "initial_state": {"player": 26, "boxes": [16, 24]}, "generator_seed": 20261263, "rules_version": "sokoban-v1", "render": {"theme": "warm", "tile_size": 44}, "language": "zh", "max_steps": 200} +{"level_id": "sokoban-test-130", "group_id": "sokoban:540b40bd6414f396419e8f0c58b8b1b7fb17c9d16f3458fb7b820ab4979c858f", "split": "test", "board": {"height": 6, "width": 6, "floors": [7, 8, 9, 10, 13, 14, 15, 16, 19, 20, 21, 22, 25, 26, 27, 28], "goals": [8, 19]}, "initial_state": {"player": 25, "boxes": [14, 21]}, "generator_seed": 20261300, "rules_version": "sokoban-v1", "render": {"theme": "warm", "tile_size": 44}, "language": "zh", "max_steps": 200} +{"level_id": "sokoban-test-131", "group_id": "sokoban:b91e1530727a54319febefe6e598d6f7c34de6bb22890fc05a20ad56764b0422", "split": "test", "board": {"height": 7, "width": 6, "floors": [7, 8, 9, 10, 13, 14, 15, 16, 19, 20, 21, 22, 25, 26, 27, 28, 31, 32, 33, 34], "goals": [9, 27]}, "initial_state": {"player": 22, "boxes": [14, 15]}, "generator_seed": 20261301, "rules_version": "sokoban-v1", "render": {"theme": "classic", "tile_size": 44}, "language": "en", "max_steps": 200} +{"level_id": "sokoban-test-133", "group_id": "sokoban:e74042a36738cae133fdad2606358d763812aec07b25e008d5f1174395933141", "split": "test", "board": {"height": 6, "width": 9, "floors": [13, 14, 15, 16, 19, 20, 21, 22, 23, 24, 25, 29, 30, 31, 32, 33, 34, 37, 38, 39, 40, 41, 42, 43], "goals": [21, 41]}, "initial_state": {"player": 19, "boxes": [24, 31]}, "generator_seed": 20261303, "rules_version": "sokoban-v1", "render": {"theme": "warm", "tile_size": 44}, "language": "en", "max_steps": 200} +{"level_id": "sokoban-test-134", "group_id": "sokoban:34cf7ec0e710a4851c5815650667c73d1b211071a083b6d1b49ef9a25ecfa58b", "split": "test", "board": {"height": 6, "width": 7, "floors": [8, 9, 10, 11, 12, 15, 16, 17, 18, 19, 22, 23, 24, 25, 26, 29, 30, 31, 32, 33], "goals": [15, 26]}, "initial_state": {"player": 12, "boxes": [22, 23]}, "generator_seed": 20261304, "rules_version": "sokoban-v1", "render": {"theme": "classic", "tile_size": 44}, "language": "zh", "max_steps": 200} +{"level_id": "sokoban-test-141", "group_id": "sokoban:6570e1682f5d362e14c286940ec504d8d9e0e31c2df628e5d1dc9f200c77db52", "split": "test", "board": {"height": 7, "width": 7, "floors": [8, 9, 10, 11, 12, 15, 16, 17, 18, 19, 22, 23, 24, 25, 26, 29, 30, 31, 32, 33, 36, 37, 38, 39, 40], "goals": [25, 32]}, "initial_state": {"player": 8, "boxes": [16, 31]}, "generator_seed": 20261315, "rules_version": "sokoban-v1", "render": {"theme": "warm", "tile_size": 44}, "language": "en", "max_steps": 200} +{"level_id": "sokoban-test-142", "group_id": "sokoban:c227cf37d3499c2120da3b55dbd6b8bc6902e7738121ee4e35268a912d17be8b", "split": "test", "board": {"height": 8, "width": 8, "floors": [9, 10, 17, 18, 19, 20, 25, 26, 28, 29, 30, 33, 34, 35, 36, 37, 41, 42, 44, 50, 51, 52, 53], "goals": [26, 33, 42]}, "initial_state": {"player": 10, "boxes": [18, 33, 34]}, "generator_seed": 20261316, "rules_version": "sokoban-v1", "render": {"theme": "classic", "tile_size": 44}, "language": "zh", "max_steps": 200} +{"level_id": "sokoban-test-143", "group_id": "sokoban:9e1165915778cd95a287cfeef8e3c53f53f59c904bee9be2a23aacd733102ad1", "split": "test", "board": {"height": 9, "width": 6, "floors": [7, 8, 9, 10, 13, 14, 15, 16, 19, 20, 21, 22, 25, 26, 27, 28, 31, 32, 33, 34, 37, 38, 39, 40, 43, 44, 45, 46], "goals": [31, 34, 39]}, "initial_state": {"player": 21, "boxes": [13, 15, 33]}, "generator_seed": 20261317, "rules_version": "sokoban-v1", "render": {"theme": "warm", "tile_size": 44}, "language": "en", "max_steps": 200} +{"level_id": "sokoban-test-145", "group_id": "sokoban:155354eb1574655d5a3893479903825b13b18536403d416e6d654fa61817e4ff", "split": "test", "board": {"height": 9, "width": 7, "floors": [9, 16, 23, 24, 30, 31, 32, 36, 37, 38, 39, 40, 43, 44, 45, 46, 47, 51, 52, 53, 54], "goals": [16, 47]}, "initial_state": {"player": 9, "boxes": [31, 46]}, "generator_seed": 20261319, "rules_version": "sokoban-v1", "render": {"theme": "classic", "tile_size": 44}, "language": "en", "max_steps": 200} +{"level_id": "sokoban-test-150", "group_id": "sokoban:f3a13bafd639f21760c0be7d66773db145d20e87ca9c678a5a948811463f35d9", "split": "test", "board": {"height": 7, "width": 9, "floors": [10, 19, 21, 28, 29, 30, 31, 32, 33, 37, 39, 40, 41, 42, 46, 47, 48, 49], "goals": [48]}, "initial_state": {"player": 33, "boxes": [39]}, "generator_seed": 20261338, "rules_version": "sokoban-v1", "render": {"theme": "warm", "tile_size": 44}, "language": "zh", "max_steps": 200} +{"level_id": "sokoban-test-151", "group_id": "sokoban:8dbb21e433cfc0ef05ee9addbaee50818c6d7da00de87c21f5038c01ee904199", "split": "test", "board": {"height": 6, "width": 7, "floors": [8, 9, 10, 11, 12, 15, 16, 17, 18, 19, 22, 23, 24, 25, 26, 29, 30, 31, 32, 33], "goals": [17, 25, 26]}, "initial_state": {"player": 23, "boxes": [18, 19, 25]}, "generator_seed": 20261339, "rules_version": "sokoban-v1", "render": {"theme": "warm", "tile_size": 44}, "language": "en", "max_steps": 200} +{"level_id": "sokoban-test-152", "group_id": "sokoban:8ed3d7e2534aea0ba1838bab646b25cc65ad914194d2c617f74541a96a91df11", "split": "test", "board": {"height": 9, "width": 7, "floors": [11, 12, 18, 19, 23, 25, 26, 29, 30, 31, 32, 33, 37, 38, 39, 40, 45, 46, 47, 51, 52, 53], "goals": [39, 52]}, "initial_state": {"player": 32, "boxes": [18, 31]}, "generator_seed": 20261340, "rules_version": "sokoban-v1", "render": {"theme": "warm", "tile_size": 44}, "language": "zh", "max_steps": 200} +{"level_id": "sokoban-test-153", "group_id": "sokoban:9ee1630bef18caed154db6738ca8f2bac22ad47c87e266419d3ee45216bda3cb", "split": "test", "board": {"height": 8, "width": 6, "floors": [7, 8, 9, 10, 13, 14, 15, 16, 19, 20, 21, 22, 25, 26, 27, 28, 31, 32, 33, 34, 37, 38, 39, 40], "goals": [16, 21]}, "initial_state": {"player": 20, "boxes": [28, 32]}, "generator_seed": 20261341, "rules_version": "sokoban-v1", "render": {"theme": "warm", "tile_size": 44}, "language": "en", "max_steps": 200} +{"level_id": "sokoban-test-154", "group_id": "sokoban:4d8aaad09895d039343ead79f67c1ef55a4fe067ac75b5dd9f4828139b87a960", "split": "test", "board": {"height": 7, "width": 9, "floors": [10, 11, 12, 13, 19, 20, 21, 22, 23, 24, 25, 28, 29, 31, 32, 34, 37, 38, 39, 40, 41, 42, 43, 46, 49, 50, 51, 52], "goals": [21, 50]}, "initial_state": {"player": 12, "boxes": [24, 39]}, "generator_seed": 20261342, "rules_version": "sokoban-v1", "render": {"theme": "warm", "tile_size": 44}, "language": "zh", "max_steps": 200} +{"level_id": "sokoban-test-155", "group_id": "sokoban:457c5abe3a3190a071cc31a0ff968f36a08af67dc109059cd5cb8286ca8c888c", "split": "test", "board": {"height": 6, "width": 8, "floors": [9, 10, 11, 12, 13, 14, 17, 18, 19, 20, 21, 22, 25, 26, 27, 28, 29, 30, 33, 34, 35, 36, 37, 38], "goals": [19, 37]}, "initial_state": {"player": 37, "boxes": [28, 36]}, "generator_seed": 20261343, "rules_version": "sokoban-v1", "render": {"theme": "warm", "tile_size": 44}, "language": "en", "max_steps": 200} +{"level_id": "sokoban-test-156", "group_id": "sokoban:60ebfa957506bebe5c4014d8b20df9606f159bef95fd44e4d211150726cfb912", "split": "test", "board": {"height": 9, "width": 8, "floors": [9, 10, 11, 12, 13, 17, 18, 19, 20, 21, 22, 26, 27, 30, 35, 36, 41, 42, 43, 50, 51, 52, 59], "goals": [19, 20, 51]}, "initial_state": {"player": 12, "boxes": [18, 19, 20]}, "generator_seed": 20261344, "rules_version": "sokoban-v1", "render": {"theme": "warm", "tile_size": 44}, "language": "zh", "max_steps": 200} +{"level_id": "sokoban-test-160", "group_id": "sokoban:a4b7b2df836a6d11a4eb93453609878b1bfed491a0652d3c630e73fe0c5d5b1f", "split": "test", "board": {"height": 6, "width": 7, "floors": [8, 9, 10, 11, 12, 15, 16, 17, 18, 19, 22, 23, 24, 25, 26, 29, 30, 31, 32, 33], "goals": [18, 25]}, "initial_state": {"player": 31, "boxes": [16, 24]}, "generator_seed": 20261366, "rules_version": "sokoban-v1", "render": {"theme": "warm", "tile_size": 44}, "language": "zh", "max_steps": 200} +{"level_id": "sokoban-test-161", "group_id": "sokoban:7c37823c3cb5b365a9c96d35d9156f31260339fa8fb60acb31c4e19d35041c1d", "split": "test", "board": {"height": 8, "width": 8, "floors": [9, 10, 11, 12, 17, 18, 19, 20, 21, 25, 26, 27, 28, 29, 33, 34, 35, 41, 42, 43, 50], "goals": [11, 27]}, "initial_state": {"player": 28, "boxes": [11, 26]}, "generator_seed": 20261368, "rules_version": "sokoban-v1", "render": {"theme": "classic", "tile_size": 44}, "language": "en", "max_steps": 200} +{"level_id": "sokoban-test-162", "group_id": "sokoban:071341444bd23226626a63d0170fba803a118dadb274778cbc84db7edf3c1dd5", "split": "test", "board": {"height": 8, "width": 8, "floors": [9, 10, 14, 17, 18, 19, 22, 25, 26, 27, 28, 30, 33, 34, 35, 36, 37, 38, 41, 42, 43, 44, 45, 49, 50, 51, 52, 53, 54], "goals": [41, 44, 53]}, "initial_state": {"player": 35, "boxes": [26, 27, 50]}, "generator_seed": 20261369, "rules_version": "sokoban-v1", "render": {"theme": "warm", "tile_size": 44}, "language": "zh", "max_steps": 200} +{"level_id": "sokoban-test-163", "group_id": "sokoban:da459b20881ab573f70aadde86092c7f76fbecece8fadddb0b540f6c2fb60ade", "split": "test", "board": {"height": 9, "width": 8, "floors": [27, 30, 34, 35, 36, 37, 38, 42, 43, 44, 45, 46, 50, 51, 52, 53, 54, 59, 60, 61, 62], "goals": [37]}, "initial_state": {"player": 30, "boxes": [43]}, "generator_seed": 20261371, "rules_version": "sokoban-v1", "render": {"theme": "warm", "tile_size": 44}, "language": "en", "max_steps": 200} +{"level_id": "sokoban-test-164", "group_id": "sokoban:a3197b221b506c6e449b0b7019c875fcf33d98377bf7a8299fb495b5427f2ef5", "split": "test", "board": {"height": 6, "width": 9, "floors": [10, 11, 12, 13, 14, 15, 19, 20, 21, 22, 23, 24, 25, 29, 30, 31, 32, 33, 34, 38, 39, 40, 41, 42, 43], "goals": [20, 29, 40]}, "initial_state": {"player": 43, "boxes": [21, 31, 33]}, "generator_seed": 20261375, "rules_version": "sokoban-v1", "render": {"theme": "warm", "tile_size": 44}, "language": "zh", "max_steps": 200} +{"level_id": "sokoban-test-166", "group_id": "sokoban:c856bdb2370d025dd4a711f6d5e15653e62e58f2c96751b810609fbb8a9c5fee", "split": "test", "board": {"height": 7, "width": 7, "floors": [8, 9, 10, 11, 12, 15, 16, 17, 18, 19, 22, 23, 24, 25, 26, 29, 30, 31, 32, 33, 36, 37, 38, 39, 40], "goals": [32]}, "initial_state": {"player": 40, "boxes": [16]}, "generator_seed": 20261377, "rules_version": "sokoban-v1", "render": {"theme": "warm", "tile_size": 44}, "language": "zh", "max_steps": 200} +{"level_id": "sokoban-test-170", "group_id": "sokoban:5bfdbc1214296fb83f1a53891f07310ee7e5062564a52874c2dabc7586f36a6f", "split": "test", "board": {"height": 8, "width": 7, "floors": [9, 10, 15, 16, 19, 22, 23, 25, 26, 30, 31, 32, 33, 36, 37, 38, 39, 40, 44, 45, 46, 47], "goals": [23, 31]}, "initial_state": {"player": 44, "boxes": [30, 39]}, "generator_seed": 20261382, "rules_version": "sokoban-v1", "render": {"theme": "classic", "tile_size": 44}, "language": "zh", "max_steps": 200} +{"level_id": "sokoban-test-172", "group_id": "sokoban:03c01de692d3505efa386c8a6aa049e93680300b6f005b6dfa49592b189bdee5", "split": "test", "board": {"height": 6, "width": 9, "floors": [10, 11, 12, 13, 14, 15, 16, 19, 20, 21, 22, 23, 24, 25, 28, 29, 30, 31, 32, 33, 34, 37, 38, 39, 40, 41, 42, 43], "goals": [11, 14, 25]}, "initial_state": {"player": 11, "boxes": [14, 20, 32]}, "generator_seed": 20261384, "rules_version": "sokoban-v1", "render": {"theme": "classic", "tile_size": 44}, "language": "zh", "max_steps": 200} +{"level_id": "sokoban-test-175", "group_id": "sokoban:1b477cf45c4dce9fcd757816103623ad8f3d87715538dba70bd5d83470992bbb", "split": "test", "board": {"height": 8, "width": 6, "floors": [7, 8, 10, 13, 14, 15, 16, 19, 20, 21, 22, 25, 26, 27, 28, 31, 32, 33, 34, 37, 38, 39, 40], "goals": [19, 26]}, "initial_state": {"player": 40, "boxes": [14, 20]}, "generator_seed": 20261387, "rules_version": "sokoban-v1", "render": {"theme": "classic", "tile_size": 44}, "language": "en", "max_steps": 200} +{"level_id": "sokoban-test-176", "group_id": "sokoban:9c55fc730e19c2c6458ce37630d04a9b5dc298ea7d357f79d81e0c1665271414", "split": "test", "board": {"height": 7, "width": 8, "floors": [9, 10, 11, 12, 13, 14, 18, 21, 22, 26, 28, 29, 30, 33, 34, 35, 36, 37, 38, 42, 44, 45, 46], "goals": [30]}, "initial_state": {"player": 29, "boxes": [35]}, "generator_seed": 20261388, "rules_version": "sokoban-v1", "render": {"theme": "classic", "tile_size": 44}, "language": "zh", "max_steps": 200} +{"level_id": "sokoban-test-180", "group_id": "sokoban:fedf9958ac1f960eb01864135f88c4fc497319e775ea796e187b96fb1fbac6a6", "split": "test", "board": {"height": 7, "width": 8, "floors": [9, 10, 11, 12, 13, 14, 17, 18, 19, 20, 21, 22, 25, 26, 27, 28, 29, 30, 33, 34, 35, 36, 37, 38, 41, 42, 43, 44, 45, 46], "goals": [12, 17, 30]}, "initial_state": {"player": 46, "boxes": [17, 20, 29]}, "generator_seed": 20261414, "rules_version": "sokoban-v1", "render": {"theme": "warm", "tile_size": 44}, "language": "zh", "max_steps": 200} +{"level_id": "sokoban-test-181", "group_id": "sokoban:1b7da96dcf2621c6ca5fa1ebe4014a461a2a8b91f2559c31341ffa2bad9fc00b", "split": "test", "board": {"height": 7, "width": 7, "floors": [8, 9, 10, 11, 12, 15, 16, 17, 18, 19, 22, 23, 24, 25, 26, 29, 30, 31, 32, 33, 36, 37, 38, 39, 40], "goals": [10, 39]}, "initial_state": {"player": 22, "boxes": [9, 38]}, "generator_seed": 20261415, "rules_version": "sokoban-v1", "render": {"theme": "classic", "tile_size": 44}, "language": "en", "max_steps": 200} +{"level_id": "sokoban-test-182", "group_id": "sokoban:a351d5678e74bcc88ca6a5497c769b4c50d611ab43b0ff69ab54d2fbf9300196", "split": "test", "board": {"height": 9, "width": 7, "floors": [10, 11, 18, 23, 24, 25, 26, 30, 31, 32, 33, 36, 37, 38, 39, 40, 43, 45, 46, 47, 50, 51, 52, 53, 54], "goals": [32, 51, 53]}, "initial_state": {"player": 26, "boxes": [31, 38, 52]}, "generator_seed": 20261416, "rules_version": "sokoban-v1", "render": {"theme": "classic", "tile_size": 44}, "language": "zh", "max_steps": 200} +{"level_id": "sokoban-test-184", "group_id": "sokoban:ba8e2d2e2a381575604e2c69c91b9bd474cc49a27dabb9811c3344aa9837f9d9", "split": "test", "board": {"height": 6, "width": 9, "floors": [10, 12, 13, 14, 15, 16, 19, 20, 21, 22, 23, 24, 25, 28, 29, 30, 31, 32, 33, 34, 37, 38, 39, 42, 43], "goals": [32, 34]}, "initial_state": {"player": 15, "boxes": [22, 33]}, "generator_seed": 20261418, "rules_version": "sokoban-v1", "render": {"theme": "classic", "tile_size": 44}, "language": "zh", "max_steps": 200} +{"level_id": "sokoban-test-185", "group_id": "sokoban:1df91755aeb64bb5dfcea0658fcdb451e1b74e01a3d906fda186010b768b8d93", "split": "test", "board": {"height": 9, "width": 6, "floors": [8, 9, 15, 16, 19, 20, 22, 25, 26, 27, 28, 31, 32, 33, 34, 37, 38, 39, 40, 43, 44, 45, 46], "goals": [27, 37]}, "initial_state": {"player": 46, "boxes": [26, 32]}, "generator_seed": 20261419, "rules_version": "sokoban-v1", "render": {"theme": "warm", "tile_size": 44}, "language": "en", "max_steps": 200} +{"level_id": "sokoban-test-186", "group_id": "sokoban:8d41a74331e5866020c8da1124f96deb3e20902b45649cb7fa4454299a10bf72", "split": "test", "board": {"height": 9, "width": 8, "floors": [9, 10, 11, 12, 13, 14, 17, 19, 20, 21, 22, 27, 29, 30, 36, 37, 38, 43, 44, 51, 59], "goals": [11, 21]}, "initial_state": {"player": 17, "boxes": [10, 29]}, "generator_seed": 20261420, "rules_version": "sokoban-v1", "render": {"theme": "warm", "tile_size": 44}, "language": "zh", "max_steps": 200} +{"level_id": "sokoban-test-190", "group_id": "sokoban:53112234e29ebee0fd07a8cc1835f67c0e809cb71bfc0062814fb8648ae27f43", "split": "test", "board": {"height": 9, "width": 9, "floors": [10, 11, 12, 13, 19, 20, 21, 22, 23, 28, 29, 30, 31, 32, 37, 38, 39, 40, 41, 42, 46, 47, 48, 51, 56, 57, 58, 59, 65, 67], "goals": [30, 31, 41]}, "initial_state": {"player": 22, "boxes": [30, 31, 39]}, "generator_seed": 20261425, "rules_version": "sokoban-v1", "render": {"theme": "warm", "tile_size": 44}, "language": "zh", "max_steps": 200} +{"level_id": "sokoban-test-191", "group_id": "sokoban:5eb128d5b5661ad04a0276287c3b1a0ef8ea983e923b556438b6f5d52b8f948e", "split": "test", "board": {"height": 7, "width": 8, "floors": [9, 10, 11, 12, 13, 14, 17, 18, 19, 20, 21, 22, 25, 26, 27, 28, 29, 30, 33, 34, 35, 36, 37, 38, 41, 42, 43, 44, 45, 46], "goals": [17, 22, 38]}, "initial_state": {"player": 20, "boxes": [21, 27, 38]}, "generator_seed": 20261426, "rules_version": "sokoban-v1", "render": {"theme": "classic", "tile_size": 44}, "language": "en", "max_steps": 200} +{"level_id": "sokoban-test-193", "group_id": "sokoban:c85b78b909ee8af1f41aea182dff6339692ec1bc19bfb53f4012a0bc3543d68b", "split": "test", "board": {"height": 6, "width": 7, "floors": [8, 9, 10, 11, 12, 15, 16, 17, 18, 19, 22, 23, 24, 25, 26, 29, 30, 31, 32, 33], "goals": [22, 26]}, "initial_state": {"player": 26, "boxes": [16, 18]}, "generator_seed": 20261428, "rules_version": "sokoban-v1", "render": {"theme": "warm", "tile_size": 44}, "language": "en", "max_steps": 200} +{"level_id": "sokoban-test-195", "group_id": "sokoban:381c8448a57dd70c3f4bf7fc04c2db93ae9ecc4f8935dd16e413aba25120b7bd", "split": "test", "board": {"height": 6, "width": 6, "floors": [7, 8, 9, 10, 13, 14, 15, 16, 19, 20, 21, 22, 25, 26, 27, 28], "goals": [15, 19, 21]}, "initial_state": {"player": 25, "boxes": [13, 14, 21]}, "generator_seed": 20261430, "rules_version": "sokoban-v1", "render": {"theme": "warm", "tile_size": 44}, "language": "en", "max_steps": 200} diff --git a/s1a/agents/_sokoban.py b/s1a/agents/_sokoban.py new file mode 100644 index 0000000..b9da05c --- /dev/null +++ b/s1a/agents/_sokoban.py @@ -0,0 +1,148 @@ +# Adapted from Liuziyu77/Valen, commit c96c4f736c6928d2cb1166bc105a24cebdfcdc40. +# Apache-2.0; see THIRD_PARTY_LICENSES.md. Removed unused hashing/clone helpers; formatted. +"""Pure, deterministic rules. No renderer, policy or solver dependencies.""" + +from dataclasses import dataclass +from typing import NamedTuple + +ACTIONS = ("up", "down", "left", "right") +DELTAS = ((-1, 0), (1, 0), (0, -1), (0, 1)) + + +class State(NamedTuple): + player: int + boxes: tuple + + def to_dict(self): + return {"player": self.player, "boxes": list(self.boxes)} + + @classmethod + def from_dict(cls, value): + return cls(int(value["player"]), tuple(sorted(value["boxes"]))) + + +@dataclass(frozen=True) +class Board: + height: int + width: int + floors: frozenset + goals: frozenset + + def __post_init__(self): + if self.height < 3 or self.width < 3 or not self.goals or not self.goals <= self.floors: + raise ValueError("Invalid board dimensions or goals") + if any(not 0 <= p < self.height * self.width for p in self.floors): + raise ValueError("Floor outside board") + if any(p // self.width in (0, self.height - 1) or p % self.width in (0, self.width - 1) for p in self.floors): + raise ValueError("Board must have an enclosing wall") + + def neighbor(self, position, action): + dr, dc = DELTAS[ACTIONS.index(action)] + r, c = divmod(position, self.width) + r, c = r + dr, c + dc + return r * self.width + c if 0 <= r < self.height and 0 <= c < self.width else -1 + + def validate(self, state): + if ( + state.player not in self.floors + or state.player in state.boxes + or len(state.boxes) != len(self.goals) + or len(set(state.boxes)) != len(state.boxes) + or not set(state.boxes) <= self.floors + or tuple(sorted(state.boxes)) != state.boxes + ): + raise ValueError("Invalid player or boxes") + return state + + def solved(self, state): + return frozenset(state.boxes) == self.goals + + def to_dict(self): + return {"height": self.height, "width": self.width, "floors": sorted(self.floors), "goals": sorted(self.goals)} + + @classmethod + def from_dict(cls, value): + return cls(value["height"], value["width"], frozenset(value["floors"]), frozenset(value["goals"])) + + def ascii(self, state=None): + rows = [] + for r in range(self.height): + row = [] + for c in range(self.width): + p = r * self.width + c + char = "#" if p not in self.floors else "." if p in self.goals else " " + if state is not None: + if p in state.boxes: + char = "*" if p in self.goals else "$" + if p == state.player: + char = "+" if p in self.goals else "@" + row.append(char) + rows.append("".join(row)) + return rows + + +def parse_ascii(text): + rows = text.splitlines() if isinstance(text, str) else text + if not rows or len(set(map(len, rows))) != 1: + raise ValueError("ASCII board must be rectangular") + width = len(rows[0]) + floors, goals, boxes, players = set(), set(), [], [] + for r, row in enumerate(rows): + for c, char in enumerate(row): + p = r * width + c + if char not in "# .$*@+": + raise ValueError("Unknown tile: " + char) + if char != "#": + floors.add(p) + if char in ".*+": + goals.add(p) + if char in "$*": + boxes.append(p) + if char in "@+": + players.append(p) + if len(players) != 1: + raise ValueError("Exactly one player required") + board = Board(len(rows), width, frozenset(floors), frozenset(goals)) + return board, board.validate(State(players[0], tuple(sorted(boxes)))) + + +def transition(board, state, action): + """Invalid moves are no-ops; returns (state, moved, pushed).""" + if action not in ACTIONS: + raise ValueError("Unknown action: " + str(action)) + dest = board.neighbor(state.player, action) + if dest not in board.floors: + return state, False, False + if dest not in state.boxes: + return State(dest, state.boxes), True, False + beyond = board.neighbor(dest, action) + if beyond not in board.floors or beyond in state.boxes: + return state, False, False + return State(dest, tuple(sorted((set(state.boxes) - {dest}) | {beyond}))), True, True + + +class SokobanEnv: + def __init__(self, board, initial_state, max_steps=200): + if max_steps < 1: + raise ValueError("max_steps must be positive") + self.board, self.initial_state, self.max_steps = board, board.validate(initial_state), max_steps + self.reset() + + def reset(self): + self.state, self.steps, self.pushes = self.initial_state, 0, 0 + return self.state + + def step(self, action): + if self.board.solved(self.state) or self.steps >= self.max_steps: + raise RuntimeError("Episode has terminated") + self.state, moved, pushed = transition(self.board, self.state, action) + self.steps += 1 + self.pushes += int(pushed) + solved = self.board.solved(self.state) + return self.state, { + "moved": moved, + "pushed": pushed, + "solved": solved, + "terminated": solved, + "truncated": not solved and self.steps >= self.max_steps, + } diff --git a/s1a/agents/_sokoban_render.py b/s1a/agents/_sokoban_render.py new file mode 100644 index 0000000..18c634e --- /dev/null +++ b/s1a/agents/_sokoban_render.py @@ -0,0 +1,60 @@ +# Adapted from Liuziyu77/Valen, commit c96c4f736c6928d2cb1166bc105a24cebdfcdc40. +# Apache-2.0; see THIRD_PARTY_LICENSES.md. Drawing logic unchanged; formatted. +"""Deterministic vector-like tiles rendered into PNGs with Pillow.""" + +from PIL import Image, ImageDraw + +THEMES = { + "classic": { + "wall": "#475569", + "floor": "#f8fafc", + "line": "#d5dce5", + "goal": "#0f766e", + "box": "#d99b45", + "player": "#2563eb", + }, + "warm": { + "wall": "#625555", + "floor": "#fff9ed", + "line": "#e6ddcb", + "goal": "#007f73", + "box": "#d89655", + "player": "#7345ba", + }, +} + + +def render(board, state, theme="classic", tile_size=44): + board.validate(state) + if theme not in THEMES or not 24 <= tile_size <= 128: + raise ValueError("Unknown theme or tile size outside 24..128") + colors = THEMES[theme] + size = tile_size + image = Image.new("RGB", (board.width * size, board.height * size), colors["wall"]) + draw = ImageDraw.Draw(image) + for p in sorted(board.floors): + r, c = divmod(p, board.width) + x, y = c * size, r * size + draw.rectangle((x, y, x + size - 1, y + size - 1), fill=colors["floor"], outline=colors["line"]) + if p in board.goals: + pad = max(2, size // 14) + draw.ellipse( + (x + pad, y + pad, x + size - pad - 1, y + size - pad - 1), + outline=colors["goal"], + width=max(2, size // 14), + ) + draw.ellipse((x + size * 0.42, y + size * 0.42, x + size * 0.58, y + size * 0.58), fill=colors["goal"]) + if p in state.boxes: + pad = size // 5 + rect = (x + pad, y + pad, x + size - pad, y + size - pad) + draw.rectangle(rect, fill=colors["box"], outline="#603d24", width=2) + draw.line((rect[0] + 3, rect[1] + 3, rect[2] - 3, rect[3] - 3), fill="#603d24", width=2) + draw.line((rect[0] + 3, rect[3] - 3, rect[2] - 3, rect[1] + 3), fill="#603d24", width=2) + if p == state.player: + pad = size // 4 + draw.ellipse( + (x + pad, y + pad, x + size - pad, y + size - pad), fill=colors["player"], outline="#172033", width=2 + ) + for eye in (0.40, 0.58): + draw.ellipse((x + size * eye, y + size * 0.40, x + size * eye + 2, y + size * 0.40 + 2), fill="white") + return image diff --git a/s1a/agents/sokoban.py b/s1a/agents/sokoban.py new file mode 100644 index 0000000..a3771f0 --- /dev/null +++ b/s1a/agents/sokoban.py @@ -0,0 +1,173 @@ +"""Sokoban on Valen's selected 100 evaluation levels: text boards, or the same boards rendered per move.""" + +from __future__ import annotations + +import argparse +import contextlib +import io +import json +from pathlib import Path +from typing import Any, Callable + +from s1a.agents._sokoban import ACTIONS, Board, State, SokobanEnv as Game +from s1a.decision_models.types import Image +from s1a.env import Env +from s1a.jobs import Episode +from s1a.spec import Budget, Series, ToolAgentSpec + +LEVELS = Path(__file__).with_name("_data") / "sokoban_levels.jsonl" +RULES = ( + "Push every box onto a goal. Board legend: # wall, space floor, . goal, @ player, " + "$ box, * box on goal, + player on goal. Up/down change row; left/right change column. " + "You can push one box into an empty floor or goal, but cannot pull or push two boxes. " + "All four directions are offered; blocked moves consume a step. Avoid trapping boxes away from goals." +) +# Valen evaluation/sokoban/dataset.py at c96c4f73, verbatim: the instructions the preview checkpoint read. +VISUAL_RULES = { + "en": ( + "Legend: dark walls, light floor, green rings are goals, crossed squares are boxes, and the round face " + "is the player. Goal rings remain visible under boxes and the player. Move one cell up, down, left or " + "right. You can push one box, but cannot pull, pass through walls, or push two boxes together. Put " + "every box on a goal using the fewest total moves. An invalid move consumes a step without changing " + "the board." + ), + "zh": ( + "图例：深色墙，浅色地板，绿色圆环为目标，带叉方块为箱子，圆脸为玩家；箱子和玩家下方的目标环仍可见。" + "每次向上下左右移动一格，可以推动一个箱子，不能拉箱、穿墙或连推两个箱子。" + "请用最少移动步数将所有箱子推到目标。无效动作消耗一步但不改变棋盘。" + ), +} +DIRECTION_NAMES = { + "en": dict(zip(ACTIONS, ("Up", "Down", "Left", "Right"))), + "zh": dict(zip(ACTIONS, ("向上", "向下", "向左", "向右"))), +} + + +def load_levels() -> list[dict[str, Any]]: + return [json.loads(line) for line in LEVELS.read_text(encoding="utf-8").splitlines()] + + +def require_renderer() -> Callable[..., Any]: + """Valen's ``render``, imported only on the visual path so text runs need no Pillow.""" + try: + from s1a.agents._sokoban_render import render + except ImportError as exc: + raise ValueError(f"sokoban --visual needs Pillow ({exc}); uv sync --extra visual") from exc + return render + + +class SokobanEnv: + def __init__(self, level: dict[str, Any], max_steps: int = 200, *, visual: bool = False) -> None: + self.level_id = level["level_id"] + self.visual = visual + self.language = level.get("language") or "en" + render = level.get("render") or {} + self.theme, self.tile_size = render.get("theme", "classic"), render.get("tile_size", 44) + if visual and self.language not in VISUAL_RULES: + raise ValueError(f"unknown level language {self.language!r}") + self._render = require_renderer() if visual else None + self.game = Game( + Board.from_dict(level["board"]), + State.from_dict(level["initial_state"]), + max_steps=min(max_steps, level["max_steps"]), + ) + + async def reset(self) -> None: + self.game.reset() + + async def observe(self) -> dict[str, Any]: + if not self.visual: # the ASCII board never rides with the picture: the checkpoint read images only + return { + "board": "\n".join(self.game.board.ascii(self.game.state)), + "steps_remaining": self.game.max_steps - self.game.steps, + } + return { + "steps_remaining": self.game.max_steps - self.game.steps, + "board_size": [self.game.board.width, self.game.board.height], + "render": {"theme": self.theme, "tile_size": self.tile_size}, + "language": self.language, + } + + async def candidates(self) -> dict[str, str]: + if self.done: + return {} + return dict(DIRECTION_NAMES[self.language]) if self.visual else {action: f"Move {action}" for action in ACTIONS} + + def rules(self) -> str: + return VISUAL_RULES[self.language] if self.visual else RULES + + def requires_images(self) -> bool: + return self.visual + + async def images(self) -> tuple[Image, ...]: + if self._render is None: + return () + picture = self._render(self.game.board, self.game.state, self.theme, self.tile_size) + buffer = io.BytesIO() + picture.save(buffer, format="PNG") + return (Image(buffer.getvalue(), "image/png"),) + + async def step(self, key: str) -> None: + self.game.step(key) + + @property + def done(self) -> bool: + return self.game.board.solved(self.game.state) or self.game.steps >= self.game.max_steps + + @property + def score(self) -> float: + return float(self.game.board.solved(self.game.state)) + + +def annotate(env: Env, episode: Episode) -> None: + assert isinstance(env, SokobanEnv) + episode.extra["sokoban"] = { + "level_id": env.level_id, + "observation_mode": "visual" if env.visual else "text", + "solved": bool(env.score), + "pushes": env.game.pushes, + "step_limit_reached": env.game.steps >= env.game.max_steps and not env.score, + } + + +def add_flags(parser: argparse.ArgumentParser) -> None: + parser.add_argument( + "--visual", + action="store_true", + help="Valen's rendered board instead of ASCII: one PNG per move, rules per level language; " + "needs an image-reading model and uv sync --extra visual", + ) + + +def make_series(flags: argparse.Namespace) -> Series: + if flags.rethink != "off": + raise ValueError("sokoban uses unassisted decisions; use --rethink off") + if flags.model == "rule": + raise ValueError("sokoban has no rule baseline; use --model random, jev, laya, cua or llm") + if flags.visual and flags.model == "llm": + raise ValueError("sokoban --visual needs a model that reads images; --model llm reads tool text only") + if flags.visual: + require_renderer() + levels = load_levels() + if flags.seed < 0 or flags.seed + flags.episodes > len(levels): + raise ValueError(f"sokoban requires 0 <= --seed and --seed + --episodes <= {len(levels)}") + return Series( + seeds=range(flags.seed, flags.seed + flags.episodes), + env_for=lambda seed: SokobanEnv(levels[seed], max_steps=flags.max_steps, visual=flags.visual), + session=contextlib.nullcontext(), + baseline=None, + annotate=annotate, + ) + + +SPEC = ToolAgentSpec( + name="sokoban", + description=( + "Sokoban on Valen's selected 100 levels as a text board, or with --visual as Valen's rendered image; " + "score is 1 when solved, otherwise 0." + ), + rules=RULES, + budget=Budget(max_steps=200, timeout_s=1200, stall_after=0), + flags=add_flags, + series=make_series, +) diff --git a/s1a/env.py b/s1a/env.py index 3863adb..32509b3 100644 --- a/s1a/env.py +++ b/s1a/env.py @@ -15,6 +15,21 @@ class VisualEnv(Protocol): async def images(self) -> tuple[Image, ...]: ... +@runtime_checkable +class RulesEnv(Protocol): + """Optional per-episode rules that replace the spec's static text in the questions a model reads.""" + + def rules(self) -> str: ... + + +@runtime_checkable +class RequiresImages(Protocol): + """Optional claim that the env's decisions depend on the images it produces: a text-only model is then a + configuration error, refused before the first decision (the ``random`` smoke baseline excepted).""" + + def requires_images(self) -> bool: ... + + class Env(Protocol): async def reset(self) -> None: ... async def observe(self) -> dict[str, Any]: ... # a ``progress`` key, when present, is what the stall check compares diff --git a/s1a/tool/loop.py b/s1a/tool/loop.py index 6c687f4..e2f53a1 100644 --- a/s1a/tool/loop.py +++ b/s1a/tool/loop.py @@ -18,7 +18,7 @@ from openjiuwen.harness.factory import create_deep_agent from s1a.decision_models import DecisionModel -from s1a.env import Env +from s1a.env import Env, RequiresImages from s1a.jobs import Episode, now_iso from s1a.config import HOME from s1a.counting_model import CountingModel, call_digests, usage_known @@ -239,6 +239,14 @@ async def run_episode( raise RuntimeError("bounded rethink needs a decision model; --model llm cannot use it") if limits is not None and rethink_on and spec.budget.stall_after > 0 and refresh_method is None: raise RuntimeError("bounded rethink needs an Env with a refresh callback") + if isinstance(env, RequiresImages) and env.requires_images() and model_name != "random": + # A ``RequiresImages`` environment refuses a text-only backend up front — deciding + # over an observation whose images were stripped would measure a blind run (``random`` + # never reads the observation and stays the offline smoke) + if decision_model is None or not decision_model.supports_images: + raise ValueError( + f"{spec.name}: this environment decides over its images; --model {model_name} reads text only" + ) state = EvalState(max_acts=max_acts) await env.reset() first_view = await view_of(env, state) diff --git a/s1a/tool/models.py b/s1a/tool/models.py index 5f91009..15fdb9c 100644 --- a/s1a/tool/models.py +++ b/s1a/tool/models.py @@ -18,7 +18,7 @@ from openjiuwen.core.foundation.llm import AssistantMessage, AssistantMessageChunk, Model, ToolCall, init_model from s1a.decision_models import DecisionModel, ChoiceQuestion, Observation -from s1a.env import Env, VisualEnv +from s1a.env import Env, RulesEnv, VisualEnv from s1a.recovery import recovery_next_action ACT_TOOL = "act" @@ -143,9 +143,10 @@ async def _decide(self) -> AssistantMessage: request_state["harness_notices"] = list(state.notices) started = time.perf_counter() images = await env.images() if isinstance(env, VisualEnv) else () + rules = env.rules() if isinstance(env, RulesEnv) else self._rules try: decision = await self._decision_model.decide_many( - Observation(request_state, images), {"pick": ChoiceQuestion(offered, rules=self._rules)} + Observation(request_state, images), {"pick": ChoiceQuestion(offered, rules=rules)} ) except BaseError as exc: # any decisions failure ends the episode as BLOCKED, recorded state.error = f"decision failed: {exc}" diff --git a/scripts/smoke.sh b/scripts/smoke.sh index 1ac2452..114afba 100755 --- a/scripts/smoke.sh +++ b/scripts/smoke.sh @@ -9,7 +9,7 @@ set -euo pipefail cd "$(dirname "$0")/.." S1A=${S1A:-uv run --no-sync s1a} PY=${PY:-uv run --no-sync python} -AGENTS="alfworld allrecipes blackjack desktop flights game2048 injection_guard millionaire ticket_router" +AGENTS="alfworld allrecipes blackjack desktop flights game2048 injection_guard millionaire sokoban ticket_router" fail() { echo "smoke: $*" >&2; exit 1; } @@ -18,7 +18,7 @@ listed=$($S1A list | tr -d '\r') [ "$(echo "$listed" | tr '\n' ' ' | sed 's/ $//')" = "$AGENTS" ] || fail "list printed: $listed" echo "list: $AGENTS" -for agent in allrecipes desktop flights game2048 injection_guard millionaire ticket_router; do +for agent in allrecipes desktop flights game2048 injection_guard millionaire sokoban ticket_router; do $S1A run "$agent" --help >/dev/null || fail "run $agent --help failed" echo "help: $agent" done @@ -64,7 +64,7 @@ async def main() -> None: async with create_connected_server_and_client_session(mcp_server.server) as session: result = await session.call_tool("list_agents", {}) rows = result.structuredContent["result"] if result.structuredContent else json.loads(result.content[0].text) - assert not result.isError and len(rows) == 9, rows + assert not result.isError and len(rows) == 10, rows for row in rows: print(f"mcp list_agents: {row['name']} ({row['front']})") diff --git a/tests/fixtures/sokoban_reference.jsonl b/tests/fixtures/sokoban_reference.jsonl new file mode 100644 index 0000000..9363ed0 --- /dev/null +++ b/tests/fixtures/sokoban_reference.jsonl @@ -0,0 +1,100 @@ +{"level_id": "sokoban-test-000", "optimal_distance": 10, "difficulty": "easy", "solution_actions": ["left", "down", "right", "down", "left", "down", "left", "left", "up", "up"], "reference_pushes": 4, "label_method": "reverse_multisource_bfs", "solver_status": "optimal_proven"} +{"level_id": "sokoban-test-001", "optimal_distance": 6, "difficulty": "easy", "solution_actions": ["right", "down", "right", "up", "right", "down"], "reference_pushes": 2, "label_method": "reverse_multisource_bfs", "solver_status": "optimal_proven"} +{"level_id": "sokoban-test-002", "optimal_distance": 14, "difficulty": "medium", "solution_actions": ["left", "down", "down", "right", "right", "left", "left", "up", "right", "right", "right", "up", "right", "down"], "reference_pushes": 6, "label_method": "reverse_multisource_bfs", "solver_status": "optimal_proven"} +{"level_id": "sokoban-test-005", "optimal_distance": 12, "difficulty": "medium", "solution_actions": ["right", "down", "right", "down", "down", "down", "left", "left", "left", "up", "right", "right"], "reference_pushes": 2, "label_method": "reverse_multisource_bfs", "solver_status": "optimal_proven"} +{"level_id": "sokoban-test-006", "optimal_distance": 16, "difficulty": "medium", "solution_actions": ["up", "right", "down", "down", "right", "down", "left", "down", "left", "up", "down", "left", "left", "down", "right", "right"], "reference_pushes": 6, "label_method": "reverse_multisource_bfs", "solver_status": "optimal_proven"} +{"level_id": "sokoban-test-010", "optimal_distance": 8, "difficulty": "easy", "solution_actions": ["left", "left", "up", "left", "left", "up", "left", "down"], "reference_pushes": 3, "label_method": "reverse_multisource_bfs", "solver_status": "optimal_proven"} +{"level_id": "sokoban-test-011", "optimal_distance": 9, "difficulty": "easy", "solution_actions": ["up", "up", "up", "right", "right", "up", "right", "up", "left"], "reference_pushes": 5, "label_method": "reverse_multisource_bfs", "solver_status": "optimal_proven"} +{"level_id": "sokoban-test-012", "optimal_distance": 12, "difficulty": "medium", "solution_actions": ["up", "right", "right", "right", "up", "up", "left", "down", "down", "left", "down", "right"], "reference_pushes": 4, "label_method": "reverse_multisource_bfs", "solver_status": "optimal_proven"} +{"level_id": "sokoban-test-014", "optimal_distance": 12, "difficulty": "medium", "solution_actions": ["up", "right", "right", "right", "right", "down", "left", "left", "down", "left", "up", "up"], "reference_pushes": 4, "label_method": "reverse_multisource_bfs", "solver_status": "optimal_proven"} +{"level_id": "sokoban-test-016", "optimal_distance": 15, "difficulty": "medium", "solution_actions": ["up", "left", "up", "up", "right", "right", "left", "left", "down", "right", "right", "right", "up", "right", "down"], "reference_pushes": 7, "label_method": "reverse_multisource_bfs", "solver_status": "optimal_proven"} +{"level_id": "sokoban-test-020", "optimal_distance": 10, "difficulty": "easy", "solution_actions": ["up", "right", "up", "up", "up", "right", "right", "down", "down", "down"], "reference_pushes": 4, "label_method": "reverse_multisource_bfs", "solver_status": "optimal_proven"} +{"level_id": "sokoban-test-021", "optimal_distance": 6, "difficulty": "easy", "solution_actions": ["left", "left", "up", "right", "right", "right"], "reference_pushes": 3, "label_method": "reverse_multisource_bfs", "solver_status": "optimal_proven"} +{"level_id": "sokoban-test-023", "optimal_distance": 11, "difficulty": "medium", "solution_actions": ["down", "down", "right", "up", "left", "up", "right", "right", "right", "down", "left"], "reference_pushes": 5, "label_method": "reverse_multisource_bfs", "solver_status": "optimal_proven"} +{"level_id": "sokoban-test-024", "optimal_distance": 23, "difficulty": "medium", "solution_actions": ["down", "right", "down", "left", "up", "left", "down", "left", "down", "down", "right", "right", "up", "left", "up", "up", "right", "right", "down", "left", "left", "down", "left"], "reference_pushes": 7, "label_method": "reverse_multisource_bfs", "solver_status": "optimal_proven"} +{"level_id": "sokoban-test-025", "optimal_distance": 11, "difficulty": "medium", "solution_actions": ["up", "left", "left", "down", "down", "right", "right", "right", "down", "right", "up"], "reference_pushes": 4, "label_method": "reverse_multisource_bfs", "solver_status": "optimal_proven"} +{"level_id": "sokoban-test-026", "optimal_distance": 11, "difficulty": "medium", "solution_actions": ["up", "up", "right", "up", "right", "down", "right", "down", "left", "left", "left"], "reference_pushes": 4, "label_method": "reverse_multisource_bfs", "solver_status": "optimal_proven"} +{"level_id": "sokoban-test-030", "optimal_distance": 7, "difficulty": "easy", "solution_actions": ["right", "down", "up", "right", "right", "up", "left"], "reference_pushes": 2, "label_method": "reverse_multisource_bfs", "solver_status": "optimal_proven"} +{"level_id": "sokoban-test-031", "optimal_distance": 10, "difficulty": "easy", "solution_actions": ["up", "right", "right", "down", "right", "right", "down", "down", "left", "left"], "reference_pushes": 2, "label_method": "reverse_multisource_bfs", "solver_status": "optimal_proven"} +{"level_id": "sokoban-test-033", "optimal_distance": 11, "difficulty": "medium", "solution_actions": ["left", "left", "left", "up", "right", "up", "right", "up", "up", "left", "down"], "reference_pushes": 2, "label_method": "reverse_multisource_bfs", "solver_status": "optimal_proven"} +{"level_id": "sokoban-test-034", "optimal_distance": 16, "difficulty": "medium", "solution_actions": ["up", "up", "left", "left", "up", "up", "right", "down", "down", "up", "right", "right", "down", "down", "left", "left"], "reference_pushes": 5, "label_method": "reverse_multisource_bfs", "solver_status": "optimal_proven"} +{"level_id": "sokoban-test-036", "optimal_distance": 12, "difficulty": "medium", "solution_actions": ["up", "right", "up", "right", "right", "up", "up", "left", "left", "down", "down", "down"], "reference_pushes": 3, "label_method": "reverse_multisource_bfs", "solver_status": "optimal_proven"} +{"level_id": "sokoban-test-041", "optimal_distance": 8, "difficulty": "easy", "solution_actions": ["up", "right", "up", "down", "right", "up", "up", "left"], "reference_pushes": 3, "label_method": "reverse_multisource_bfs", "solver_status": "optimal_proven"} +{"level_id": "sokoban-test-042", "optimal_distance": 12, "difficulty": "medium", "solution_actions": ["left", "left", "down", "right", "down", "left", "left", "up", "left", "up", "right", "right"], "reference_pushes": 7, "label_method": "reverse_multisource_bfs", "solver_status": "optimal_proven"} +{"level_id": "sokoban-test-045", "optimal_distance": 11, "difficulty": "medium", "solution_actions": ["down", "down", "down", "left", "left", "up", "up", "left", "up", "right", "right"], "reference_pushes": 4, "label_method": "reverse_multisource_bfs", "solver_status": "optimal_proven"} +{"level_id": "sokoban-test-050", "optimal_distance": 4, "difficulty": "easy", "solution_actions": ["left", "up", "left", "down"], "reference_pushes": 2, "label_method": "reverse_multisource_bfs", "solver_status": "optimal_proven"} +{"level_id": "sokoban-test-051", "optimal_distance": 3, "difficulty": "easy", "solution_actions": ["up", "up", "right"], "reference_pushes": 2, "label_method": "reverse_multisource_bfs", "solver_status": "optimal_proven"} +{"level_id": "sokoban-test-053", "optimal_distance": 12, "difficulty": "medium", "solution_actions": ["right", "right", "down", "down", "up", "up", "up", "right", "down", "down", "down", "down"], "reference_pushes": 6, "label_method": "reverse_multisource_bfs", "solver_status": "optimal_proven"} +{"level_id": "sokoban-test-055", "optimal_distance": 12, "difficulty": "medium", "solution_actions": ["up", "up", "right", "up", "right", "down", "up", "up", "right", "right", "down", "left"], "reference_pushes": 3, "label_method": "reverse_multisource_bfs", "solver_status": "optimal_proven"} +{"level_id": "sokoban-test-060", "optimal_distance": 8, "difficulty": "easy", "solution_actions": ["down", "down", "down", "down", "right", "right", "up", "up"], "reference_pushes": 2, "label_method": "reverse_multisource_bfs", "solver_status": "optimal_proven"} +{"level_id": "sokoban-test-061", "optimal_distance": 8, "difficulty": "easy", "solution_actions": ["up", "left", "left", "right", "up", "up", "left", "down"], "reference_pushes": 2, "label_method": "reverse_multisource_bfs", "solver_status": "optimal_proven"} +{"level_id": "sokoban-test-064", "optimal_distance": 12, "difficulty": "medium", "solution_actions": ["up", "right", "right", "down", "right", "down", "down", "left", "right", "up", "left", "left"], "reference_pushes": 6, "label_method": "reverse_multisource_bfs", "solver_status": "optimal_proven"} +{"level_id": "sokoban-test-065", "optimal_distance": 12, "difficulty": "medium", "solution_actions": ["down", "down", "right", "right", "right", "right", "up", "up", "left", "down", "down", "down"], "reference_pushes": 3, "label_method": "reverse_multisource_bfs", "solver_status": "optimal_proven"} +{"level_id": "sokoban-test-066", "optimal_distance": 16, "difficulty": "medium", "solution_actions": ["right", "down", "down", "left", "up", "right", "up", "left", "left", "up", "left", "left", "down", "down", "right", "right"], "reference_pushes": 4, "label_method": "reverse_multisource_bfs", "solver_status": "optimal_proven"} +{"level_id": "sokoban-test-070", "optimal_distance": 8, "difficulty": "easy", "solution_actions": ["up", "right", "left", "left", "up", "left", "up", "right"], "reference_pushes": 3, "label_method": "reverse_multisource_bfs", "solver_status": "optimal_proven"} +{"level_id": "sokoban-test-071", "optimal_distance": 5, "difficulty": "easy", "solution_actions": ["up", "right", "right", "up", "left"], "reference_pushes": 1, "label_method": "reverse_multisource_bfs", "solver_status": "optimal_proven"} +{"level_id": "sokoban-test-073", "optimal_distance": 12, "difficulty": "medium", "solution_actions": ["right", "down", "right", "down", "left", "left", "up", "down", "down", "left", "up", "up"], "reference_pushes": 6, "label_method": "reverse_multisource_bfs", "solver_status": "optimal_proven"} +{"level_id": "sokoban-test-074", "optimal_distance": 11, "difficulty": "medium", "solution_actions": ["up", "left", "left", "left", "down", "right", "right", "down", "right", "up", "up"], "reference_pushes": 4, "label_method": "reverse_multisource_bfs", "solver_status": "optimal_proven"} +{"level_id": "sokoban-test-076", "optimal_distance": 17, "difficulty": "medium", "solution_actions": ["right", "down", "down", "down", "right", "up", "up", "up", "right", "right", "down", "left", "left", "down", "left", "up", "up"], "reference_pushes": 7, "label_method": "reverse_multisource_bfs", "solver_status": "optimal_proven"} +{"level_id": "sokoban-test-080", "optimal_distance": 5, "difficulty": "easy", "solution_actions": ["left", "up", "up", "right", "down"], "reference_pushes": 1, "label_method": "reverse_multisource_bfs", "solver_status": "optimal_proven"} +{"level_id": "sokoban-test-081", "optimal_distance": 7, "difficulty": "easy", "solution_actions": ["up", "up", "right", "up", "right", "down", "down"], "reference_pushes": 4, "label_method": "reverse_multisource_bfs", "solver_status": "optimal_proven"} +{"level_id": "sokoban-test-082", "optimal_distance": 12, "difficulty": "medium", "solution_actions": ["left", "down", "up", "left", "down", "down", "up", "right", "right", "down", "left", "left"], "reference_pushes": 5, "label_method": "reverse_multisource_bfs", "solver_status": "optimal_proven"} +{"level_id": "sokoban-test-083", "optimal_distance": 14, "difficulty": "medium", "solution_actions": ["left", "up", "up", "right", "up", "right", "up", "left", "left", "right", "down", "down", "left", "up"], "reference_pushes": 4, "label_method": "reverse_multisource_bfs", "solver_status": "optimal_proven"} +{"level_id": "sokoban-test-084", "optimal_distance": 25, "difficulty": "medium", "solution_actions": ["down", "down", "down", "left", "left", "left", "left", "up", "right", "right", "right", "up", "up", "left", "left", "down", "right", "down", "right", "up", "down", "left", "left", "left", "left"], "reference_pushes": 9, "label_method": "reverse_multisource_bfs", "solver_status": "optimal_proven"} +{"level_id": "sokoban-test-085", "optimal_distance": 23, "difficulty": "medium", "solution_actions": ["right", "right", "right", "down", "right", "down", "left", "right", "right", "down", "down", "left", "up", "right", "up", "left", "down", "left", "left", "left", "down", "left", "up"], "reference_pushes": 8, "label_method": "reverse_multisource_bfs", "solver_status": "optimal_proven"} +{"level_id": "sokoban-test-086", "optimal_distance": 15, "difficulty": "medium", "solution_actions": ["up", "right", "right", "down", "down", "right", "down", "left", "up", "left", "down", "down", "up", "left", "up"], "reference_pushes": 6, "label_method": "reverse_multisource_bfs", "solver_status": "optimal_proven"} +{"level_id": "sokoban-test-090", "optimal_distance": 8, "difficulty": "easy", "solution_actions": ["right", "down", "up", "up", "up", "up", "right", "down"], "reference_pushes": 2, "label_method": "reverse_multisource_bfs", "solver_status": "optimal_proven"} +{"level_id": "sokoban-test-093", "optimal_distance": 14, "difficulty": "medium", "solution_actions": ["up", "up", "left", "left", "down", "right", "down", "left", "right", "down", "right", "down", "left", "left"], "reference_pushes": 5, "label_method": "reverse_multisource_bfs", "solver_status": "optimal_proven"} +{"level_id": "sokoban-test-094", "optimal_distance": 12, "difficulty": "medium", "solution_actions": ["down", "down", "down", "right", "down", "right", "up", "up", "up", "right", "up", "left"], "reference_pushes": 4, "label_method": "reverse_multisource_bfs", "solver_status": "optimal_proven"} +{"level_id": "sokoban-test-095", "optimal_distance": 17, "difficulty": "medium", "solution_actions": ["up", "left", "left", "left", "up", "up", "right", "up", "right", "right", "up", "up", "left", "down", "up", "left", "down"], "reference_pushes": 4, "label_method": "reverse_multisource_bfs", "solver_status": "optimal_proven"} +{"level_id": "sokoban-test-096", "optimal_distance": 18, "difficulty": "medium", "solution_actions": ["right", "down", "right", "up", "right", "up", "left", "left", "down", "left", "up", "up", "up", "down", "left", "left", "up", "right"], "reference_pushes": 9, "label_method": "reverse_multisource_bfs", "solver_status": "optimal_proven"} +{"level_id": "sokoban-test-100", "optimal_distance": 6, "difficulty": "easy", "solution_actions": ["up", "left", "up", "right", "right", "right"], "reference_pushes": 3, "label_method": "reverse_multisource_bfs", "solver_status": "optimal_proven"} +{"level_id": "sokoban-test-101", "optimal_distance": 3, "difficulty": "easy", "solution_actions": ["up", "up", "up"], "reference_pushes": 3, "label_method": "reverse_multisource_bfs", "solver_status": "optimal_proven"} +{"level_id": "sokoban-test-102", "optimal_distance": 12, "difficulty": "medium", "solution_actions": ["right", "down", "down", "right", "right", "right", "up", "right", "up", "left", "left", "left"], "reference_pushes": 4, "label_method": "reverse_multisource_bfs", "solver_status": "optimal_proven"} +{"level_id": "sokoban-test-103", "optimal_distance": 12, "difficulty": "medium", "solution_actions": ["down", "left", "left", "down", "right", "down", "right", "up", "right", "up", "left", "left"], "reference_pushes": 5, "label_method": "reverse_multisource_bfs", "solver_status": "optimal_proven"} +{"level_id": "sokoban-test-104", "optimal_distance": 15, "difficulty": "medium", "solution_actions": ["down", "down", "down", "right", "up", "left", "up", "up", "right", "up", "left", "up", "left", "up", "right"], "reference_pushes": 3, "label_method": "reverse_multisource_bfs", "solver_status": "optimal_proven"} +{"level_id": "sokoban-test-110", "optimal_distance": 8, "difficulty": "easy", "solution_actions": ["up", "up", "right", "up", "left", "left", "down", "down"], "reference_pushes": 5, "label_method": "reverse_multisource_bfs", "solver_status": "optimal_proven"} +{"level_id": "sokoban-test-111", "optimal_distance": 10, "difficulty": "easy", "solution_actions": ["left", "left", "up", "up", "left", "right", "right", "up", "left", "left"], "reference_pushes": 6, "label_method": "reverse_multisource_bfs", "solver_status": "optimal_proven"} +{"level_id": "sokoban-test-113", "optimal_distance": 11, "difficulty": "medium", "solution_actions": ["right", "down", "down", "down", "up", "up", "up", "right", "down", "down", "down"], "reference_pushes": 6, "label_method": "reverse_multisource_bfs", "solver_status": "optimal_proven"} +{"level_id": "sokoban-test-114", "optimal_distance": 20, "difficulty": "medium", "solution_actions": ["up", "up", "up", "up", "up", "right", "right", "down", "left", "up", "left", "down", "right", "right", "right", "right", "up", "left", "left", "left"], "reference_pushes": 7, "label_method": "reverse_multisource_bfs", "solver_status": "optimal_proven"} +{"level_id": "sokoban-test-121", "optimal_distance": 9, "difficulty": "easy", "solution_actions": ["down", "down", "right", "down", "down", "right", "up", "left", "left"], "reference_pushes": 3, "label_method": "reverse_multisource_bfs", "solver_status": "optimal_proven"} +{"level_id": "sokoban-test-122", "optimal_distance": 17, "difficulty": "medium", "solution_actions": ["down", "left", "down", "down", "down", "left", "up", "up", "right", "up", "left", "down", "down", "left", "down", "left", "up"], "reference_pushes": 5, "label_method": "reverse_multisource_bfs", "solver_status": "optimal_proven"} +{"level_id": "sokoban-test-123", "optimal_distance": 16, "difficulty": "medium", "solution_actions": ["up", "right", "right", "right", "right", "down", "left", "left", "left", "right", "right", "right", "down", "down", "left", "up"], "reference_pushes": 4, "label_method": "reverse_multisource_bfs", "solver_status": "optimal_proven"} +{"level_id": "sokoban-test-124", "optimal_distance": 17, "difficulty": "medium", "solution_actions": ["down", "left", "left", "down", "down", "right", "down", "right", "up", "up", "down", "left", "left", "up", "up", "right", "right"], "reference_pushes": 4, "label_method": "reverse_multisource_bfs", "solver_status": "optimal_proven"} +{"level_id": "sokoban-test-125", "optimal_distance": 12, "difficulty": "medium", "solution_actions": ["up", "right", "down", "down", "right", "right", "right", "right", "right", "down", "left", "left"], "reference_pushes": 3, "label_method": "reverse_multisource_bfs", "solver_status": "optimal_proven"} +{"level_id": "sokoban-test-126", "optimal_distance": 12, "difficulty": "medium", "solution_actions": ["left", "left", "up", "left", "up", "left", "down", "right", "down", "right", "down", "left"], "reference_pushes": 5, "label_method": "reverse_multisource_bfs", "solver_status": "optimal_proven"} +{"level_id": "sokoban-test-130", "optimal_distance": 7, "difficulty": "easy", "solution_actions": ["right", "right", "right", "up", "left", "left", "up"], "reference_pushes": 3, "label_method": "reverse_multisource_bfs", "solver_status": "optimal_proven"} +{"level_id": "sokoban-test-131", "optimal_distance": 10, "difficulty": "easy", "solution_actions": ["up", "up", "left", "down", "down", "left", "up", "left", "up", "right"], "reference_pushes": 4, "label_method": "reverse_multisource_bfs", "solver_status": "optimal_proven"} +{"level_id": "sokoban-test-133", "optimal_distance": 13, "difficulty": "medium", "solution_actions": ["right", "down", "right", "right", "up", "right", "down", "right", "right", "up", "left", "left", "left"], "reference_pushes": 5, "label_method": "reverse_multisource_bfs", "solver_status": "optimal_proven"} +{"level_id": "sokoban-test-134", "optimal_distance": 11, "difficulty": "medium", "solution_actions": ["down", "down", "down", "left", "left", "left", "left", "up", "right", "right", "right"], "reference_pushes": 4, "label_method": "reverse_multisource_bfs", "solver_status": "optimal_proven"} +{"level_id": "sokoban-test-141", "optimal_distance": 9, "difficulty": "easy", "solution_actions": ["right", "down", "left", "down", "right", "right", "left", "down", "right"], "reference_pushes": 4, "label_method": "reverse_multisource_bfs", "solver_status": "optimal_proven"} +{"level_id": "sokoban-test-142", "optimal_distance": 11, "difficulty": "medium", "solution_actions": ["left", "down", "down", "right", "down", "up", "left", "up", "up", "right", "down"], "reference_pushes": 2, "label_method": "reverse_multisource_bfs", "solver_status": "optimal_proven"} +{"level_id": "sokoban-test-143", "optimal_distance": 18, "difficulty": "medium", "solution_actions": ["left", "up", "up", "left", "down", "down", "down", "up", "up", "right", "right", "up", "right", "down", "down", "down", "left", "down"], "reference_pushes": 8, "label_method": "reverse_multisource_bfs", "solver_status": "optimal_proven"} +{"level_id": "sokoban-test-145", "optimal_distance": 14, "difficulty": "medium", "solution_actions": ["down", "down", "down", "down", "down", "right", "right", "up", "up", "left", "down", "left", "up", "up"], "reference_pushes": 4, "label_method": "reverse_multisource_bfs", "solver_status": "optimal_proven"} +{"level_id": "sokoban-test-150", "optimal_distance": 4, "difficulty": "easy", "solution_actions": ["left", "left", "left", "down"], "reference_pushes": 1, "label_method": "reverse_multisource_bfs", "solver_status": "optimal_proven"} +{"level_id": "sokoban-test-151", "optimal_distance": 7, "difficulty": "easy", "solution_actions": ["up", "up", "right", "right", "right", "down", "left"], "reference_pushes": 2, "label_method": "reverse_multisource_bfs", "solver_status": "optimal_proven"} +{"level_id": "sokoban-test-152", "optimal_distance": 26, "difficulty": "medium", "solution_actions": ["up", "right", "up", "up", "left", "down", "down", "down", "down", "right", "down", "left", "up", "left", "down", "up", "left", "up", "right", "down", "right", "right", "up", "up", "left", "down"], "reference_pushes": 8, "label_method": "reverse_multisource_bfs", "solver_status": "optimal_proven"} +{"level_id": "sokoban-test-153", "optimal_distance": 12, "difficulty": "medium", "solution_actions": ["down", "left", "down", "right", "down", "right", "up", "up", "down", "right", "up", "up"], "reference_pushes": 5, "label_method": "reverse_multisource_bfs", "solver_status": "optimal_proven"} +{"level_id": "sokoban-test-154", "optimal_distance": 16, "difficulty": "medium", "solution_actions": ["down", "left", "down", "down", "right", "right", "up", "right", "down", "right", "right", "up", "up", "left", "left", "left"], "reference_pushes": 6, "label_method": "reverse_multisource_bfs", "solver_status": "optimal_proven"} +{"level_id": "sokoban-test-155", "optimal_distance": 11, "difficulty": "medium", "solution_actions": ["up", "up", "left", "left", "down", "down", "right", "up", "right", "up", "left"], "reference_pushes": 3, "label_method": "reverse_multisource_bfs", "solver_status": "optimal_proven"} +{"level_id": "sokoban-test-156", "optimal_distance": 13, "difficulty": "medium", "solution_actions": ["left", "down", "down", "up", "up", "left", "left", "down", "right", "down", "right", "down", "down"], "reference_pushes": 5, "label_method": "reverse_multisource_bfs", "solver_status": "optimal_proven"} +{"level_id": "sokoban-test-160", "optimal_distance": 8, "difficulty": "easy", "solution_actions": ["left", "up", "right", "left", "left", "up", "right", "right"], "reference_pushes": 3, "label_method": "reverse_multisource_bfs", "solver_status": "optimal_proven"} +{"level_id": "sokoban-test-161", "optimal_distance": 6, "difficulty": "easy", "solution_actions": ["up", "left", "left", "left", "down", "right"], "reference_pushes": 1, "label_method": "reverse_multisource_bfs", "solver_status": "optimal_proven"} +{"level_id": "sokoban-test-162", "optimal_distance": 25, "difficulty": "medium", "solution_actions": ["left", "left", "up", "up", "right", "down", "up", "right", "down", "left", "down", "left", "down", "down", "right", "right", "right", "up", "left", "left", "up", "right", "up", "right", "down"], "reference_pushes": 9, "label_method": "reverse_multisource_bfs", "solver_status": "optimal_proven"} +{"level_id": "sokoban-test-163", "optimal_distance": 11, "difficulty": "medium", "solution_actions": ["down", "down", "down", "left", "left", "left", "up", "left", "up", "right", "right"], "reference_pushes": 3, "label_method": "reverse_multisource_bfs", "solver_status": "optimal_proven"} +{"level_id": "sokoban-test-164", "optimal_distance": 14, "difficulty": "medium", "solution_actions": ["up", "up", "left", "down", "right", "down", "left", "left", "up", "left", "left", "right", "up", "left"], "reference_pushes": 6, "label_method": "reverse_multisource_bfs", "solver_status": "optimal_proven"} +{"level_id": "sokoban-test-166", "optimal_distance": 13, "difficulty": "medium", "solution_actions": ["up", "up", "up", "up", "left", "left", "left", "down", "down", "left", "down", "right", "right"], "reference_pushes": 4, "label_method": "reverse_multisource_bfs", "solver_status": "optimal_proven"} +{"level_id": "sokoban-test-170", "optimal_distance": 10, "difficulty": "easy", "solution_actions": ["up", "up", "down", "down", "right", "right", "up", "right", "up", "left"], "reference_pushes": 3, "label_method": "reverse_multisource_bfs", "solver_status": "optimal_proven"} +{"level_id": "sokoban-test-172", "optimal_distance": 13, "difficulty": "medium", "solution_actions": ["left", "down", "down", "right", "up", "down", "right", "right", "right", "right", "down", "right", "up"], "reference_pushes": 4, "label_method": "reverse_multisource_bfs", "solver_status": "optimal_proven"} +{"level_id": "sokoban-test-175", "optimal_distance": 11, "difficulty": "medium", "solution_actions": ["up", "up", "up", "up", "left", "left", "up", "left", "down", "right", "down"], "reference_pushes": 3, "label_method": "reverse_multisource_bfs", "solver_status": "optimal_proven"} +{"level_id": "sokoban-test-176", "optimal_distance": 14, "difficulty": "medium", "solution_actions": ["up", "up", "left", "left", "left", "down", "down", "down", "right", "right", "right", "down", "right", "up"], "reference_pushes": 4, "label_method": "reverse_multisource_bfs", "solver_status": "optimal_proven"} +{"level_id": "sokoban-test-180", "optimal_distance": 7, "difficulty": "easy", "solution_actions": ["up", "left", "left", "up", "up", "down", "right"], "reference_pushes": 2, "label_method": "reverse_multisource_bfs", "solver_status": "optimal_proven"} +{"level_id": "sokoban-test-181", "optimal_distance": 8, "difficulty": "easy", "solution_actions": ["up", "up", "right", "down", "down", "down", "down", "right"], "reference_pushes": 2, "label_method": "reverse_multisource_bfs", "solver_status": "optimal_proven"} +{"level_id": "sokoban-test-182", "optimal_distance": 22, "difficulty": "medium", "solution_actions": ["left", "left", "left", "down", "right", "up", "right", "down", "down", "right", "down", "down", "left", "left", "up", "up", "left", "up", "right", "down", "right", "down"], "reference_pushes": 7, "label_method": "reverse_multisource_bfs", "solver_status": "optimal_proven"} +{"level_id": "sokoban-test-184", "optimal_distance": 12, "difficulty": "medium", "solution_actions": ["down", "left", "down", "right", "up", "up", "left", "left", "down", "left", "down", "right"], "reference_pushes": 3, "label_method": "reverse_multisource_bfs", "solver_status": "optimal_proven"} +{"level_id": "sokoban-test-185", "optimal_distance": 11, "difficulty": "medium", "solution_actions": ["up", "left", "left", "left", "up", "up", "right", "down", "right", "down", "left"], "reference_pushes": 3, "label_method": "reverse_multisource_bfs", "solver_status": "optimal_proven"} +{"level_id": "sokoban-test-186", "optimal_distance": 15, "difficulty": "medium", "solution_actions": ["up", "right", "right", "down", "right", "right", "up", "left", "down", "right", "right", "down", "down", "left", "up"], "reference_pushes": 4, "label_method": "reverse_multisource_bfs", "solver_status": "optimal_proven"} +{"level_id": "sokoban-test-190", "optimal_distance": 6, "difficulty": "easy", "solution_actions": ["left", "left", "down", "down", "right", "right"], "reference_pushes": 2, "label_method": "reverse_multisource_bfs", "solver_status": "optimal_proven"} +{"level_id": "sokoban-test-191", "optimal_distance": 8, "difficulty": "easy", "solution_actions": ["right", "down", "left", "left", "left", "down", "left", "up"], "reference_pushes": 4, "label_method": "reverse_multisource_bfs", "solver_status": "optimal_proven"} +{"level_id": "sokoban-test-193", "optimal_distance": 13, "difficulty": "medium", "solution_actions": ["up", "up", "left", "down", "up", "left", "left", "down", "right", "down", "left", "right", "right"], "reference_pushes": 4, "label_method": "reverse_multisource_bfs", "solver_status": "optimal_proven"} +{"level_id": "sokoban-test-195", "optimal_distance": 11, "difficulty": "medium", "solution_actions": ["right", "right", "right", "up", "up", "up", "left", "left", "left", "down", "right"], "reference_pushes": 2, "label_method": "reverse_multisource_bfs", "solver_status": "optimal_proven"} diff --git a/tests/test_agents_sokoban.py b/tests/test_agents_sokoban.py new file mode 100644 index 0000000..30366d7 --- /dev/null +++ b/tests/test_agents_sokoban.py @@ -0,0 +1,280 @@ +"""Sokoban transitions, level selection, the offline agent loop, and the visual mode.""" + +import json +import struct +import sys +from pathlib import Path +from unittest import IsolatedAsyncioTestCase, TestCase, skipUnless +from unittest.mock import patch + +from s1a.agents import sokoban +from s1a.agents._sokoban import parse_ascii +from s1a.decision_models import RandomModel, RuleModel, ScriptedModel +from s1a.run import started_runner +from s1a.tool import loop, series + +try: + from s1a.agents import _sokoban_render +except ImportError: # the visual extra is not installed + _sokoban_render = None + + +def level(text, max_steps=200, **fields): + board, state = parse_ascii(text) + row = {"level_id": "fixture", "board": board.to_dict(), "initial_state": state.to_dict(), "max_steps": max_steps} + row.update(fields) + return row + + +SIMPLE = "#####\n#@$.#\n#####" + + +class TextOnlyModel(ScriptedModel): + """The stand-in for every text-only backend — jev, clm, laya, laya-served, Cua Nano, Cua 4B in text modality — + whose `supports_images` is False; the real classes need their extras, and the guard reads only the attribute.""" + + supports_images = False + + +def png_size(data): + """The width and height out of a PNG's IHDR chunk.""" + assert data[:8] == b"\x89PNG\r\n\x1a\n" + return struct.unpack(">II", data[16:24]) + + +class TestSokoban(IsolatedAsyncioTestCase): + async def test_push_win_reset_and_all_four_candidates(self): + env = sokoban.SokobanEnv(level(SIMPLE)) + await env.reset() + self.assertEqual(set(await env.candidates()), {"up", "down", "left", "right"}) + before = await env.observe() + await env.step("up") # Wall moves consume the budget. + self.assertEqual((await env.observe())["board"], before["board"]) + self.assertEqual(env.game.steps, 1) + await env.step("right") + self.assertTrue(env.done) + self.assertEqual(env.score, 1) + self.assertEqual(await env.candidates(), {}) + with self.assertRaises(RuntimeError): + await env.step("right") + await env.reset() + self.assertEqual(await env.observe(), before) + self.assertEqual(env.score, 0) + + async def test_cannot_push_two_boxes_or_pull(self): + env = sokoban.SokobanEnv(level("########\n# @$$..#\n########")) + await env.reset() + boxes = env.game.state.boxes + await env.step("right") + self.assertEqual(env.game.state.boxes, boxes) + await env.step("left") + self.assertEqual(env.game.state.boxes, boxes) + + async def test_budget_termination_and_unknown_action(self): + env = sokoban.SokobanEnv(level(SIMPLE, max_steps=1)) + await env.reset() + with self.assertRaises(ValueError): + await env.step("undo") + self.assertEqual(env.game.steps, 0) + await env.step("up") + self.assertTrue(env.done) + self.assertEqual(env.score, 0) + self.assertEqual(await env.candidates(), {}) + + async def test_offline_deepagent_completes_and_records_text_board(self): + env = sokoban.SokobanEnv(level(SIMPLE)) + async with started_runner(): + episode = await loop.run_episode( + sokoban.SPEC, + env, + model_name="rule", + seed=0, + chat=None, + decision_model=RuleModel("test-right", lambda state, candidates: "right"), + rethink_on=False, + max_acts=3, + timeout_s=30, + prices=None, + log=False, + ) + sokoban.annotate(env, episode) + self.assertIsNone(episode.error) + self.assertEqual((episode.score, episode.steps), (1, 1)) + self.assertEqual(episode.extra["sokoban"]["observation_mode"], "text") + self.assertIn("board", episode.views[0]["state"]) + self.assertNotIn("solution", json.dumps(episode.views)) + + +class TestSokobanLevels(TestCase): + def flags(self, *extra): + return series.parser(sokoban.SPEC).parse_args( + ["--model", "random", "--rethink", "off", "--episodes", "1", *extra] + ) + + def test_level_selection_does_not_wrap_and_honors_budget(self): + run = sokoban.make_series(self.flags("--seed", "99", "--max-steps", "7")) + env = run.env_for(99) + self.assertEqual(env.game.max_steps, 7) + for args in [ + ("--seed", "-1"), + ("--seed", "100"), + ("--seed", "99", "--episodes", "2"), + ("--rethink", "on"), + ("--model", "rule"), + ]: + with self.subTest(args=args), self.assertRaises(ValueError): + sokoban.make_series(self.flags(*args)) + + def test_all_imported_levels_replay_reference_solutions(self): + # Evaluator-only references never enter the agent's observations or baseline. + path = Path(__file__).parent / "fixtures/sokoban_reference.jsonl" + references = {r["level_id"]: r for r in map(json.loads, path.read_text().splitlines())} + levels = sokoban.load_levels() + self.assertEqual(len(levels), 100) + self.assertEqual(set(references), {r["level_id"] for r in levels}) + for row in levels: + game = sokoban.SokobanEnv(row).game + for action in references[row["level_id"]]["solution_actions"]: + game.step(action) + self.assertTrue(game.board.solved(game.state), row["level_id"]) + self.assertEqual(game.steps, references[row["level_id"]]["optimal_distance"]) + + +@skipUnless(_sokoban_render, "the visual extra (Pillow)") +class TestSokobanVisual(IsolatedAsyncioTestCase): + async def test_the_picture_replaces_the_board_and_follows_the_level(self): + env = sokoban.SokobanEnv(level(SIMPLE, language="zh", render={"theme": "warm", "tile_size": 24}), visual=True) + await env.reset() + state = await env.observe() + self.assertNotIn("board", state) + self.assertEqual(state["board_size"], [5, 3]) + self.assertEqual(state["render"], {"theme": "warm", "tile_size": 24}) + self.assertEqual(state["language"], "zh") + self.assertEqual(await env.candidates(), {"up": "向上", "down": "向下", "left": "向左", "right": "向右"}) + self.assertEqual(env.rules(), sokoban.VISUAL_RULES["zh"]) + self.assertTrue(env.requires_images()) + (image,) = await env.images() + self.assertEqual(image.media_type, "image/png") + self.assertEqual(png_size(image.data), (5 * 24, 3 * 24)) + + async def test_a_blocked_move_renders_the_same_board_and_a_push_changes_it(self): + env = sokoban.SokobanEnv(level(SIMPLE), visual=True) + await env.reset() + still = (await env.images())[0].data + await env.step("up") # into the wall: the board does not change + self.assertEqual((await env.images())[0].data, still) + await env.step("right") # the winning push + self.assertNotEqual((await env.images())[0].data, still) + + async def test_text_envs_keep_the_spec_rules_and_move_labels(self): + env = sokoban.SokobanEnv(level(SIMPLE, language="zh")) + await env.reset() + self.assertEqual(env.rules(), sokoban.RULES) + self.assertFalse(env.requires_images()) + self.assertEqual( + await env.candidates(), {action: f"Move {action}" for action in ("up", "down", "left", "right")} + ) + self.assertEqual(await env.images(), ()) + + async def test_offline_deepagent_decides_over_the_picture(self): + env = sokoban.SokobanEnv(level(SIMPLE), visual=True) + model = ScriptedModel(choose="right") + async with started_runner(): + episode = await loop.run_episode( + sokoban.SPEC, + env, + model_name="rule", + seed=0, + chat=None, + decision_model=model, + rethink_on=False, + max_acts=3, + timeout_s=30, + prices=None, + log=False, + ) + sokoban.annotate(env, episode) + self.assertIsNone(episode.error) + self.assertEqual((episode.score, episode.steps), (1, 1)) + self.assertEqual(episode.extra["sokoban"]["observation_mode"], "visual") + observation, questions = model.calls[0] + self.assertEqual(len(observation.images), 1) + self.assertEqual(observation.images[0].media_type, "image/png") + self.assertNotIn("board", observation.state) + self.assertEqual(questions["pick"].rules, (sokoban.VISUAL_RULES["en"],)) + + async def test_a_text_only_backend_is_refused_before_any_decision(self): + env = sokoban.SokobanEnv(level(SIMPLE), visual=True) + model = TextOnlyModel(choose="right") # jev, clm, laya, laya-served, Cua Nano, Cua 4B in text modality + with self.assertRaisesRegex(ValueError, "reads text only"): + async with started_runner(): + await loop.run_episode( + sokoban.SPEC, + env, + model_name="jev", + seed=0, + chat=None, + decision_model=model, + rethink_on=False, + max_acts=3, + timeout_s=30, + prices=None, + log=False, + ) + self.assertEqual(model.calls, []) # refused before the backend was ever asked + + async def test_random_stays_the_offline_smoke_exception(self): + env = sokoban.SokobanEnv(level(SIMPLE, max_steps=3), visual=True) + async with started_runner(): + episode = await loop.run_episode( + sokoban.SPEC, + env, + model_name="random", + seed=0, + chat=None, + decision_model=RandomModel(seed=0), + rethink_on=False, + max_acts=3, + timeout_s=30, + prices=None, + log=False, + ) + self.assertIsNone(episode.error) + sokoban.annotate(env, episode) + self.assertEqual(episode.extra["sokoban"]["observation_mode"], "visual") + + async def test_the_reference_solutions_replay_over_rendered_boards(self): + path = Path(__file__).parent / "fixtures/sokoban_reference.jsonl" + references = {r["level_id"]: r["solution_actions"] for r in map(json.loads, path.read_text().splitlines())} + run = sokoban.make_series(self.flags_for_visual()) + self.assertTrue(run.env_for(0).visual) + for row in sokoban.load_levels()[:5]: + env = sokoban.SokobanEnv(row, visual=True) + await env.reset() + first = (await env.images())[0].data + for action in references[env.level_id]: + await env.step(action) + self.assertTrue((await env.images())[0].data.startswith(b"\x89PNG")) + self.assertEqual(env.score, 1.0) + await env.reset() + self.assertEqual((await env.images())[0].data, first) + + @staticmethod + def flags_for_visual(): + return series.parser(sokoban.SPEC).parse_args( + ["--model", "random", "--rethink", "off", "--episodes", "1", "--visual"] + ) + + +class TestVisualFlags(TestCase): + def flags(self, *extra): + return series.parser(sokoban.SPEC).parse_args( + ["--model", "random", "--rethink", "off", "--episodes", "1", *extra] + ) + + def test_visual_needs_an_image_model_and_names_the_missing_extra(self): + with self.assertRaisesRegex(ValueError, "image"): + sokoban.make_series(self.flags("--visual", "--model", "llm")) + with patch.dict(sys.modules, {"s1a.agents._sokoban_render": None}): + with self.assertRaisesRegex(ValueError, "uv sync --extra visual"): + sokoban.make_series(self.flags("--visual")) diff --git a/tests/test_decision_models_cua_four_b.py b/tests/test_decision_models_cua_four_b.py index 09184f0..de099fb 100644 --- a/tests/test_decision_models_cua_four_b.py +++ b/tests/test_decision_models_cua_four_b.py @@ -95,3 +95,10 @@ async def test_text_adapter_uses_state_without_requesting_an_image(self) -> None await model.decide_many(Observation({"goal": "choose B"}), {"pick": ChoiceQuestion({"a": "A", "b": "B"})}) self.assertIn("choose B", scorer.seen[0][1]["ax_tree"]) self.assertIsNone(scorer.seen[0][2]) + + async def test_the_modality_declares_whether_the_model_reads_images(self) -> None: + # The attribute the tool loop's RequiresImages guard reads: text 4B and Nano are refused on a + # visual env, multimodal 4B is not (Nano's False is pinned in test_decision_models_cua.py). + scorer = FakeFourB() + self.assertFalse(CuaFourBModel(scorer, model="test/text", modality="text").supports_images) + self.assertTrue(CuaFourBModel(scorer, model="test/mm", modality="multimodal").supports_images) diff --git a/tests/test_sokoban_render.py b/tests/test_sokoban_render.py new file mode 100644 index 0000000..7a57e58 --- /dev/null +++ b/tests/test_sokoban_render.py @@ -0,0 +1,66 @@ +# coding: utf-8 +"""The ported Valen renderer: sizes, themes, colors and byte-for-byte determinism.""" + +import io +from unittest import TestCase, skipUnless + +from s1a.agents._sokoban import parse_ascii, transition + +try: + from s1a.agents import _sokoban_render +except ImportError: # the visual extra is not installed + _sokoban_render = None + +SIMPLE = "#####\n#@$.#\n#####" # 5x3: walls around, player left of one box on a goal tile's row +ROOMY = "######\n#@$. #\n######" # the same, one empty floor tile right of the goal +WALL = (0x47, 0x55, 0x69) # classic #475569 +FLOOR = (0xF8, 0xFA, 0xFC) # classic #f8fafc +BOX = (0xD9, 0x9B, 0x45) # classic #d99b45 +PLAYER = (0x25, 0x63, 0xEB) # classic #2563eb +GOAL = (0x0F, 0x76, 0x6E) # classic #0f766e +WARM_WALL = (0x62, 0x55, 0x55) # warm #625555 + + +def png_bytes(image) -> bytes: + buffer = io.BytesIO() + image.save(buffer, format="PNG") + return buffer.getvalue() + + +@skipUnless(_sokoban_render, "the visual extra (Pillow)") +class TestRender(TestCase): + def render(self, text=SIMPLE, **kwargs): + board, state = parse_ascii(text) + return _sokoban_render.render(board, state, **kwargs) + + def test_size_is_tile_times_the_board(self): + image = self.render(tile_size=24) + self.assertEqual(image.size, (5 * 24, 3 * 24)) + + def test_tile_colors(self): + image = self.render(ROOMY, tile_size=24).convert("RGB") + self.assertEqual(image.getpixel((0, 0)), WALL) # a wall corner + self.assertEqual(image.getpixel((4 * 24 + 12, 24 + 12)), FLOOR) # the empty floor tile centre + self.assertEqual(image.getpixel((2 * 24 + 9, 24 + 12)), BOX) # the box fill, off its cross + self.assertEqual(image.getpixel((24 + 12, 24 + 12)), PLAYER) # the player circle + self.assertEqual(image.getpixel((3 * 24 + 12, 24 + 12)), GOAL) # the goal dot + + def test_warm_theme_walls(self): + image = self.render(theme="warm", tile_size=24).convert("RGB") + self.assertEqual(image.getpixel((0, 0)), WARM_WALL) + + def test_unknown_theme_or_tile_size_is_rejected(self): + with self.assertRaises(ValueError): + self.render(theme="neon") + for size in (23, 129): + with self.assertRaises(ValueError): + self.render(tile_size=size) + + def test_the_same_state_renders_the_same_png_and_a_move_changes_it(self): + board, state = parse_ascii(SIMPLE) + before = png_bytes(_sokoban_render.render(board, state, tile_size=32)) + again = png_bytes(_sokoban_render.render(board, state, tile_size=32)) + moved, _, _ = transition(board, state, "right") + after = png_bytes(_sokoban_render.render(board, moved, tile_size=32)) + self.assertEqual(before, again) + self.assertNotEqual(before, after) diff --git a/tests/test_tool_loop.py b/tests/test_tool_loop.py index a6f0de6..141044b 100644 --- a/tests/test_tool_loop.py +++ b/tests/test_tool_loop.py @@ -308,6 +308,37 @@ def _refusing() -> JevModel: return JevModel(ScriptedTransport(error=error)) +class RulesCountingEnv(CountingEnv): + """CountingEnv plus ``rules()``: the per-episode text a RulesEnv swaps into the question.""" + + def __init__(self, text: str) -> None: + super().__init__() + self._text = text + + def rules(self) -> str: + return self._text + + +class TestRulesEnv(IsolatedAsyncioTestCase): + async def test_an_env_with_rules_swaps_the_question_text_and_plain_envs_keep_the_spec_rules(self) -> None: + swapping = ScriptedModel(choose="inc") + plain = ScriptedModel(choose="inc") + bilingual = ToolDecisionModel( + RulesCountingEnv("count to three, in Chinese"), + EvalState(), + rules="count to three", + decision_model=swapping, + fallback=None, + ) + unilingual = ToolDecisionModel( + CountingEnv(), EvalState(), rules="count to three", decision_model=plain, fallback=None + ) + await bilingual._decide() + await unilingual._decide() + self.assertEqual(swapping.calls[0][1]["pick"].rules, ("count to three, in Chinese",)) + self.assertEqual(plain.calls[0][1]["pick"].rules, ("count to three",)) + + class TestEpisodeThroughTheAgent(IsolatedAsyncioTestCase): """Episodes through ``create_deep_agent`` and the Runner, offline: a rule in the slot, no chat model.""" diff --git a/uv.lock b/uv.lock index 32932c7..bb4ea60 100644 --- a/uv.lock +++ b/uv.lock @@ -758,7 +758,7 @@ resolution-markers = [ "python_full_version < '3.12' and sys_platform != 'emscripten' and sys_platform != 'win32'", ] dependencies = [ - { name = "numpy", version = "2.4.6", source = { registry = "https://pypi.org/simple" } }, + { name = "numpy", version = "2.4.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.12'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/58/01/1253e6698a07380cd31a736d248a3f2a50a7c88779a1813da27503cadc2a/contourpy-1.3.3.tar.gz", hash = "sha256:083e12155b210502d0bca491432bb04d56dc3432f95a979b429f2848c3dbe880", size = 13466174, upload-time = "2025-07-26T12:03:12.549Z" } wheels = [ @@ -848,7 +848,7 @@ resolution-markers = [ "python_full_version >= '3.12' and python_full_version < '3.14' and sys_platform != 'emscripten' and sys_platform != 'win32'", ] dependencies = [ - { name = "numpy", version = "2.5.3", source = { registry = "https://pypi.org/simple" } }, + { name = "numpy", version = "2.5.3", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.12'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/83/5a/a55177dd22553a277388e8a1b3220e92de91bacb28356cdc73caa240121d/contourpy-1.4.0.tar.gz", hash = "sha256:20156f5a1ac4f8ce02656e39a61e82164a3d359796dc8026f75b062783d500e1", size = 13323726, upload-time = "2026-09-11T19:05:05.808Z" } wheels = [ @@ -1009,7 +1009,7 @@ name = "cuda-bindings" version = "13.4.2" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "cuda-pathfinder" }, + { name = "cuda-pathfinder", marker = "sys_platform != 'emscripten' and sys_platform != 'win32'" }, ] wheels = [ { url = "https://files.pythonhosted.org/packages/19/6f/e00ffcebcad6405a2326a52612eff98240a67c637dfed516e61b881e2db3/cuda_bindings-13.4.2-cp311-cp311-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:fc0a18f65b26459cd3c49c0250a44b419cdf790cffcb096ff4d46a1973f6f4b7", size = 6484069, upload-time = "2026-09-17T22:05:52.594Z" }, @@ -1042,43 +1042,43 @@ wheels = [ [package.optional-dependencies] cublas = [ - { name = "nvidia-cublas", marker = "platform_machine == 'aarch64' or platform_machine == 'x86_64'" }, - { name = "nvidia-cuda-nvrtc", marker = "platform_machine == 'aarch64' or platform_machine == 'x86_64'" }, + { name = "nvidia-cublas", marker = "(platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "nvidia-cuda-nvrtc", marker = "(platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, ] cudart = [ - { name = "nvidia-cuda-runtime", marker = "platform_machine == 'aarch64' or platform_machine == 'x86_64'" }, + { name = "nvidia-cuda-runtime", marker = "(platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, ] cufft = [ - { name = "nvidia-cufft", marker = "platform_machine == 'aarch64' or platform_machine == 'x86_64'" }, - { name = "nvidia-nvjitlink", marker = "platform_machine == 'aarch64' or platform_machine == 'x86_64'" }, + { name = "nvidia-cufft", marker = "(platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "nvidia-nvjitlink", marker = "(platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, ] cufile = [ - { name = "nvidia-cufile", marker = "platform_machine == 'aarch64' or platform_machine == 'x86_64'" }, + { name = "nvidia-cufile", marker = "(platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, ] cupti = [ - { name = "nvidia-cuda-cupti", marker = "platform_machine == 'aarch64' or platform_machine == 'x86_64'" }, + { name = "nvidia-cuda-cupti", marker = "(platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, ] curand = [ - { name = "nvidia-curand", marker = "platform_machine == 'aarch64' or platform_machine == 'x86_64'" }, + { name = "nvidia-curand", marker = "(platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, ] cusolver = [ - { name = "nvidia-cublas", marker = "platform_machine == 'aarch64' or platform_machine == 'x86_64'" }, - { name = "nvidia-cusolver", marker = "platform_machine == 'aarch64' or platform_machine == 'x86_64'" }, - { name = "nvidia-cusparse", marker = "platform_machine == 'aarch64' or platform_machine == 'x86_64'" }, - { name = "nvidia-nvjitlink", marker = "platform_machine == 'aarch64' or platform_machine == 'x86_64'" }, + { name = "nvidia-cublas", marker = "(platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "nvidia-cusolver", marker = "(platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "nvidia-cusparse", marker = "(platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "nvidia-nvjitlink", marker = "(platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, ] cusparse = [ - { name = "nvidia-cusparse", marker = "platform_machine == 'aarch64' or platform_machine == 'x86_64'" }, - { name = "nvidia-nvjitlink", marker = "platform_machine == 'aarch64' or platform_machine == 'x86_64'" }, + { name = "nvidia-cusparse", marker = "(platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "nvidia-nvjitlink", marker = "(platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, ] nvjitlink = [ - { name = "nvidia-nvjitlink", marker = "platform_machine == 'aarch64' or platform_machine == 'x86_64'" }, + { name = "nvidia-nvjitlink", marker = "(platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, ] nvrtc = [ - { name = "nvidia-cuda-nvrtc", marker = "platform_machine == 'aarch64' or platform_machine == 'x86_64'" }, + { name = "nvidia-cuda-nvrtc", marker = "(platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, ] nvtx = [ - { name = "nvidia-nvtx", marker = "platform_machine == 'aarch64' or platform_machine == 'x86_64'" }, + { name = "nvidia-nvtx", marker = "(platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, ] [[package]] @@ -1765,8 +1765,8 @@ name = "httpcore2" version = "2.13.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "h11" }, - { name = "truststore" }, + { name = "h11", marker = "sys_platform != 'emscripten'" }, + { name = "truststore", marker = "sys_platform != 'emscripten'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/15/8c/e925b1c92018abb3a1863ce1549d76d2381e334d21d65d4ac8f65dabd78a/httpcore2-2.13.0.tar.gz", hash = "sha256:2adc8be4fb285fbcd6d894298db3b52c177e74b6674eda3a76bd36be3292a3db", size = 67740, upload-time = "2026-09-14T14:18:04.717Z" } wheels = [ @@ -1857,7 +1857,7 @@ name = "importlib-metadata" version = "9.0.1" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "zipp" }, + { name = "zipp", marker = "python_full_version < '3.12'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/6f/7e/1e7e8dc30634b93ebb3d58a3dea569ad146e656218d3960ab04f62047b29/importlib_metadata-9.0.1.tar.gz", hash = "sha256:ab830580bc0ef3db61ce8fae716389e5462b67e033018bab6d8f80ef17172f99", size = 59124, upload-time = "2026-08-28T15:30:34.646Z" } wheels = [ @@ -3142,7 +3142,7 @@ name = "nvidia-cublas" version = "13.1.1.3" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "nvidia-cuda-nvrtc" }, + { name = "nvidia-cuda-nvrtc", marker = "sys_platform != 'emscripten' and sys_platform != 'win32'" }, ] wheels = [ { url = "https://files.pythonhosted.org/packages/a7/a1/0bd24ee8c8d03adac032fd2909426a00c88f8c57961b1277ded97f91119f/nvidia_cublas-13.1.1.3-py3-none-manylinux_2_27_aarch64.whl", hash = "sha256:b7a210458267ac818974c53038fbec2e969d5c99f305ab15c72522fa9f001dd5", size = 542848918, upload-time = "2026-04-08T18:46:22.985Z" }, @@ -3181,7 +3181,7 @@ name = "nvidia-cudnn-cu13" version = "9.24.0.43" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "nvidia-cublas" }, + { name = "nvidia-cublas", marker = "sys_platform != 'emscripten' and sys_platform != 'win32'" }, ] wheels = [ { url = "https://files.pythonhosted.org/packages/ca/30/7c257e3d5cb4fecb147b93895c66e29c93f8e76d74b45bb418ff0587c4ec/nvidia_cudnn_cu13-9.24.0.43-py3-none-manylinux_2_27_aarch64.whl", hash = "sha256:a6812a554a1ff0413e9c52b84c26c050380649ab9615f9c16bded368ce9f421f", size = 650976863, upload-time = "2026-07-02T16:23:39.248Z" }, @@ -3193,7 +3193,7 @@ name = "nvidia-cufft" version = "12.0.0.61" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "nvidia-nvjitlink" }, + { name = "nvidia-nvjitlink", marker = "sys_platform != 'emscripten' and sys_platform != 'win32'" }, ] wheels = [ { url = "https://files.pythonhosted.org/packages/8b/ae/f417a75c0259e85c1d2f83ca4e960289a5f814ed0cea74d18c353d3e989d/nvidia_cufft-12.0.0.61-py3-none-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:2708c852ef8cd89d1d2068bdbece0aa188813a0c934db3779b9b1faa8442e5f5", size = 214053554, upload-time = "2025-09-04T08:31:38.196Z" }, @@ -3223,9 +3223,9 @@ name = "nvidia-cusolver" version = "12.0.4.66" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "nvidia-cublas" }, - { name = "nvidia-cusparse" }, - { name = "nvidia-nvjitlink" }, + { name = "nvidia-cublas", marker = "sys_platform != 'emscripten' and sys_platform != 'win32'" }, + { name = "nvidia-cusparse", marker = "sys_platform != 'emscripten' and sys_platform != 'win32'" }, + { name = "nvidia-nvjitlink", marker = "sys_platform != 'emscripten' and sys_platform != 'win32'" }, ] wheels = [ { url = "https://files.pythonhosted.org/packages/c8/c3/b30c9e935fc01e3da443ec0116ed1b2a009bb867f5324d3f2d7e533e776b/nvidia_cusolver-12.0.4.66-py3-none-manylinux_2_27_aarch64.whl", hash = "sha256:02c2457eaa9e39de20f880f4bd8820e6a1cfb9f9a34f820eb12a155aa5bc92d2", size = 223467760, upload-time = "2025-09-04T08:33:04.222Z" }, @@ -3237,7 +3237,7 @@ name = "nvidia-cusparse" version = "12.6.3.3" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "nvidia-nvjitlink" }, + { name = "nvidia-nvjitlink", marker = "sys_platform != 'emscripten' and sys_platform != 'win32'" }, ] wheels = [ { url = "https://files.pythonhosted.org/packages/f8/94/5c26f33738ae35276672f12615a64bd008ed5be6d1ebcb23579285d960a9/nvidia_cusparse-12.6.3.3-py3-none-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:80bcc4662f23f1054ee334a15c72b8940402975e0eab63178fc7e670aa59472c", size = 162155568, upload-time = "2025-09-04T08:33:42.864Z" }, @@ -4914,8 +4914,8 @@ name = "secretstorage" version = "3.5.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "cryptography" }, - { name = "jeepney" }, + { name = "cryptography", marker = "sys_platform != 'emscripten' and sys_platform != 'win32'" }, + { name = "jeepney", marker = "sys_platform != 'emscripten' and sys_platform != 'win32'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/1c/03/e834bcd866f2f8a49a85eaff47340affa3bfa391ee9912a952a1faa68c7b/secretstorage-3.5.0.tar.gz", hash = "sha256:f04b8e4689cbce351744d5537bf6b1329c6fc68f91fa666f60a380edddcd11be", size = 19884, upload-time = "2025-11-23T19:02:53.191Z" } wheels = [ @@ -5283,6 +5283,9 @@ report = [ { name = "pillow" }, { name = "playwright" }, ] +visual = [ + { name = "pillow" }, +] [package.metadata] requires-dist = [ @@ -5304,6 +5307,7 @@ requires-dist = [ { name = "pillow", marker = "extra == 'cua-four-b'", specifier = ">=10" }, { name = "pillow", marker = "extra == 'omnijev'" }, { name = "pillow", marker = "extra == 'report'", specifier = ">=10" }, + { name = "pillow", marker = "extra == 'visual'", specifier = ">=10" }, { name = "playwright", marker = "extra == 'report'", specifier = ">=1.45" }, { name = "pytest", marker = "extra == 'dev'", specifier = ">=8" }, { name = "pytest-asyncio", marker = "extra == 'dev'", specifier = ">=0.24" }, @@ -5326,7 +5330,7 @@ requires-dist = [ { name = "ty", marker = "extra == 'dev'", specifier = ">=0.0.83" }, { name = "werkzeug", marker = "extra == 'alfworld-visual'", specifier = "==2.0.3" }, ] -provides-extras = ["blackjack", "alfworld", "alfworld-visual", "report", "laya", "cua", "omnijev", "cua-four-b", "dev"] +provides-extras = ["blackjack", "alfworld", "alfworld-visual", "report", "visual", "laya", "cua", "omnijev", "cua-four-b", "dev"] [[package]] name = "tatsu"