From abbcc7ffe7f216a2995a4f0d49c5c18a05b73025 Mon Sep 17 00:00:00 2001
From: "Mr.Dong" <1042542469@qq.com>
Date: Wed, 14 Oct 2026 21:44:56 +0800
Subject: [PATCH 1/8] d3: expose media capability and pinned decode wrappers

---
 src/models/d3/engine.py | 64 +++++++++++++++++++++++++++++++++++++++--
 1 file changed, 61 insertions(+), 3 deletions(-)

diff --git a/src/models/d3/engine.py b/src/models/d3/engine.py
index 9327306c..97e1e883 100644
--- a/src/models/d3/engine.py
+++ b/src/models/d3/engine.py
@@ -186,6 +186,23 @@ def __init__(
         self._lock = threading.RLock()
         self._closed = False
 
+    @property
+    def image_unavailable(self) -> str | None:
+        """Why the loaded checkpoint cannot read images; ``None`` when it can.
+
+        Mirrors the runtime's own flag: it is set when the vision tower
+        weights, the patch embedding or the image processor are missing, and
+        the upstream server refuses non-empty ``images`` on it.
+        """
+
+        return self._image_unavailable
+
+    @property
+    def video_unavailable(self) -> str | None:
+        """Why the loaded checkpoint cannot read videos; ``None`` when it can."""
+
+        return self._video_unavailable
+
     @property
     def modalities(self) -> list[str]:
         """The loaded model's input modalities, in upstream /health order."""
@@ -193,14 +210,55 @@ def modalities(self) -> list[str]:
         names = ["text", "image"] if self._image_unavailable is None else ["text"]
         return names + ["video"] if self._video_unavailable is None else names
 
-    def prepare(self, state: Any, questions: dict[str, Any]) -> Any:
+    def load_images(self, images: list[Any]) -> list[Any]:
+        """Decode one request's images (RGB PIL) through strict loading.
+
+        The pinned runtime is the per-item validation authority — data URL
+        shape, PNG/JPEG/WEBP format, the 8,000,000-byte and 16,000,000-pixel
+        budgets — and its ``ValueError`` rejections name the item
+        (``images[i]: ...``). The empty short-circuit and the capability gate
+        with the upstream server's message live in the worker's decode stage,
+        mirroring ``d3_server.py``'s ``decode_images``.
+        """
+
+        with self._lock:
+            if self._closed:
+                raise RuntimeError("D3 executor is closed")
+            return self.model.load_images(images, strict=True)
+
+    def load_videos(self, videos: list[Any]) -> list[Any]:
+        """Decode and sample one request's videos through strict loading.
+
+        Per-item validation (MP4/WebM/QuickTime/Matroska data URLs, the
+        32,000,000-byte, 300-second and per-frame budgets, 2 fps sampling of
+        4-32 frames) is the pinned runtime's; rejections name the item
+        (``videos[i]: ...``).
+        """
+
+        with self._lock:
+            if self._closed:
+                raise RuntimeError("D3 executor is closed")
+            return self.model.load_videos(videos, strict=True)
+
+    def prepare(
+        self,
+        state: Any,
+        questions: dict[str, Any],
+        images: Any = None,
+        videos: Any = None,
+    ) -> Any:
         """Validate and tokenize one request; per-question failures land in
-        ``prepared.errors`` instead of raising (envelope problems raise)."""
+        ``prepared.errors`` instead of raising (envelope problems raise).
+
+        ``images``/``videos`` are already-decoded media from the worker's
+        decode stage; the runtime picks the video, image or text path by
+        their truthiness, exactly as the upstream server passes them.
+        """
 
         with self._lock:
             if self._closed:
                 raise RuntimeError("D3 executor is closed")
-            return self.model.prepare(state, questions)
+            return self.model.prepare(state, questions, images, videos)
 
     def run(self, prepared: Any) -> tuple[dict[str, list[float]], int]:
         """Probabilities per runnable question and the input token count."""

From 6419c062640f6d7644241eca3bfc838132c4215a Mon Sep 17 00:00:00 2001
From: "Mr.Dong" <1042542469@qq.com>
Date: Wed, 14 Oct 2026 21:44:56 +0800
Subject: [PATCH 2/8] d3: open the media pipeline from admission to decode

Admission now validates media list shape only and passes non-empty
images/videos through, mirroring upstream system_one's decode order
(empty short-circuit, capability gate, strict per-item decode) with
ValueError mapped to a plain-detail 422. MAX_BODY moves to a 64 MiB
slice-scoped cap: upstream sets no total-body limit, so the cap is a
deliberate difference admitting at most one full-size video per
request; decode stays inside the inference lock.
---
 src/frontend/d3.py             |  66 ++++++++--
 src/models/d3/README.md        |  81 ++++++++----
 src/models/d3/protocol.py      |  56 +++++----
 tests/d3/test_protocol.py      |  39 ++++--
 tests/d3/test_worker.py        | 222 +++++++++++++++++++++++++++++++--
 tests/d3/test_worker_server.py |  82 +++++++++++-
 6 files changed, 469 insertions(+), 77 deletions(-)

diff --git a/src/frontend/d3.py b/src/frontend/d3.py
index b9fcae38..fb13e92f 100644
--- a/src/frontend/d3.py
+++ b/src/frontend/d3.py
@@ -7,11 +7,12 @@
 response finishing in models.d3.postprocess.
 
 The error surface has three layers: transport errors follow the shared
-Python-worker discipline; a request with an invalid or over-limit question is
-rejected whole with HTTP 422 before any execution (nothing is truncated); and
-a question whose model output cannot be decoded keeps its HTTP 200 response
-with the failure embedded in that question's answer — plus a worker warning,
-because a silent partial failure is otherwise undetectable.
+Python-worker discipline; a request with an invalid or over-limit question,
+or with media the pinned runtime cannot decode, is rejected whole with HTTP
+422 before any execution (nothing is truncated); and a question whose model
+output cannot be decoded keeps its HTTP 200 response with the failure
+embedded in that question's answer — plus a worker warning, because a silent
+partial failure is otherwise undetectable.
 """
 
 from __future__ import annotations
@@ -21,6 +22,7 @@
 import logging
 import socket
 import threading
+from collections.abc import Callable
 from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
 from typing import Any
 
@@ -31,14 +33,62 @@
 
 LOG = logging.getLogger(__name__)
 
+# Upstream d3_server.py decode_images/decode_videos capability messages,
+# kept verbatim so clients see the reference server's wording.
+TEXT_ONLY_IMAGES_MESSAGE = "This model reads text only; images are not supported."
+TEXT_ONLY_VIDEOS_MESSAGE = "This model does not read videos."
+
+
+def _decode(
+    media: Any, loader: Callable[[list], list], unavailable: str | None, refusal: str
+) -> list:
+    """One media field through the upstream server's decode stage.
+
+    Absent, null and empty media decode to nothing — the upstream server
+    never capability-checks an empty list. A non-empty list against a
+    checkpoint that cannot read it is refused with the upstream message.
+    Otherwise the pinned runtime's strict loading decodes item by item, and
+    its ``ValueError`` rejections reject the whole request with the runtime's
+    message (``images[i]: ...`` / ``videos[i]: ...``) as plain detail.
+    """
+
+    if not media:
+        return []
+    if unavailable is not None:
+        raise RequestError(refusal, 422)
+    try:
+        return loader(media)
+    except ValueError as exc:
+        raise RequestError(str(exc), 422) from exc
+
 
 def decide(raw: bytes, executor: D3Executor) -> dict[str, Any]:
-    """Admit, execute, and finish one complete request."""
+    """Admit, decode, execute, and finish one complete request."""
 
     request = parse_request(parse_body(raw))
     questions = dict(request.questions)
+    # The decode stage mirrors upstream d3_server.py system_one: images then
+    # videos, each through the server's decode_images/decode_videos body
+    # (empty short-circuit, capability gate, strict loading) before prepare.
+    images = _decode(
+        request.images,
+        executor.load_images,
+        executor.image_unavailable,
+        TEXT_ONLY_IMAGES_MESSAGE,
+    )
+    videos = _decode(
+        request.videos,
+        executor.load_videos,
+        executor.video_unavailable,
+        TEXT_ONLY_VIDEOS_MESSAGE,
+    )
     try:
-        prepared = executor.prepare(request.state, questions)
+        if videos:
+            prepared = executor.prepare(request.state, questions, images, videos)
+        elif images:
+            prepared = executor.prepare(request.state, questions, images)
+        else:
+            prepared = executor.prepare(request.state, questions)
     except ValueError as exc:
         # The pinned runtime is the validation authority for what the wire
         # layer cannot foresee (envelope-level question problems); its
@@ -124,7 +174,7 @@ def do_POST(self):
             self.send_json(400, {"detail": "invalid Content-Length"})
             return
         if length < 0 or length > MAX_BODY:
-            self.send_json(413, {"detail": "request exceeds body limit"})
+            self.send_json(413, {"detail": "request exceeds the 64 MiB body limit"})
             return
         if self.headers.get_content_type() != "application/json":
             self.send_json(415, {"detail": "Content-Type must be application/json"})
diff --git a/src/models/d3/README.md b/src/models/d3/README.md
index 40faf986..3ebeff1a 100644
--- a/src/models/d3/README.md
+++ b/src/models/d3/README.md
@@ -1,11 +1,11 @@
 # D3 reference contract
 
-This directory contains the first System1-Omni integration slice for
+This directory contains the System1-Omni integration slice for
 D3 (Decision 3.0, [`vllm-sr/d3`](https://huggingface.co/vllm-sr/d3)). It is a
 reference-serving boundary only: a Python worker that runs the checkpoint's
-own pinned runtime for text state and the three typed question kinds
-(`choice`, `noul`, `score`) on `/v1/systemone`. Native CUDA execution and
-multimodal inputs are not part of this slice.
+own pinned runtime for text, image and video state and the three typed
+question kinds (`choice`, `noul`, `score`) on `/v1/systemone`. Native CUDA
+execution is not part of this slice.
 
 ## Pinned artifacts
 
@@ -51,11 +51,31 @@ upstream `verify` mechanism (`fast` default, `full` via CLI).
   - `noul`: `criteria` is optional and may only describe `"false"` and
     `"true"`;
   - `score`: `criteria` is an ordered list of 2–10 distinct nonempty levels.
-- `images`/`videos` are accepted only as absent, `null` or empty lists;
-  anything else is rejected with HTTP 422. Serving media is stage 2; when it
-  opens, the limits re-align to the upstream per-item budgets (8,000,000
-  bytes per image, 32,000,000 per video) instead of this slice's 8 MiB
-  transport cap.
+- `images`/`videos` are lists of inline base64 data URLs that every
+  question of the request sees. The envelope checks only the list shape —
+  a non-list is rejected with the upstream server's message; per-item
+  decoding and its budgets belong to the pinned runtime's strict loading,
+  whose rejections name the item (`images[i]: ...`, `videos[i]: ...`) and
+  refuse the whole request with HTTP 422:
+  - `images`: PNG, JPEG or WebP data URLs, each at most 8,000,000 decoded
+    bytes (base64 length at most 4·⌈8,000,000/3⌉ ≈ 10.7 MB, re-checked
+    after decoding) and 16,000,000 source pixels;
+  - `videos`: MP4, WebM, QuickTime or Matroska data URLs, each at most
+    32,000,000 bytes (≈ 42.7 MB of base64), 300 seconds and 8,294,400
+    pixels per frame, sampled at 2 fps into 4–32 frames of at most
+    200,704 pixels each, and at most 16,384 video tokens per request.
+
+  These budgets are the pinned `d3_runtime.py` constants of the pinned
+  revision. Media capability follows the loaded checkpoint: when the vision
+  tower weights, the `qwen3_5` backbone or the video processor is missing,
+  non-empty media is refused with the upstream server's messages (`This
+  model reads text only; images are not supported.` / `This model does not
+  read videos.`), while `/health`'s `modalities` reports what the loaded
+  checkpoint reads. Absent, `null` and empty lists take the text path and
+  never trigger the capability check. Decoding runs inside the worker's
+  inference lock — one request at a time covers decode, prepare, run and
+  respond (upstream decodes outside its lock; a deliberate difference
+  recorded with the integration).
 - No token generation: one forward pass per question scores 255 single-token
   answer codes at the last prompt position.
 
@@ -77,8 +97,9 @@ Each answer is built by the pinned runtime from per-option probabilities
   (1.0 for fewer than two options) — the upstream "Decision 2.0 measure",
   not max-p.
 
-`usage.input_tokens` sums each runnable question's prompt tokens;
-`usage.output_tokens` is 0. The pinned package reports no
+`usage.input_tokens` sums each runnable question's prompt tokens — media
+tokens included, as counted by the pinned runtime; `usage.output_tokens`
+is 0. The pinned package reports no
 `max_input_tokens` limit (the manifest records `null`), so
 `/health` reports `max_input_tokens: null` and `max_length_exceeded` cannot
 trigger for it; the admission path stays for packages that set a limit.
@@ -90,15 +111,19 @@ Three layers, matching the upstream server:
 1. Transport follows the shared Python-worker discipline: `400` (unusable
    JSON body, mismatched or invalid `Content-Length`), `408` body timeout,
    `411` without exactly one `Content-Length`, `413` over the slice-scoped
-   8 MiB body limit, `415` non-JSON content type, `503` while another
-   inference holds the lock, `404` unknown routes, `500` anything else
-   (generic detail, no request content in logs).
-2. Before execution, the pinned runtime's per-question validation rejects
-   the whole request with HTTP 422: invalid questions produce
-   `{"invalid_questions": {name: message}}` in request order, and an
+   64 MiB body limit (a deliberate difference from upstream, which sets no
+   total-body limit; sized for one full-budget video plus envelope
+   headroom), `415` non-JSON content type, `503` while another inference
+   holds the lock, `404` unknown routes, `500` anything else (generic
+   detail, no request content in logs).
+2. Before execution, the pinned runtime rejects the whole request with
+   HTTP 422: invalid questions produce
+   `{"invalid_questions": {name: message}}` in request order, an
    over-limit question produces a plain `detail` naming the actual token
-   count and limit — nothing is ever truncated. `max_length_exceeded` wins
-   over invalid questions.
+   count and limit — nothing is ever truncated — and media the runtime
+   cannot decode (strict loading) or cannot read (capability) produces a
+   plain `detail` with the upstream message, item-prefixed where the
+   runtime names it. `max_length_exceeded` wins over invalid questions.
 3. After execution, a question whose probabilities cannot be decoded
    (non-finite, negative, or all zero) keeps its HTTP 200 response with
    `{"type", "error": "invalid_model_output", "message"}` embedded in that
@@ -113,16 +138,20 @@ after loading and warmup; the upstream `authentication` field is dropped
 
 ## Ownership boundary
 
-- `protocol.py` owns the envelope: body parsing, unknown-field and
-  slice-scoped media rejection, question-ID admission.
+- `protocol.py` owns the envelope: body parsing, unknown-field rejection,
+  question-ID admission and the media list-shape check.
 - `preprocess.py` owns the mapping of the pinned runtime's per-question
   preparation errors to the two pre-execution 422 shapes.
 - `engine.py` owns the pinned runtime, the identity pin and the pre-import
   manifest gate (full runtime hashes, no unlisted importables), device
-  state, and warmup.
+  state, warmup, the exposed media-capability flags and the strict-loading
+  wrappers.
 - `postprocess.py` owns the pure answer formulas, response finishing, and
   the embedded-failure warning.
-- `src/frontend/d3.py` owns HTTP lifecycle and orchestration.
+- `src/frontend/d3.py` owns HTTP lifecycle and orchestration, including
+  the media decode stage (empty short-circuit, capability gate with the
+  upstream messages, strict-loading dispatch) that mirrors
+  `d3_server.py`'s `decode_images`/`decode_videos`.
 - The Rust frontend owns transport and forwarding, not model state.
 
 The server-side error surface and envelope naming are recorded as this
@@ -133,7 +162,11 @@ the noncausal full-attention hook inside the pinned runtime is adapted from
 (Apache-2.0), as its module docstring records.
 
 The current tests are CPU-only and do not require model weights, torch or a
-GPU (`PYTHONPATH=src python -m pytest tests/d3 -q`). The real worker path
+GPU (`PYTHONPATH=src python -m pytest tests/d3 -q`); the media cases cover
+the envelope, capability and rejection surface with a contract fake plus
+Pillow-built PNG fixtures — real image/video decoding against the pinned
+runtime is author-side and opt-in, and nothing here claims measured media
+behavior on GPU. The real worker path
 additionally requires the pinned package and a CUDA-capable environment;
 the procedure is [recipe/d3/reference.md](../../recipe/d3/reference.md) and
 measured-run evidence is recorded in the integration pull request, not in
diff --git a/src/models/d3/protocol.py b/src/models/d3/protocol.py
index a314c4f6..33537395 100644
--- a/src/models/d3/protocol.py
+++ b/src/models/d3/protocol.py
@@ -2,9 +2,9 @@
 
 This module deliberately has no torch dependency and validates only the
 request envelope: the five-field System One request, unknown-field rejection,
-and the slice-scoped media refusal. Per-question validation belongs to the
-checkpoint's pinned runtime, whose aggregated rejections ``preprocess`` maps
-to HTTP 422.
+and the list shape of the ``images``/``videos`` fields. Per-question and
+per-item media validation belongs to the checkpoint's pinned runtime, whose
+aggregated rejections ``preprocess`` and the decode stage map to HTTP 422.
 """
 
 from __future__ import annotations
@@ -14,10 +14,11 @@
 from typing import Any
 
 MODEL_NAME = "d3"
-# Slice-scoped transport cap. Upstream d3_server.py sets no total-body limit
-# (its budgets are per image and per video); stage 2 re-aligns this when the
-# media paths open.
-MAX_BODY = 8 * 1024 * 1024
+# Slice-scoped transport cap: one full-budget video (~42.7 MB of base64) plus
+# envelope headroom. Upstream d3_server.py sets no total-body limit (its
+# budgets are per image and per video); this cap is a deliberate difference
+# recorded in the integration PR.
+MAX_BODY = 64 * 1024 * 1024
 REQUEST_FIELDS = {"model", "state", "questions", "images", "videos"}
 
 
@@ -45,12 +46,16 @@ class Request:
 
     ``model`` is accepted and ignored (upstream semantics). ``state`` passes
     through unchanged; the pinned runtime renders it. ``questions`` keeps the
-    request order for response reconstruction.
+    request order for response reconstruction. ``images``/``videos`` pass
+    through as the raw lists (``None`` for absent or null); the worker's
+    decode stage hands them to the pinned runtime's strict loading.
     """
 
     model: Any
     state: Any
     questions: tuple[tuple[str, Any], ...]
+    images: Any = None
+    videos: Any = None
 
 
 def _object(pairs: list[tuple[str, Any]]) -> dict[str, Any]:
@@ -70,7 +75,7 @@ def parse_body(raw: bytes) -> dict[str, Any]:
     """Decode a bounded JSON body and reject ambiguous JSON values."""
 
     if len(raw) > MAX_BODY:
-        raise RequestError("request body exceeds 8 MiB", 413)
+        raise RequestError("request body exceeds 64 MiB", 413)
     try:
         value = json.loads(
             raw.decode("utf-8"),
@@ -88,35 +93,40 @@ def parse_body(raw: bytes) -> dict[str, Any]:
     return value
 
 
-def _media(value: dict[str, Any], field: str) -> None:
-    """Admit an absent, null or empty media list; refuse anything else."""
+def _media(value: dict[str, Any], field: str) -> Any:
+    """Admit a media field as absent, null or a list; pass the list through.
+
+    Only the list shape is checked here, with the upstream server's message:
+    its ``decode_images``/``decode_videos`` reject anything else before any
+    decoding. Absent, null and empty lists all mean "no media of this kind";
+    a non-empty list is not inspected — per-item validation (format, byte and
+    pixel budgets) is the pinned runtime's strict loading.
+    """
 
     media = value.get(field)
     if media is None:
-        return
+        return None
     if not isinstance(media, list):
         raise RequestError(f"{field} must be a list of base64 data URLs")
-    if media:
-        raise RequestError(
-            f"this reference worker serves text state only; non-empty {field}"
-            " are not supported yet (stage 2)"
-        )
+    return media
 
 
 def parse_request(value: dict[str, Any]) -> Request:
     """Validate the system1-omni wire request envelope.
 
     Unknown fields are rejected with upstream's message; ``model`` is not
-    checked (upstream ignores it); per-question content validation is left
-    to the pinned runtime.
+    checked (upstream ignores it); per-question content and per-item media
+    validation are left to the pinned runtime.
     """
 
     unknown = set(value) - REQUEST_FIELDS
     if unknown:
         raise RequestError(f"Unknown request fields: {sorted(unknown)}")
 
-    for field in ("images", "videos"):
-        _media(value, field)
+    # Media shape first, mirroring the upstream decode order: a non-list
+    # images/videos field is refused before question validation runs.
+    images = _media(value, "images")
+    videos = _media(value, "videos")
 
     questions = value.get("questions")
     if not isinstance(questions, dict) or not questions:
@@ -124,4 +134,6 @@ def parse_request(value: dict[str, Any]) -> Request:
     if any(not isinstance(name, str) or not name for name in questions):
         raise RequestError("question IDs must be nonempty strings")
 
-    return Request(value.get("model"), value.get("state"), tuple(questions.items()))
+    return Request(
+        value.get("model"), value.get("state"), tuple(questions.items()), images, videos
+    )
diff --git a/tests/d3/test_protocol.py b/tests/d3/test_protocol.py
index 9295f6d4..0c01f803 100644
--- a/tests/d3/test_protocol.py
+++ b/tests/d3/test_protocol.py
@@ -42,6 +42,7 @@ def test_parse_body_enforces_the_slice_scoped_body_limit():
     with pytest.raises(RequestError) as excinfo:
         parse_body(padding)
     assert excinfo.value.status == 413
+    assert "64 MiB" in str(excinfo.value)
 
 
 def test_parse_rejects_unknown_fields_with_the_upstream_message():
@@ -66,18 +67,34 @@ def test_parse_passes_state_through_in_every_wire_shape():
         assert parsed.state == state
 
 
-def test_parse_admits_empty_media_lists_and_refuses_the_rest():
+def test_parse_admits_media_lists_and_checks_only_the_shape():
     for field in ("images", "videos"):
-        assert parse_request(body(**{field: []})) is not None
-        assert parse_request(body(**{field: None})) is not None
-
-        value = body(**{field: ["data:image/png;base64,AAAA"]})
-        with pytest.raises(RequestError, match="text state only"):
-            parse_request(value)
-
-        value = body(**{field: "data:image/png;base64,AAAA"})
-        with pytest.raises(RequestError, match="must be a list"):
-            parse_request(value)
+        # Absent and null mean "no media of this kind".
+        assert getattr(parse_request(body()), field) is None
+        assert getattr(parse_request(body(**{field: None})), field) is None
+        assert getattr(parse_request(body(**{field: []})), field) == []
+
+        # A non-empty list passes through untouched — order and identity
+        # preserved, no data-URL inspection at the envelope layer.
+        items = ["data:image/png;base64,AAAA", "not a data url", 17]
+        parsed = parse_request(body(**{field: items}))
+        assert getattr(parsed, field) == items
+        assert getattr(parsed, field)[0] is items[0]
+
+        # Anything but a list is refused with the upstream server's message.
+        for bad in ("data:image/png;base64,AAAA", 17, {}, True):
+            with pytest.raises(RequestError) as excinfo:
+                parse_request(body(**{field: bad}))
+            assert excinfo.value.status == 422
+            assert str(excinfo.value) == f"{field} must be a list of base64 data URLs"
+
+
+def test_parse_checks_media_shape_before_questions():
+    # Mirroring the upstream decode order: a non-list images field is
+    # refused even when the questions field is also broken.
+    value = body(images="data:image/png;base64,AAAA", questions={})
+    with pytest.raises(RequestError, match="must be a list"):
+        parse_request(value)
 
 
 def test_parse_rejects_empty_or_malformed_questions():
diff --git a/tests/d3/test_worker.py b/tests/d3/test_worker.py
index d24af71b..3643964a 100644
--- a/tests/d3/test_worker.py
+++ b/tests/d3/test_worker.py
@@ -1,12 +1,15 @@
 """End-to-end decide() tests with a CPU-only FakeD3: no weights, no torch.
 
 FakeD3 mirrors the upstream runtime surface the worker drives —
-prepare/run/respond plus model_name, max_length and modalities — with the
-pinned question-validation rules reimplemented just far enough to produce
-upstream-shaped per-question errors, and answer construction delegated to
-models.d3.postprocess (which locks the pinned formulas).
+prepare/run/respond plus model_name, max_length, the capability flags and
+load_images/load_videos — with the pinned question-validation rules
+reimplemented just far enough to produce upstream-shaped per-question
+errors, and answer construction delegated to models.d3.postprocess (which
+locks the pinned formulas).
 """
 
+import base64
+import binascii
 import json
 import logging
 
@@ -18,6 +21,20 @@
 
 from .test_protocol import body
 
+# Upstream strict-loading budgets (d3_runtime.py), reimplemented only for
+# the fake's rejections: real pixel decode and frame sampling belong to the
+# pinned runtime and stay author-side.
+MAX_IMAGE_BYTES = 8_000_000
+MAX_VIDEO_BYTES = 32_000_000
+
+
+def image_data_url(payload=b"tiny-png-bytes"):
+    return f"data:image/png;base64,{base64.b64encode(payload).decode()}"
+
+
+def video_data_url(payload=b"tiny-mp4-bytes"):
+    return f"data:video/mp4;base64,{base64.b64encode(payload).decode()}"
+
 
 class FakePrepared:
     def __init__(self, keys, questions, errors):
@@ -27,17 +44,114 @@ def __init__(self, keys, questions, errors):
 
 
 class FakeD3:
-    """The pinned runtime's contract, driven without its code."""
+    """The pinned runtime's contract, driven without its code.
+
+    Two shapes, switched by ``media``: a vision-capable checkpoint
+    (``image_unavailable``/``video_unavailable`` are None) and the text-only
+    shape the runtime produces when the vision tower is missing.
+    ``load_images``/``load_videos`` reimplement the data-URL slice of the
+    runtime's strict loading just far enough to raise upstream-shaped
+    rejections; they return ``("image"/"video", payload)`` markers instead
+    of decoded pixels or sampled frames.
+    """
 
     model_name = MODEL_NAME
-    max_length = None
-    modalities = ["text"]
 
-    def __init__(self, probabilities=None, max_length=None):
+    def __init__(self, probabilities=None, max_length=None, media=False):
         self.probabilities = dict(probabilities or {})
         self.max_length = max_length
         self.warmed = False
         self.closed = False
+        self.image_unavailable = (
+            None if media else "the checkpoint has no vision tower (visual.* weights)"
+        )
+        self.video_unavailable = None if media else self.image_unavailable
+        self.modalities = ["text", "image", "video"] if media else ["text"]
+        self.loaded_images = []
+        self.loaded_videos = []
+        self.prepare_calls = []
+
+    # -- upstream strict loading, reduced to the data-URL envelope ---------
+    @staticmethod
+    def _image_payload(value):
+        if not isinstance(value, str) or not value.startswith("data:"):
+            raise ValueError("images must be base64 PNG, JPEG or WebP data URLs")
+        header, separator, encoded = value.partition(",")
+        kind = header.strip().lower()
+        if (
+            not separator
+            or not kind.startswith("data:image/")
+            or not kind.endswith(";base64")
+        ):
+            raise ValueError("a data URL image is data:image/<format>;base64,<data>")
+        if kind[len("data:image/") : -len(";base64")] not in ("png", "jpeg", "jpg", "webp"):
+            raise ValueError("images must be base64 PNG, JPEG or WebP data URLs")
+        if len(encoded) > 4 * -(-MAX_IMAGE_BYTES // 3):
+            raise ValueError(f"each image must be at most {MAX_IMAGE_BYTES:,} bytes")
+        try:
+            payload = base64.b64decode(encoded, validate=True)
+        except binascii.Error as exc:
+            raise ValueError("invalid base64 image data") from exc
+        if len(payload) > MAX_IMAGE_BYTES:
+            raise ValueError(f"each image must be at most {MAX_IMAGE_BYTES:,} bytes")
+        return payload
+
+    @staticmethod
+    def _video_payload(value):
+        header, separator, encoded = value.partition(",")
+        kind = header.strip().lower()
+        if (
+            not separator
+            or not kind.startswith("data:video/")
+            or not kind.endswith(";base64")
+        ):
+            raise ValueError("a data URL video is data:video/<format>;base64,<data>")
+        if kind[len("data:video/") :].split(";")[0] not in (
+            "mp4",
+            "webm",
+            "quicktime",
+            "x-matroska",
+        ):
+            raise ValueError(
+                "videos must be base64 MP4, WebM, QuickTime or Matroska data URLs"
+            )
+        if len(encoded) > 4 * -(-MAX_VIDEO_BYTES // 3):
+            raise ValueError(f"each video must be at most {MAX_VIDEO_BYTES:,} bytes")
+        try:
+            payload = base64.b64decode(encoded, validate=True)
+        except binascii.Error as exc:
+            raise ValueError("invalid base64 video data") from exc
+        if len(payload) > MAX_VIDEO_BYTES:
+            raise ValueError(f"each video must be at most {MAX_VIDEO_BYTES:,} bytes")
+        return payload
+
+    def load_images(self, images):
+        if not images:
+            return []
+        if self.image_unavailable is not None:
+            raise ValueError(f"image inputs are not available: {self.image_unavailable}")
+        self.loaded_images.append(list(images))
+        decoded = []
+        for number, value in enumerate(images):
+            try:
+                decoded.append(("image", self._image_payload(value)))
+            except ValueError as exc:
+                raise ValueError(f"images[{number}]: {exc}") from exc
+        return decoded
+
+    def load_videos(self, videos):
+        if not videos:
+            return []
+        if self.video_unavailable is not None:
+            raise ValueError(f"video inputs are not available: {self.video_unavailable}")
+        self.loaded_videos.append(list(videos))
+        decoded = []
+        for number, value in enumerate(videos):
+            try:
+                decoded.append(("video", self._video_payload(value)))
+            except ValueError as exc:
+                raise ValueError(f"videos[{number}]: {exc}") from exc
+        return decoded
 
     # -- upstream normalize_question, reduced to what the tests exercise ----
     @staticmethod
@@ -74,7 +188,8 @@ def _normalize(question):
     def _tokens(self, state, question):
         return 8 + len(json.dumps(state)) // 8 + len(json.dumps(question)) // 8
 
-    def prepare(self, state, questions):
+    def prepare(self, state, questions, images=None, videos=None):
+        self.prepare_calls.append((state, questions, images, videos))
         errors = {}
         normalized = {}
         for key, question in questions.items():
@@ -251,3 +366,92 @@ def respond(self, prepared, probabilities, tokens):
 
     with pytest.raises(RuntimeError, match="question order"):
         decide(json.dumps(three_kind_body()).encode("utf-8"), ReorderedD3())
+
+
+def test_decide_decodes_images_and_passes_them_to_prepare():
+    executor = FakeD3(media=True)
+    value = three_kind_body()
+    value["images"] = [image_data_url(b"one"), image_data_url(b"two")]
+    result = decide(json.dumps(value).encode("utf-8"), executor)
+
+    state, questions, images, videos = executor.prepare_calls[0]
+    assert images == [("image", b"one"), ("image", b"two")]
+    assert videos is None
+    assert executor.loaded_images == [list(value["images"])]
+    assert executor.loaded_videos == []
+    assert result["model"] == "d3"
+    assert list(result["answers"]) == ["refund", "urgent", "severity"]
+
+
+def test_decide_routes_media_to_the_upstream_prepare_branches():
+    # Videos (with or without images) take the four-argument branch, images
+    # alone the three-argument one, and empty or absent media stays text.
+    executor = FakeD3(media=True)
+    value = three_kind_body()
+    value["images"] = [image_data_url()]
+    value["videos"] = [video_data_url()]
+    decide(json.dumps(value).encode("utf-8"), executor)
+    _, _, images, videos = executor.prepare_calls[0]
+    assert images == [("image", b"tiny-png-bytes")]
+    assert videos == [("video", b"tiny-mp4-bytes")]
+
+    executor = FakeD3(media=True)
+    value = three_kind_body()
+    value["images"] = []
+    value["videos"] = []
+    decide(json.dumps(value).encode("utf-8"), executor)
+    _, _, images, videos = executor.prepare_calls[0]
+    assert images is None and videos is None
+    assert executor.loaded_images == [] and executor.loaded_videos == []
+
+    executor = FakeD3(media=True)
+    decide(json.dumps(three_kind_body()).encode("utf-8"), executor)
+    _, _, images, videos = executor.prepare_calls[0]
+    assert images is None and videos is None
+
+
+def test_decide_refuses_media_a_text_only_checkpoint_cannot_read():
+    value = three_kind_body()
+    value["images"] = [image_data_url()]
+    with pytest.raises(RequestError) as excinfo:
+        decide(json.dumps(value).encode("utf-8"), FakeD3())
+    assert excinfo.value.status == 422
+    assert str(excinfo.value) == "This model reads text only; images are not supported."
+
+    value = three_kind_body()
+    value["videos"] = [video_data_url()]
+    with pytest.raises(RequestError) as excinfo:
+        decide(json.dumps(value).encode("utf-8"), FakeD3())
+    assert excinfo.value.status == 422
+    assert str(excinfo.value) == "This model does not read videos."
+
+    # The gates are independent: an image-capable checkpoint without video
+    # support still refuses videos, after the images have decoded.
+    executor = FakeD3(media=True)
+    executor.video_unavailable = "video inputs need a Qwen3.5 backbone"
+    value = three_kind_body()
+    value["images"] = [image_data_url()]
+    value["videos"] = [video_data_url()]
+    with pytest.raises(RequestError) as excinfo:
+        decide(json.dumps(value).encode("utf-8"), executor)
+    assert str(excinfo.value) == "This model does not read videos."
+    assert executor.prepare_calls == []
+
+
+def test_decide_rejects_undecodable_media_with_the_runtime_message():
+    for field, bad, expected in (
+        ("images", "https://example.com/pic.png", "images must be base64 PNG"),
+        ("images", "data:image/tiff;base64,AAAA", "images must be base64 PNG"),
+        ("images", "data:image/png;base64,!!!!", "invalid base64 image data"),
+        ("images", "data:image/png;base64," + "A" * 11_000_000, "8,000,000 bytes"),
+        ("videos", "data:video/avi;base64,AAAA", "videos must be base64 MP4"),
+        ("videos", "data:video/mp4;base64," + "A" * 43_000_000, "32,000,000 bytes"),
+    ):
+        value = three_kind_body()
+        value[field] = [bad]
+        with pytest.raises(RequestError) as excinfo:
+            decide(json.dumps(value).encode("utf-8"), FakeD3(media=True))
+        assert excinfo.value.status == 422
+        assert excinfo.value.body is None
+        assert str(excinfo.value).startswith(f"{field}[0]: ")
+        assert expected in str(excinfo.value)
diff --git a/tests/d3/test_worker_server.py b/tests/d3/test_worker_server.py
index a4c0f4ec..f7078c7a 100644
--- a/tests/d3/test_worker_server.py
+++ b/tests/d3/test_worker_server.py
@@ -3,19 +3,24 @@
 Starts the real WorkerServer on an ephemeral port with the FakeD3 from
 test_worker: no weights, no torch, no GPU. The 408 slow-client timeout is not
 covered because the handler's 15 s connection timeout is deliberately not
-configurable for tests.
+configurable for tests. Strict video decode (OpenCV against the pinned
+runtime) cannot run here; the media cases exercise the decode stage's
+admission, capability and rejection surface only.
 
     PYTHONPATH=src python -m pytest tests/d3 -q
 """
 
 from __future__ import annotations
 
+import base64
 import json
 import logging
 import threading
 from http.client import HTTPConnection
+from io import BytesIO
 
 import pytest
+from PIL import Image
 
 from frontend.d3 import WorkerServer
 from models.d3.protocol import MAX_BODY
@@ -24,10 +29,26 @@
 from .test_worker import FakeD3, three_kind_body
 
 
+def png_data_url():
+    """One small real PNG as an inline data URL (valen-test precedent)."""
+
+    stream = BytesIO()
+    Image.new("RGB", (2, 2), "white").save(stream, "PNG")
+    return f"data:image/png;base64,{base64.b64encode(stream.getvalue()).decode()}"
+
+
 class FailingExecutor:
     model_name = "d3"
     max_length = None
     modalities = ["text"]
+    image_unavailable = "the checkpoint has no vision tower (visual.* weights)"
+    video_unavailable = image_unavailable
+
+    def load_images(self, images):
+        raise RuntimeError("boom")
+
+    def load_videos(self, videos):
+        raise RuntimeError("boom")
 
     def prepare(self, state, questions):
         raise RuntimeError("boom")
@@ -101,6 +122,18 @@ def test_health_reports_model_limits_and_modalities(server):
     assert status == 404 and json.loads(raw) == {"detail": "unknown route"}
 
 
+def test_health_reports_three_modalities_for_a_vision_checkpoint():
+    vision = start(FakeD3(media=True))
+    try:
+        status, _, raw = request(vision, "GET", "/health")
+    finally:
+        stop(vision)
+    assert status == 200
+    health = json.loads(raw)
+    assert health["status"] == "ready"
+    assert health["modalities"] == ["text", "image", "video"]
+
+
 def test_valid_three_kind_decision_returns_200(server):
     status, ctype, raw = post(server, json.dumps(three_kind_body()).encode("utf-8"))
     assert status == 200 and ctype == "application/json"
@@ -110,6 +143,32 @@ def test_valid_three_kind_decision_returns_200(server):
     assert result["usage"]["output_tokens"] == 0
 
 
+def test_media_request_returns_200_on_a_vision_checkpoint():
+    vision = start(FakeD3(media=True))
+    try:
+        value = three_kind_body()
+        value["images"] = [png_data_url()]
+        status, ctype, raw = post(vision, json.dumps(value).encode("utf-8"))
+    finally:
+        stop(vision)
+    assert status == 200 and ctype == "application/json"
+    result = json.loads(raw)
+    assert result["model"] == "d3"
+    assert result["answers"]["urgent"] == {"type": "noul", "noul": 0.5}
+
+
+def test_undecodable_media_rejects_the_whole_request():
+    vision = start(FakeD3(media=True))
+    try:
+        value = three_kind_body()
+        value["images"] = ["data:image/png;base64,!!!!"]
+        status, _, raw = post(vision, json.dumps(value).encode("utf-8"))
+    finally:
+        stop(vision)
+    assert status == 422
+    assert json.loads(raw) == {"detail": "images[0]: invalid base64 image data"}
+
+
 def test_unknown_post_route_is_404(server):
     status, _, _ = request(
         server,
@@ -178,6 +237,7 @@ def test_body_limit_and_content_type_are_enforced(server):
 
     status, _, raw = post(server, payload, headers=(("Content-Length", str(MAX_BODY + 1)),))
     assert status == 413 and "body limit" in json.loads(raw)["detail"]
+    assert "64 MiB" in json.loads(raw)["detail"]
 
     status, _, raw = post(server, payload, headers=(("Content-Type", "text/plain"),))
     assert status == 415 and "application/json" in json.loads(raw)["detail"]
@@ -191,10 +251,26 @@ def test_transport_errors_map_to_their_status_and_detail(server):
     status, _, raw = post(server, json.dumps(value).encode("utf-8"))
     assert status == 422 and "Unknown request fields" in json.loads(raw)["detail"]
 
+    # A non-list media field is refused with the upstream server's message.
     value = body()
-    value["images"] = ["data:image/png;base64,AAAA"]
+    value["images"] = "data:image/png;base64,AAAA"
     status, _, raw = post(server, json.dumps(value).encode("utf-8"))
-    assert status == 422 and "text state only" in json.loads(raw)["detail"]
+    assert status == 422
+    assert json.loads(raw)["detail"] == "images must be a list of base64 data URLs"
+
+    # Non-empty media on the text-only fixture is refused with the upstream
+    # capability messages (empty lists would take the text path).
+    value = body()
+    value["images"] = [png_data_url()]
+    status, _, raw = post(server, json.dumps(value).encode("utf-8"))
+    assert status == 422
+    assert json.loads(raw)["detail"] == "This model reads text only; images are not supported."
+
+    value = body()
+    value["videos"] = ["data:video/mp4;base64,AAAA"]
+    status, _, raw = post(server, json.dumps(value).encode("utf-8"))
+    assert status == 422
+    assert json.loads(raw)["detail"] == "This model does not read videos."
 
 
 def test_invalid_questions_return_the_upstream_422_body(server):

From e1dfb1d32a8d37575eefd7c6adbbd68076fe165e Mon Sep 17 00:00:00 2001
From: "Mr.Dong" <1042542469@qq.com>
Date: Wed, 14 Oct 2026 21:45:02 +0800
Subject: [PATCH 3/8] d3: extend the reference recipe for media serving

The compare tool's reference arm now replays the full upstream
system_one body (decode_images/decode_videos then decide), treating
the worker's fixed prepare-rejection prefix as an agreed difference;
rejection agreement for media 422s is compared verbatim.
---
 recipe/d3/compare_reference.py       |  82 ++++++++++++---
 recipe/d3/example-request-media.json |  26 +++++
 recipe/d3/reference.md               | 143 ++++++++++++++++++++++-----
 recipe/d3/test_compare_reference.py  | 141 +++++++++++++++++++++++++-
 4 files changed, 352 insertions(+), 40 deletions(-)
 create mode 100644 recipe/d3/example-request-media.json

diff --git a/recipe/d3/compare_reference.py b/recipe/d3/compare_reference.py
index b7ef07de..3a47fa6f 100644
--- a/recipe/d3/compare_reference.py
+++ b/recipe/d3/compare_reference.py
@@ -5,15 +5,18 @@
 the comparison isolates this repository's admission and orchestration from
 the checkpoint's own code:
 
-- reference: the decide() body of the upstream ``d3_server.py`` (prepare,
-  reject over-limit and invalid questions with 422, run, respond), driven
-  directly on the loaded runtime;
+- reference: the ``system_one`` body of the upstream ``d3_server.py``
+  (decode_images/decode_videos with their empty short-circuits and
+  capability gates, then the decide body: prepare with the media-aware
+  branches, reject over-limit and invalid questions with 422, run,
+  respond), driven directly on the loaded runtime;
 - worker: the production ``frontend.d3.decide`` pipeline (protocol envelope,
-  prepared-error admission, run, respond, response finishing).
+  decode stage, prepared-error admission, run, respond, response finishing).
 
 Both paths execute the same verified ``d3_runtime``, so the criteria are
 exact decision agreement (including embedded ``invalid_model_output``
-shapes), equal token accounting, and a probability tolerance of 1e-6
+shapes and agreed 422 rejections, media decode errors among them), equal
+token accounting, and a probability tolerance of 1e-6
 (``--tolerance``). ``--export`` additionally freezes the oracle fixture —
 token ids, probabilities and answers per question — for the future native
 worker, and writes it only after a passing, non-rejected comparison.
@@ -53,14 +56,57 @@ def parse_args(argv=None):
     return parser.parse_args(argv)
 
 
+def _decode_images(model, images):
+    """Upstream d3_server.py decode_images, verbatim."""
+
+    if not isinstance(images, list):
+        raise ValueError("images must be a list of base64 data URLs")
+    if not images:
+        return []
+    if getattr(model, "image_unavailable", "unknown") is not None:
+        raise ValueError("This model reads text only; images are not supported.")
+    return model.load_images(images, strict=True)
+
+
+def _decode_videos(model, videos):
+    """Upstream d3_server.py decode_videos, verbatim."""
+
+    if not isinstance(videos, list):
+        raise ValueError("videos must be a list of base64 data URLs")
+    if not videos:
+        return []
+    if getattr(model, "video_unavailable", "unknown") is not None:
+        raise ValueError("This model does not read videos.")
+    return model.load_videos(videos, strict=True)
+
+
+def decode_request_media(model, body):
+    """The media half of upstream system_one: images then videos."""
+
+    images = _decode_images(model, body["images"]) if body.get("images") is not None else []
+    videos = _decode_videos(model, body["videos"]) if body.get("videos") is not None else []
+    return images, videos
+
+
 def reference_decide(model, body):
-    """The upstream d3_server.py decide() body, without FastAPI.
+    """The upstream d3_server.py system_one body, without FastAPI.
 
     Returns the 200 response dict, or a ("422", detail) tuple when the
-    upstream server would have rejected the request before execution.
+    upstream server would have rejected the request before execution —
+    including media rejections from the decode stage, which upstream maps
+    to a plain-detail 422.
     """
 
-    prepared = model.prepare(body.get("state"), body.get("questions"))
+    try:
+        images, videos = decode_request_media(model, body)
+        if videos:
+            prepared = model.prepare(body.get("state"), body.get("questions"), images, videos)
+        elif images:
+            prepared = model.prepare(body.get("state"), body.get("questions"), images)
+        else:
+            prepared = model.prepare(body.get("state"), body.get("questions"))
+    except ValueError as exc:
+        return ("422", str(exc))
     over = [e for e in prepared.errors.values() if e["error"] == "max_length_exceeded"]
     if over:
         return ("422", over[0]["message"])
@@ -113,7 +159,7 @@ def compare_case(name, worker_response, reference, tolerance=PROBABILITY_TOLERAN
     Returns (report lines, passed).
     """
 
-    lines = [f"case: {name}", "reference path: d3_server decide()", "worker model: d3"]
+    lines = [f"case: {name}", "reference path: d3_server system_one()", "worker model: d3"]
     problems = []
     deltas: list[float] = []
     if isinstance(reference, tuple):
@@ -123,9 +169,16 @@ def compare_case(name, worker_response, reference, tolerance=PROBABILITY_TOLERAN
         else:
             detail = reference[1]
             body = rejection.get("body")
-            agreed = rejection.get("status") == 422 and body == (
-                detail if isinstance(detail, dict) else {"detail": detail}
-            )
+            if isinstance(detail, dict):
+                agreed = rejection.get("status") == 422 and body == detail
+            else:
+                # The worker deliberately wraps prepare-stage rejections in a
+                # fixed prefix (src/frontend/d3.py decide); upstream system_one
+                # surfaces the raw runtime message. Both forms agree here.
+                agreed = rejection.get("status") == 422 and body in (
+                    {"detail": detail},
+                    {"detail": "request rejected by the pinned runtime: " + detail},
+                )
             lines.append(f"worker rejection: {rejection.get('status')} {body}")
             lines.append("rejection agreement: " + ("PASS" if agreed else "FAIL"))
             return lines, bool(agreed)
@@ -220,7 +273,10 @@ def main(argv=None):
         lines, ok = compare_case(Path(args.fixture).name, worker_response, reference, args.tolerance)
         print("\n".join(lines))
         if args.export and export_allowed(ok, worker_response, reference):
-            prepared = executor.prepare(body.get("state"), dict(body["questions"]))
+            images, videos = decode_request_media(executor.model, body)
+            prepared = executor.prepare(
+                body.get("state"), dict(body["questions"]), images, videos
+            )
             probabilities, _ = executor.run(prepared)
             export_oracle(body, prepared, probabilities, worker_response, args.export)
             print(f"oracle exported: {args.export}")
diff --git a/recipe/d3/example-request-media.json b/recipe/d3/example-request-media.json
new file mode 100644
index 00000000..53fc5068
--- /dev/null
+++ b/recipe/d3/example-request-media.json
@@ -0,0 +1,26 @@
+{
+  "model": "d3",
+  "state": "The attached screenshot shows the customer's order history: one order charged twice on the same card.",
+  "questions": {
+    "refund": {
+      "type": "choice",
+      "instructions": "Decide the refund action.",
+      "criteria": {
+        "refund": "Refund the duplicate charge now",
+        "wait": "Wait for manual review"
+      }
+    },
+    "severity": {
+      "type": "score",
+      "instructions": "Rate the severity of the billing error.",
+      "criteria": [
+        "minor",
+        "moderate",
+        "major"
+      ]
+    }
+  },
+  "images": [
+    "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAEAAAABACAIAAAAlC+aJAAABTElEQVR4nO3ayxHCMAwEUIWhJVIBZVMBqSd37iAnkfXZNeM9Mhz8sB1sKcu+7zJybugBeDMB6EwAOn8N2J5r2Ti6s7T+BwhH/3i9fz/UZ4Bw9K3oANXKmfv1r6JUx8vB8BTiXFe2xyih4RzwtXLYDJdmgNlwdQnRGgx7gNNg28SEBvNhjs3QcxqlMnQep3kM/fcBEoPrQsNg8N7I4IaAKyXWEHMnBhrCLvUoQ2RVAmIILqvUG+LrQsWGlMJWpSGrMldmSCwt1hhya6MFhvTibrahojqdaigqr+cZ6voDSYbSBkeGobpDE24AtJhiDZgeWaAB1uSLMiC7lCEGcJvVb8D3iZ0GPEB8BgqAOAwsAOk1EAGky8AFELuBDiDGjjojQCyG81cN4E2k45DOgBr1pxwJoK6r5gtPVNmea2tXjAE4yEhLSM0EoDMB6HwA6F2Vhz1WcLcAAAAASUVORK5CYII="
+  ]
+}
diff --git a/recipe/d3/reference.md b/recipe/d3/reference.md
index 7c81ac73..e31ec756 100644
--- a/recipe/d3/reference.md
+++ b/recipe/d3/reference.md
@@ -18,15 +18,55 @@ The slice is:
 - the Python reference worker behind the Rust frontend, executing the
   checkpoint's own pinned runtime (`d3_runtime.py` and siblings) after a
   full-hash manifest gate;
-- text/JSON state and the three typed kinds — `choice`, `noul`, `score` —
-  in one request;
+- text/JSON state plus inline base64 media — `images` and `videos` lists on
+  the five-field request envelope — and the three typed kinds — `choice`,
+  `noul`, `score` — in one request;
 - BF16 backbone with FP32 readout, noncausal full attention, last-position
   pooling, one forward pass per question.
 
-Out of scope for this slice: `images`/`videos` (rejected with 422; upstream
-supports them per-item at 8,000,000 bytes per image and 32,000,000 per
-video), upstream's FastAPI server features (API key, request ids, server
-timing), and any native Rust/CUDA executor.
+Out of scope for this slice: upstream's FastAPI server features (API key,
+request ids, server timing, `/v1/models`), and any native Rust/CUDA
+executor.
+
+## Media budgets and the transport cap
+
+Per-item validation is the pinned runtime's strict loading
+(`load_images`/`load_videos` with `strict=True`); a media item it cannot
+decode rejects the whole request with HTTP 422 naming the item
+(`images[i]: ...` / `videos[i]: ...`). The budgets, transcribed from the
+pinned `d3_runtime.py` constants of this revision:
+
+| Media | Budget |
+| --- | --- |
+| one image | PNG, JPEG or WebP data URL; at most 8,000,000 decoded bytes (base64 length ≤ 4·⌈8,000,000/3⌉ ≈ 10.7 MB, decoded length re-checked) and 16,000,000 source pixels |
+| one video | MP4, WebM, QuickTime or Matroska data URL; at most 32,000,000 bytes (≈ 42.7 MB of base64), 300 seconds, 8,294,400 pixels per frame |
+| video sampling | 2 frames per second, 4–32 frames spread over the video, each sampled frame at most 200,704 pixels |
+| all videos of one request | at most 16,384 video input tokens |
+
+Media capability follows the loaded checkpoint: when the vision tower, the
+`qwen3_5` backbone or the video processor is missing, non-empty `images`/
+`videos` are refused with the upstream server's messages (`This model reads
+text only; images are not supported.` / `This model does not read videos.`).
+Empty or absent lists never trigger the capability check.
+
+On top of the per-item budgets this worker enforces a **64 MiB total request
+body cap** (HTTP 413). Upstream sets no total-body limit — its budgets are
+per item — so this is a deliberate difference, sized for one full-budget
+video (~42.7 MB of base64) plus envelope headroom; a second full-budget
+video in one request is refused with 413, not decoded. The cap also bounds
+the memory the stdlib worker holds while reading a body.
+
+## Timeout ceilings
+
+- The Rust frontend applies a **60 s total timeout** per forwarded request
+  (default, no environment override; only `OMNI_JEV_BIND`,
+  `OMNI_JEV_BACKEND_URL`, `OMNI_JEV_HEALTH_TIMEOUT_MS` and
+  `OMNI_JEV_MAX_RESPONSE_BYTES` are configurable). Uploading a near-budget
+  video over a slow link plus video inference can exceed it and surface as
+  a frontend 504.
+- The worker's 15 s connection timeout is per `recv` on the request stream:
+  a slow but continuously flowing upload does not trip it; only a stall
+  longer than 15 s produces the 408 body timeout.
 
 ## Architecture and ownership
 
@@ -34,6 +74,8 @@ timing), and any native Rust/CUDA executor.
 Rust frontend
   -> Python worker
      -> admit: envelope (protocol), pinned-runtime question errors (preprocess)
+     -> decode: media decode stage — empty short-circuit, capability gates,
+        strict loading via the executor (frontend.d3 mirroring d3_server.py)
      -> verify: manifest identity pin (repo id + runtime digests), full
         SHA-256 of the manifest runtime table, then import it (engine)
      -> execute: the checkpoint's own d3_runtime prepare/run/respond
@@ -44,16 +86,16 @@ The ownership boundary follows
 [`docs/architecture.md`](../../docs/architecture.md) and the
 [model contract](../../src/models/d3/README.md):
 
-- `protocol.py` owns the five-field envelope and the slice-scoped media
-  refusal;
+- `protocol.py` owns the five-field envelope and the media list-shape check;
 - `preprocess.py` owns the pre-execution 422 mapping
   (`invalid_questions` map, plain `max_length_exceeded` message);
-- `engine.py` owns the pinned runtime and the manifest gate stronger than
-  the upstream default;
+- `engine.py` owns the pinned runtime, the manifest gate stronger than the
+  upstream default, the capability flags and the strict-loading wrappers;
 - `postprocess.py` owns the pure answer formulas and the warning on
   embedded `invalid_model_output` answers (partial success stays HTTP 200
   but never passes silently);
-- `frontend.d3` owns HTTP lifecycle and orchestration;
+- `frontend.d3` owns HTTP lifecycle and orchestration, including the media
+  decode stage;
 - The Rust frontend forwards `/v1/systemone` and `/health` unchanged.
 
 ## Pinned artifacts
@@ -165,7 +207,39 @@ Expected response shape:
 }
 ~~~
 
-This is a serving smoke test, not an accuracy result; the exact numbers come
+### Media request
+
+[`example-request-media.json`](example-request-media.json) is the same shape
+with one inline PNG data URL in `images` — a screenshot-style input every
+question of the request sees:
+
+~~~sh
+curl -sS http://127.0.0.1:8000/v1/systemone \
+  -H 'Content-Type: application/json' \
+  --data-binary @recipe/d3/example-request-media.json
+~~~
+
+The response shape is unchanged; `usage.input_tokens` accounts for the image
+tokens. For a video request, generate a small in-budget clip and build the
+data URL yourself (the base64 is too large to commit as an example):
+
+~~~sh
+ffmpeg -f lavfi -i testsrc=duration=4:size=320x240:rate=8 \
+  -pix_fmt yuv420p -c:v libx264 -preset veryfast -crf 30 /tmp/d3-clip.mp4
+base64 -w0 /tmp/d3-clip.mp4 > /tmp/d3-clip.b64
+python3 - <<'PY'
+import json, pathlib
+encoded = pathlib.Path("/tmp/d3-clip.b64").read_text().strip()
+request = json.loads(pathlib.Path("recipe/d3/example-request.json").read_text())
+request["videos"] = [f"data:video/mp4;base64,{encoded}"]
+pathlib.Path("/tmp/d3-video-request.json").write_text(json.dumps(request))
+PY
+curl -sS http://127.0.0.1:8000/v1/systemone \
+  -H 'Content-Type: application/json' \
+  --data-binary @/tmp/d3-video-request.json
+~~~
+
+Both are serving smoke tests, not accuracy results; the exact numbers come
 from the checkpoint and are recorded by the parity tool below.
 
 ## Start and verify the Rust frontend
@@ -221,13 +295,16 @@ the pull request description, not in this recipe.
 
 [`compare_reference.py`](compare_reference.py) drives one fixture through
 two paths that share the loaded pinned model: the upstream
-`d3_server.py` decide body (prepare, 422 admission, run, respond) and the
-production `frontend.d3.decide` pipeline. The criteria are exact decision
-agreement including embedded `invalid_model_output` shapes, equal token
-accounting, and a numeric tolerance of 1e-6. `--export` freezes the oracle
-fixture (token ids, probabilities, answers) for the future native worker,
-and writes it only after a passing, non-rejected comparison — a failed run
-or an agreed 422 never touches the existing fixture.
+`d3_server.py` `system_one` body (media decode, prepare, 422 admission, run,
+respond) and the production `frontend.d3.decide` pipeline. The criteria are
+exact decision agreement including embedded `invalid_model_output` shapes
+and agreed 422 rejections (media decode errors among them), equal token
+accounting, and a numeric tolerance of 1e-6. Point `--fixture` at
+`example-request-media.json` (or any media request) to compare the media
+paths. `--export` freezes the oracle fixture (token ids, probabilities,
+answers) for the future native worker, and writes it only after a passing,
+non-rejected comparison — a failed run or an agreed 422 never touches the
+existing fixture.
 
 ~~~sh
 PYTHONPATH=src .venv/bin/python recipe/d3/compare_reference.py \
@@ -251,12 +328,25 @@ stage-3 native worker and is committed with the integration.
 - Upstream moved after the manifest was built — do not track `main` of the
   repository; re-pin deliberately and re-record the revision and the pinned
   runtime table (`PINNED_RUNTIME_SHA256`) in `src/models/d3/engine.py`.
-- `images`/`videos` non-empty lists are rejected with 422 in this slice;
-  the 8 MiB body cap is slice-scoped and will re-align to the upstream
-  per-item budgets when media opens (stage 2).
-- `/health` reports the loaded model's modalities even while this slice
-  refuses media; the 422 marks the serving boundary, not the checkpoint's
-  capability.
+- Media 422s name the item (`images[0]: ...`, `videos[1]: ...`) with the
+  pinned runtime's message; non-empty media against a checkpoint without
+  the capability is refused whole with the upstream server's message
+  (`This model reads text only; images are not supported.` /
+  `This model does not read videos.`). `/health`'s `modalities` shows what
+  the loaded checkpoint reads.
+- Over-budget totals are 413 (`request exceeds the 64 MiB body limit`):
+  the cap is this worker's deliberate difference from upstream, sized for
+  one full-budget video plus envelope headroom.
+- An inline base64 video is written to a named temporary file by the pinned
+  runtime before OpenCV decodes it (`d3-video-*` in the temp directory);
+  that is upstream behavior, not a worker-side staging step.
+- Decoding runs inside the worker's inference lock: one request at a time
+  covers decode, prepare, run and respond (upstream decodes outside its
+  lock in a thread pool — a deliberate throughput difference of this
+  worker, recorded in the integration PR).
+- The Rust frontend's 60 s total timeout has no override; a near-budget
+  video upload plus inference can surface as a frontend 504 (see
+  [Timeout ceilings](#timeout-ceilings)).
 - `/health` has no `authentication` field: this worker has no API key
   (a deliberate difference from upstream, which reads `DECISION_API_KEY`).
 - Without optional `causal_conv1d`, the pinned runtime falls back to its
@@ -265,4 +355,5 @@ stage-3 native worker and is committed with the integration.
 - The noncausal full-attention hook inside the pinned runtime is adapted
   from perplexity-ai/pplx-decider-v1.1-27b (Apache-2.0).
 - GPU evidence for the supported-models table is recorded in the
-  integration pull request, not in this recipe.
+  integration pull request, not in this recipe; media serving evidence is
+  author-side opt-in and likewise lives there until measured.
diff --git a/recipe/d3/test_compare_reference.py b/recipe/d3/test_compare_reference.py
index c77927b6..2a398e1f 100644
--- a/recipe/d3/test_compare_reference.py
+++ b/recipe/d3/test_compare_reference.py
@@ -3,7 +3,8 @@
 
 Checks the pure comparison and export helpers without the pinned d3 package,
 weights or a GPU: the PASS/FAIL verdicts of compare_case, embedded-error
-shape agreement, the oracle export gate, and the oracle export structure.
+shape agreement, the reference arm's media decode stage, the oracle export
+gate, and the oracle export structure.
 
     python recipe/d3/test_compare_reference.py
 """
@@ -14,6 +15,7 @@
 import sys
 import tempfile
 from pathlib import Path
+from types import SimpleNamespace
 
 HERE = Path(__file__).resolve().parent
 sys.path.insert(0, str(HERE))
@@ -182,6 +184,143 @@ def check(name: str, condition: bool, detail: str = "") -> None:
         "\n".join(lines),
     )
 
+    # 5b. The reference arm mirrors the upstream decode stage for media.
+    class FakeRuntime:
+        image_unavailable = None
+        video_unavailable = None
+
+        def __init__(self, unavailable=None):
+            if unavailable == "image":
+                self.image_unavailable = "the checkpoint has no vision tower (visual.* weights)"
+                self.video_unavailable = self.image_unavailable
+            elif unavailable == "video":
+                self.video_unavailable = "video inputs need a Qwen3.5 backbone"
+            self.image_calls = []
+            self.video_calls = []
+            self.prepared = []
+
+        def load_images(self, images, strict=False):
+            self.image_calls.append((list(images), strict))
+            return [("image", value) for value in images]
+
+        def load_videos(self, videos, strict=False):
+            self.video_calls.append((list(videos), strict))
+            return [("video", value) for value in videos]
+
+        def prepare(self, state, questions, images=None, videos=None):
+            self.prepared.append((images, videos))
+            return SimpleNamespace(errors={})
+
+        def run(self, prepared):
+            return {}, 5
+
+        def respond(self, prepared, probabilities, tokens):
+            return {
+                "model": "d3",
+                "answers": {},
+                "usage": {"input_tokens": tokens, "output_tokens": 0},
+            }
+
+    capable = FakeRuntime()
+    images, videos = tool.decode_request_media(
+        capable, {"images": ["data:image/png;base64,AAAA"], "videos": ["data:video/mp4;base64,AAAA"]}
+    )
+    check(
+        "decode passes media through strict loading",
+        images == [("image", "data:image/png;base64,AAAA")]
+        and videos == [("video", "data:video/mp4;base64,AAAA")]
+        and capable.image_calls == [(["data:image/png;base64,AAAA"], True)]
+        and capable.video_calls == [(["data:video/mp4;base64,AAAA"], True)],
+    )
+
+    empty = FakeRuntime()
+    images, videos = tool.decode_request_media(empty, {"images": [], "videos": None})
+    check(
+        "empty and absent media decode to nothing",
+        images == [] and videos == [] and empty.image_calls == [] and empty.video_calls == [],
+    )
+
+    text_only = FakeRuntime(unavailable="image")
+    try:
+        tool.decode_request_media(text_only, {"images": ["data:image/png;base64,AAAA"]})
+        check("text-only checkpoint refuses images", False, "no ValueError")
+    except ValueError as exc:
+        check(
+            "text-only checkpoint refuses images with the upstream message",
+            str(exc) == "This model reads text only; images are not supported.",
+            str(exc),
+        )
+
+    no_video = FakeRuntime(unavailable="video")
+    try:
+        tool.decode_request_media(no_video, {"videos": ["data:video/mp4;base64,AAAA"]})
+        check("video-incapable checkpoint refuses videos", False, "no ValueError")
+    except ValueError as exc:
+        check(
+            "video-incapable checkpoint refuses videos with the upstream message",
+            str(exc) == "This model does not read videos.",
+            str(exc),
+        )
+
+    for field, bad in (("images", "data:image/png;base64,AAAA"), ("videos", 17)):
+        try:
+            tool.decode_request_media(capable, {field: bad})
+            check(f"non-list {field} is refused", False, "no ValueError")
+        except ValueError as exc:
+            check(
+                f"non-list {field} is refused with the upstream message",
+                str(exc) == f"{field} must be a list of base64 data URLs",
+                str(exc),
+            )
+
+    media = FakeRuntime()
+    result = tool.reference_decide(
+        media,
+        {"state": "s", "questions": {"q": {"type": "noul"}}, "images": ["data:image/png;base64,AAAA"]},
+    )
+    check(
+        "reference_decide prepares decoded images on the image branch",
+        media.prepared == [([("image", "data:image/png;base64,AAAA")], None)]
+        and result["usage"] == {"input_tokens": 5, "output_tokens": 0},
+        str(media.prepared),
+    )
+
+    media = FakeRuntime()
+    tool.reference_decide(
+        media,
+        {
+            "state": "s",
+            "questions": {"q": {"type": "noul"}},
+            "images": ["data:image/png;base64,AAAA"],
+            "videos": ["data:video/mp4;base64,AAAA"],
+        },
+    )
+    check(
+        "reference_decide prepares both media on the video branch",
+        media.prepared
+        == [
+            (
+                [("image", "data:image/png;base64,AAAA")],
+                [("video", "data:video/mp4;base64,AAAA")],
+            )
+        ],
+        str(media.prepared),
+    )
+
+    class RejectingLoad(FakeRuntime):
+        def load_images(self, images, strict=False):
+            raise ValueError("images[0]: each image must be at most 8,000,000 bytes")
+
+    result = tool.reference_decide(
+        RejectingLoad(),
+        {"state": "s", "questions": {"q": {"type": "noul"}}, "images": ["data:image/png;base64,AAAA"]},
+    )
+    check(
+        "reference_decide maps decode rejections to plain-detail 422",
+        result == ("422", "images[0]: each image must be at most 8,000,000 bytes"),
+        str(result),
+    )
+
     # 6. The oracle export freezes the full stage-3 comparison surface.
     body = {"model": "d3", "state": "s", "questions": {"q": {"type": "noul"}}}
 

From 55c69ec7811577009933dfc8bdb13915bb169492 Mon Sep 17 00:00:00 2001
From: "Mr.Dong" <1042542469@qq.com>
Date: Wed, 14 Oct 2026 21:45:02 +0800
Subject: [PATCH 4/8] docs: record d3 media serving

---
 README.md                | 4 ++--
 docs/supported-models.md | 7 ++++---
 recipe/README.md         | 5 +++--
 3 files changed, 9 insertions(+), 7 deletions(-)

diff --git a/README.md b/README.md
index e4316082..3f254ea6 100644
--- a/README.md
+++ b/README.md
@@ -164,7 +164,7 @@ Python screenshot worker, Valen-Preview-0923 has a Python reference worker for
 text and one-image state, CLM has a stub-encoder contract recipe, and OmniJev-4B
 v1.1 has its checkpoint export and CPU contract, with no worker yet. D3
 (Decision 3.0) has a Python reference worker that runs the checkpoint's own
-pinned runtime for text state:
+pinned runtime for text, image and video state:
 
 | Model | Status |
 | --- | --- |
@@ -178,7 +178,7 @@ pinned runtime for text state:
 | CLM-v0.1-8B | [External worker with a CPU stub encoder](recipe/clm/README.md); contract checks only, real Qwen3-8B decisions unverified by this recipe |
 | LFM2.5-350M | [Transformers worker](recipe/lfm2/README.md); text state and independent choice questions |
 | OmniJev-4B v1.1 | [Checkpoint export and CPU contract](recipe/omnijev/README.md) only; the native worker is in progress ([#114](https://github.com/ThinkFlowLab/system1-omni/issues/114)) |
-| D3 (Decision 3.0) | [Python reference worker](recipe/d3/reference.md) on the checkpoint's pinned runtime; `choice`/`noul`/`score` text state, media refused until stage 2; author-run on A100 (evidence in the integration PR) |
+| D3 (Decision 3.0) | [Python reference worker](recipe/d3/reference.md) on the checkpoint's pinned runtime; `choice`/`noul`/`score` over text, image and video state; author-run on A100 for text serving (media evidence pending, see the integration PR) |
 
 [Supported models and hardware](docs/supported-models.md) lists the devices
 and where each worker has been run.
diff --git a/docs/supported-models.md b/docs/supported-models.md
index 3ca05477..9741859c 100644
--- a/docs/supported-models.md
+++ b/docs/supported-models.md
@@ -23,7 +23,7 @@ Models that are being added are also tracked in issues labeled [new model](https
 | LFM2.5-350M | [Transformers choice worker](../recipe/lfm2/README.md) | Tiny-model regression tests; full checkpoint unverified | L40S FP16 [recorded execution checks](../recipe/lfm2/RESULTS.md) | Unverified | Python 3.12, [pinned dependencies](../src/models/lfm2/requirements.lock.txt), explicit candidate batch size |
 | CLM-v0.1-8B | [Native Rust engine](../src/models/clm/README.md), `clm-run`, in front of a Qwen3-8B `/v1/embeddings` server | Engine-only agreement **4.5e-06** on identical vectors ([#29](https://github.com/ThinkFlowLab/system1-omni/pull/29), [validation](../recipe/clm/native/VALIDATION.md)); contract checks documented ([#23](https://github.com/ThinkFlowLab/system1-omni/pull/23)) | **Validated on compute capability 8.9** ([#29](https://github.com/ThinkFlowLab/system1-omni/pull/29), [validation](../recipe/clm/native/VALIDATION.md)) | Unverified | Rust to build and a separate Qwen3-8B `/v1/embeddings` server for the encoder; the engine itself has no device-specific code |
 | OmniJev-4B v1.1 | None yet: [checkpoint export and CPU contract](../recipe/omnijev/README.md); the native worker is in progress ([#114](https://github.com/ThinkFlowLab/system1-omni/issues/114)) | Request contract, preparation, heads and finishing checked against the reference ([#120](https://github.com/ThinkFlowLab/system1-omni/pull/120)); no inference | Not yet | Not supported | Python 3.10 reference environment for the export and fixtures |
-| D3 (Decision 3.0) | [Python reference worker](../recipe/d3/reference.md) running the checkpoint's pinned runtime | Unverified | Author-run on NVIDIA A100 (sm_80): three-kind serving smoke with byte-identical frontend forwarding and reference parity (exact decisions, max delta 0.0, 348 == 348 tokens); evidence in the integration PR. Owner: @MrDongsls. | Unverified | Python 3.12, the [pinned runtime requirements](../recipe/d3/requirements.txt), the [pinned package](https://huggingface.co/vllm-sr/d3) (~52 GB) |
+| D3 (Decision 3.0) | [Python reference worker](../recipe/d3/reference.md) running the checkpoint's pinned runtime | Unverified | Author-run on NVIDIA A100 (sm_80): three-kind serving smoke with byte-identical frontend forwarding and reference parity (exact decisions, max delta 0.0, 348 == 348 tokens); evidence in the integration PR. Media inputs are not yet measured on GPU. Owner: @MrDongsls. | Unverified | Python 3.12, the [pinned runtime requirements](../recipe/d3/requirements.txt), the [pinned package](https://huggingface.co/vllm-sr/d3) (~52 GB); [media budgets and the 64 MiB total request cap](../recipe/d3/reference.md) |
 
 - **Validated:** covered by the recipe on `main` or by the checks in the linked merged pull request.
 - **Author-run:** recorded by the recipe author on the stated hardware and not independently reproduced; the row names its owner.
@@ -32,8 +32,9 @@ Models that are being added are also tracked in issues labeled [new model](https
 
 The Cua-S1 workers answer `choice` questions only.
 LAYA's English worker, Open-Jev and CLM support `choice`, `score`, and `noul` text questions.
-D3's reference worker serves the same three kinds for text state; its media
-inputs stay refused for now and will be integrated soon.
+D3's reference worker serves the same three kinds for text, image and video
+state (inline base64 media under the pinned runtime's per-item budgets);
+its media serving evidence is author-side opt-in and not yet recorded.
 CLM has a row per path: the merged `clm-serve` recipe exercises those answer shapes with stub
 embeddings and says nothing about decision quality, while the native engine's CUDA row is the
 path through a real Qwen3-8B encoder. Its engine-only figure comes from the same comparison run
diff --git a/recipe/README.md b/recipe/README.md
index 32f83cd0..225df5bb 100644
--- a/recipe/README.md
+++ b/recipe/README.md
@@ -26,8 +26,9 @@ For a first real decision, follow the [complete CPU walkthrough](../docs/getting
 - [CLM behind the frontend](clm/README.md): run CLM's own server behind the frontend on
   CPU with a stub encoder, and what the response comparison has to allow for.
 - [D3 reference worker](d3/reference.md): download the pinned vllm-sr/d3 package,
-  verify its runtime manifest, and serve `choice`/`noul`/`score` text decisions
-  through the Python worker and the Rust frontend (serving and parity validated on A100).
+  verify its runtime manifest, and serve `choice`/`noul`/`score` decisions over
+  text, image and video state through the Python worker and the Rust frontend
+  (text serving and parity validated on A100; media budgets and caps in the recipe).
 
 - [LFM2.5-350M choice worker](lfm2/README.md): score text decisions with independent
   questions, hybrid cache branching and configurable candidate batches.

From 2574f875286cad4899b6c523d01e49177f3f87e7 Mon Sep 17 00:00:00 2001
From: "Mr.Dong" <1042542469@qq.com>
Date: Fri, 16 Oct 2026 00:49:22 +0800
Subject: [PATCH 5/8] docs: record the d3 media validation on A100

---
 README.md                | 2 +-
 docs/supported-models.md | 4 ++--
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/README.md b/README.md
index 3f254ea6..c6e61d8b 100644
--- a/README.md
+++ b/README.md
@@ -178,7 +178,7 @@ pinned runtime for text, image and video state:
 | CLM-v0.1-8B | [External worker with a CPU stub encoder](recipe/clm/README.md); contract checks only, real Qwen3-8B decisions unverified by this recipe |
 | LFM2.5-350M | [Transformers worker](recipe/lfm2/README.md); text state and independent choice questions |
 | OmniJev-4B v1.1 | [Checkpoint export and CPU contract](recipe/omnijev/README.md) only; the native worker is in progress ([#114](https://github.com/ThinkFlowLab/system1-omni/issues/114)) |
-| D3 (Decision 3.0) | [Python reference worker](recipe/d3/reference.md) on the checkpoint's pinned runtime; `choice`/`noul`/`score` over text, image and video state; author-run on A100 for text serving (media evidence pending, see the integration PR) |
+| D3 (Decision 3.0) | [Python reference worker](recipe/d3/reference.md) on the checkpoint's pinned runtime; `choice`/`noul`/`score` over text, image and video state; author-run on A100 for text and media serving (see the integration and media PRs) |
 
 [Supported models and hardware](docs/supported-models.md) lists the devices
 and where each worker has been run.
diff --git a/docs/supported-models.md b/docs/supported-models.md
index 9741859c..b06e67ac 100644
--- a/docs/supported-models.md
+++ b/docs/supported-models.md
@@ -23,7 +23,7 @@ Models that are being added are also tracked in issues labeled [new model](https
 | LFM2.5-350M | [Transformers choice worker](../recipe/lfm2/README.md) | Tiny-model regression tests; full checkpoint unverified | L40S FP16 [recorded execution checks](../recipe/lfm2/RESULTS.md) | Unverified | Python 3.12, [pinned dependencies](../src/models/lfm2/requirements.lock.txt), explicit candidate batch size |
 | CLM-v0.1-8B | [Native Rust engine](../src/models/clm/README.md), `clm-run`, in front of a Qwen3-8B `/v1/embeddings` server | Engine-only agreement **4.5e-06** on identical vectors ([#29](https://github.com/ThinkFlowLab/system1-omni/pull/29), [validation](../recipe/clm/native/VALIDATION.md)); contract checks documented ([#23](https://github.com/ThinkFlowLab/system1-omni/pull/23)) | **Validated on compute capability 8.9** ([#29](https://github.com/ThinkFlowLab/system1-omni/pull/29), [validation](../recipe/clm/native/VALIDATION.md)) | Unverified | Rust to build and a separate Qwen3-8B `/v1/embeddings` server for the encoder; the engine itself has no device-specific code |
 | OmniJev-4B v1.1 | None yet: [checkpoint export and CPU contract](../recipe/omnijev/README.md); the native worker is in progress ([#114](https://github.com/ThinkFlowLab/system1-omni/issues/114)) | Request contract, preparation, heads and finishing checked against the reference ([#120](https://github.com/ThinkFlowLab/system1-omni/pull/120)); no inference | Not yet | Not supported | Python 3.10 reference environment for the export and fixtures |
-| D3 (Decision 3.0) | [Python reference worker](../recipe/d3/reference.md) running the checkpoint's pinned runtime | Unverified | Author-run on NVIDIA A100 (sm_80): three-kind serving smoke with byte-identical frontend forwarding and reference parity (exact decisions, max delta 0.0, 348 == 348 tokens); evidence in the integration PR. Media inputs are not yet measured on GPU. Owner: @MrDongsls. | Unverified | Python 3.12, the [pinned runtime requirements](../recipe/d3/requirements.txt), the [pinned package](https://huggingface.co/vllm-sr/d3) (~52 GB); [media budgets and the 64 MiB total request cap](../recipe/d3/reference.md) |
+| D3 (Decision 3.0) | [Python reference worker](../recipe/d3/reference.md) running the checkpoint's pinned runtime | Unverified | Author-run on NVIDIA A100 (sm_80): three-kind serving smoke with byte-identical frontend forwarding and reference parity (exact decisions, max delta 0.0, 348 == 348 tokens); evidence in the integration PR. Media (stage 2), same host: image/video smokes, media parity exact (max delta 0.0, media oracle frozen), per-item budgets enforced verbatim (39.2 MB video → 422 `videos[0]: … at most 32,000,000 bytes`; 18,816 planned video tokens → 422 over the 16,384 budget), 64 MiB body cap enforced (99.8 MB request → 413), one 31.2 MB video served end-to-end through the frontend in 4.1 s, concurrent request 503-busy in 10 ms; evidence in the media PR. Owner: @MrDongsls. | Unverified | Python 3.12, the [pinned runtime requirements](../recipe/d3/requirements.txt), the [pinned package](https://huggingface.co/vllm-sr/d3) (~52 GB); [media budgets and the 64 MiB total request cap](../recipe/d3/reference.md) |
 
 - **Validated:** covered by the recipe on `main` or by the checks in the linked merged pull request.
 - **Author-run:** recorded by the recipe author on the stated hardware and not independently reproduced; the row names its owner.
@@ -34,7 +34,7 @@ The Cua-S1 workers answer `choice` questions only.
 LAYA's English worker, Open-Jev and CLM support `choice`, `score`, and `noul` text questions.
 D3's reference worker serves the same three kinds for text, image and video
 state (inline base64 media under the pinned runtime's per-item budgets);
-its media serving evidence is author-side opt-in and not yet recorded.
+its media serving evidence is author-run on the A100 above.
 CLM has a row per path: the merged `clm-serve` recipe exercises those answer shapes with stub
 embeddings and says nothing about decision quality, while the native engine's CUDA row is the
 path through a real Qwen3-8B encoder. Its engine-only figure comes from the same comparison run

From 05fcce1dd0f48e53a3948c3cabb4f86be2f5bdc2 Mon Sep 17 00:00:00 2001
From: "Mr.Dong" <1042542469@qq.com>
Date: Fri, 16 Oct 2026 01:49:13 +0800
Subject: [PATCH 6/8] fix(d3): record only the observed media evidence and lock
 the decode order
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

The supported-models media row cited four GPU boundary probes (per-item
byte-budget 422, the 16,384 video-token 422, a 413 at the cap, a timed
full-budget video) that have no raw results in the pulled validation
logs — the E1 probe crashed on a missing harness fixture before sending
and E2's synthetic clips were too small to reach the cap and returned
200. Keep only the log-backed facts and mark the remaining boundaries
as CPU-suite-only until they are observed on GPU; drop the media-oracle
claim, whose export froze an empty token_ids map.

The capability-gate test now also asserts the images decoded before the
video refusal, as its comment already claimed.
---
 docs/supported-models.md | 2 +-
 tests/d3/test_worker.py  | 2 ++
 2 files changed, 3 insertions(+), 1 deletion(-)

diff --git a/docs/supported-models.md b/docs/supported-models.md
index b06e67ac..3dcee1f5 100644
--- a/docs/supported-models.md
+++ b/docs/supported-models.md
@@ -23,7 +23,7 @@ Models that are being added are also tracked in issues labeled [new model](https
 | LFM2.5-350M | [Transformers choice worker](../recipe/lfm2/README.md) | Tiny-model regression tests; full checkpoint unverified | L40S FP16 [recorded execution checks](../recipe/lfm2/RESULTS.md) | Unverified | Python 3.12, [pinned dependencies](../src/models/lfm2/requirements.lock.txt), explicit candidate batch size |
 | CLM-v0.1-8B | [Native Rust engine](../src/models/clm/README.md), `clm-run`, in front of a Qwen3-8B `/v1/embeddings` server | Engine-only agreement **4.5e-06** on identical vectors ([#29](https://github.com/ThinkFlowLab/system1-omni/pull/29), [validation](../recipe/clm/native/VALIDATION.md)); contract checks documented ([#23](https://github.com/ThinkFlowLab/system1-omni/pull/23)) | **Validated on compute capability 8.9** ([#29](https://github.com/ThinkFlowLab/system1-omni/pull/29), [validation](../recipe/clm/native/VALIDATION.md)) | Unverified | Rust to build and a separate Qwen3-8B `/v1/embeddings` server for the encoder; the engine itself has no device-specific code |
 | OmniJev-4B v1.1 | None yet: [checkpoint export and CPU contract](../recipe/omnijev/README.md); the native worker is in progress ([#114](https://github.com/ThinkFlowLab/system1-omni/issues/114)) | Request contract, preparation, heads and finishing checked against the reference ([#120](https://github.com/ThinkFlowLab/system1-omni/pull/120)); no inference | Not yet | Not supported | Python 3.10 reference environment for the export and fixtures |
-| D3 (Decision 3.0) | [Python reference worker](../recipe/d3/reference.md) running the checkpoint's pinned runtime | Unverified | Author-run on NVIDIA A100 (sm_80): three-kind serving smoke with byte-identical frontend forwarding and reference parity (exact decisions, max delta 0.0, 348 == 348 tokens); evidence in the integration PR. Media (stage 2), same host: image/video smokes, media parity exact (max delta 0.0, media oracle frozen), per-item budgets enforced verbatim (39.2 MB video → 422 `videos[0]: … at most 32,000,000 bytes`; 18,816 planned video tokens → 422 over the 16,384 budget), 64 MiB body cap enforced (99.8 MB request → 413), one 31.2 MB video served end-to-end through the frontend in 4.1 s, concurrent request 503-busy in 10 ms; evidence in the media PR. Owner: @MrDongsls. | Unverified | Python 3.12, the [pinned runtime requirements](../recipe/d3/requirements.txt), the [pinned package](https://huggingface.co/vllm-sr/d3) (~52 GB); [media budgets and the 64 MiB total request cap](../recipe/d3/reference.md) |
+| D3 (Decision 3.0) | [Python reference worker](../recipe/d3/reference.md) running the checkpoint's pinned runtime | Unverified | Author-run on NVIDIA A100 (sm_80): three-kind serving smoke with byte-identical frontend forwarding and reference parity (exact decisions, max delta 0.0, 348 == 348 tokens); evidence in the integration PR. Media (stage 2), same host: image/video smokes, media parity exact over consecutive runs (max delta 0.0), byte-identical frontend forwarding on the media fixture, one strict-decode rejection observed verbatim (invalid base64 → 422 `images[0]: invalid base64 image data`), two 300 s videos billing 19,368 video tokens served as one 200 request, concurrent request 503-busy in 10 ms; the remaining budget boundaries (per-item byte budgets, the 16,384 video-token check, the 64 MiB cap against full-budget videos) are enforced by the CPU suite but not yet observed on GPU; evidence in the media PR. Owner: @MrDongsls. | Unverified | Python 3.12, the [pinned runtime requirements](../recipe/d3/requirements.txt), the [pinned package](https://huggingface.co/vllm-sr/d3) (~52 GB); [media budgets and the 64 MiB total request cap](../recipe/d3/reference.md) |
 
 - **Validated:** covered by the recipe on `main` or by the checks in the linked merged pull request.
 - **Author-run:** recorded by the recipe author on the stated hardware and not independently reproduced; the row names its owner.
diff --git a/tests/d3/test_worker.py b/tests/d3/test_worker.py
index 3643964a..8ba6a36b 100644
--- a/tests/d3/test_worker.py
+++ b/tests/d3/test_worker.py
@@ -435,6 +435,8 @@ def test_decide_refuses_media_a_text_only_checkpoint_cannot_read():
     with pytest.raises(RequestError) as excinfo:
         decide(json.dumps(value).encode("utf-8"), executor)
     assert str(excinfo.value) == "This model does not read videos."
+    assert executor.loaded_images == [list(value["images"])]
+    assert executor.loaded_videos == []
     assert executor.prepare_calls == []
 
 

From 33a9739aaf6d0d21c9a5ea73bee5dee718746869 Mon Sep 17 00:00:00 2001
From: "Mr.Dong" <1042542469@qq.com>
Date: Fri, 16 Oct 2026 13:08:12 +0800
Subject: [PATCH 7/8] docs: trim the D3 supported-models row to capability and
 evidence pointer

---
 docs/supported-models.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/supported-models.md b/docs/supported-models.md
index 3dcee1f5..ea54908b 100644
--- a/docs/supported-models.md
+++ b/docs/supported-models.md
@@ -23,7 +23,7 @@ Models that are being added are also tracked in issues labeled [new model](https
 | LFM2.5-350M | [Transformers choice worker](../recipe/lfm2/README.md) | Tiny-model regression tests; full checkpoint unverified | L40S FP16 [recorded execution checks](../recipe/lfm2/RESULTS.md) | Unverified | Python 3.12, [pinned dependencies](../src/models/lfm2/requirements.lock.txt), explicit candidate batch size |
 | CLM-v0.1-8B | [Native Rust engine](../src/models/clm/README.md), `clm-run`, in front of a Qwen3-8B `/v1/embeddings` server | Engine-only agreement **4.5e-06** on identical vectors ([#29](https://github.com/ThinkFlowLab/system1-omni/pull/29), [validation](../recipe/clm/native/VALIDATION.md)); contract checks documented ([#23](https://github.com/ThinkFlowLab/system1-omni/pull/23)) | **Validated on compute capability 8.9** ([#29](https://github.com/ThinkFlowLab/system1-omni/pull/29), [validation](../recipe/clm/native/VALIDATION.md)) | Unverified | Rust to build and a separate Qwen3-8B `/v1/embeddings` server for the encoder; the engine itself has no device-specific code |
 | OmniJev-4B v1.1 | None yet: [checkpoint export and CPU contract](../recipe/omnijev/README.md); the native worker is in progress ([#114](https://github.com/ThinkFlowLab/system1-omni/issues/114)) | Request contract, preparation, heads and finishing checked against the reference ([#120](https://github.com/ThinkFlowLab/system1-omni/pull/120)); no inference | Not yet | Not supported | Python 3.10 reference environment for the export and fixtures |
-| D3 (Decision 3.0) | [Python reference worker](../recipe/d3/reference.md) running the checkpoint's pinned runtime | Unverified | Author-run on NVIDIA A100 (sm_80): three-kind serving smoke with byte-identical frontend forwarding and reference parity (exact decisions, max delta 0.0, 348 == 348 tokens); evidence in the integration PR. Media (stage 2), same host: image/video smokes, media parity exact over consecutive runs (max delta 0.0), byte-identical frontend forwarding on the media fixture, one strict-decode rejection observed verbatim (invalid base64 → 422 `images[0]: invalid base64 image data`), two 300 s videos billing 19,368 video tokens served as one 200 request, concurrent request 503-busy in 10 ms; the remaining budget boundaries (per-item byte budgets, the 16,384 video-token check, the 64 MiB cap against full-budget videos) are enforced by the CPU suite but not yet observed on GPU; evidence in the media PR. Owner: @MrDongsls. | Unverified | Python 3.12, the [pinned runtime requirements](../recipe/d3/requirements.txt), the [pinned package](https://huggingface.co/vllm-sr/d3) (~52 GB); [media budgets and the 64 MiB total request cap](../recipe/d3/reference.md) |
+| D3 (Decision 3.0) | [Python reference worker](../recipe/d3/reference.md) on the checkpoint's pinned runtime — the full upstream feature set: text, image and video state over `choice`/`noul`/`score`; the native worker is in progress ([#136](https://github.com/ThinkFlowLab/system1-omni/issues/136)) | Unverified | Author-run on NVIDIA A100 (sm_80): serving smoke and reference parity for text and media (exact decisions, max delta 0.0); evidence in the integration and media PRs. Owner: @MrDongsls. | Unverified | Python 3.12, the [pinned runtime requirements](../recipe/d3/requirements.txt), the [pinned package](https://huggingface.co/vllm-sr/d3) (~52 GB); [media budgets and the 64 MiB total request cap](../recipe/d3/reference.md) |
 
 - **Validated:** covered by the recipe on `main` or by the checks in the linked merged pull request.
 - **Author-run:** recorded by the recipe author on the stated hardware and not independently reproduced; the row names its owner.

From 4e2cc59e947e73d3e2cfe07fd9d5568d4399c2fe Mon Sep 17 00:00:00 2001
From: "Mr.Dong" <1042542469@qq.com>
Date: Sun, 18 Oct 2026 11:16:11 +0800
Subject: [PATCH 8/8] docs(d3): claim modality coverage, not the full upstream
 feature set

Address the review finding on PR #148: the worker row's 'full upstream
feature set' contradicted the deliberate transport differences (64 MiB
total-body cap, no /v1/models, no API key, decode under the inference
lock) documented in the same recipe. State the modality coverage and
point to the recipe for the transport differences.
---
 docs/supported-models.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/supported-models.md b/docs/supported-models.md
index ea54908b..667d97bc 100644
--- a/docs/supported-models.md
+++ b/docs/supported-models.md
@@ -23,7 +23,7 @@ Models that are being added are also tracked in issues labeled [new model](https
 | LFM2.5-350M | [Transformers choice worker](../recipe/lfm2/README.md) | Tiny-model regression tests; full checkpoint unverified | L40S FP16 [recorded execution checks](../recipe/lfm2/RESULTS.md) | Unverified | Python 3.12, [pinned dependencies](../src/models/lfm2/requirements.lock.txt), explicit candidate batch size |
 | CLM-v0.1-8B | [Native Rust engine](../src/models/clm/README.md), `clm-run`, in front of a Qwen3-8B `/v1/embeddings` server | Engine-only agreement **4.5e-06** on identical vectors ([#29](https://github.com/ThinkFlowLab/system1-omni/pull/29), [validation](../recipe/clm/native/VALIDATION.md)); contract checks documented ([#23](https://github.com/ThinkFlowLab/system1-omni/pull/23)) | **Validated on compute capability 8.9** ([#29](https://github.com/ThinkFlowLab/system1-omni/pull/29), [validation](../recipe/clm/native/VALIDATION.md)) | Unverified | Rust to build and a separate Qwen3-8B `/v1/embeddings` server for the encoder; the engine itself has no device-specific code |
 | OmniJev-4B v1.1 | None yet: [checkpoint export and CPU contract](../recipe/omnijev/README.md); the native worker is in progress ([#114](https://github.com/ThinkFlowLab/system1-omni/issues/114)) | Request contract, preparation, heads and finishing checked against the reference ([#120](https://github.com/ThinkFlowLab/system1-omni/pull/120)); no inference | Not yet | Not supported | Python 3.10 reference environment for the export and fixtures |
-| D3 (Decision 3.0) | [Python reference worker](../recipe/d3/reference.md) on the checkpoint's pinned runtime — the full upstream feature set: text, image and video state over `choice`/`noul`/`score`; the native worker is in progress ([#136](https://github.com/ThinkFlowLab/system1-omni/issues/136)) | Unverified | Author-run on NVIDIA A100 (sm_80): serving smoke and reference parity for text and media (exact decisions, max delta 0.0); evidence in the integration and media PRs. Owner: @MrDongsls. | Unverified | Python 3.12, the [pinned runtime requirements](../recipe/d3/requirements.txt), the [pinned package](https://huggingface.co/vllm-sr/d3) (~52 GB); [media budgets and the 64 MiB total request cap](../recipe/d3/reference.md) |
+| D3 (Decision 3.0) | [Python reference worker](../recipe/d3/reference.md) on the checkpoint's pinned runtime — text, image and video state over `choice`/`noul`/`score`, with the deliberate transport differences documented in the recipe; the native worker is in progress ([#136](https://github.com/ThinkFlowLab/system1-omni/issues/136)) | Unverified | Author-run on NVIDIA A100 (sm_80): serving smoke and reference parity for text and media (exact decisions, max delta 0.0); evidence in the integration and media PRs. Owner: @MrDongsls. | Unverified | Python 3.12, the [pinned runtime requirements](../recipe/d3/requirements.txt), the [pinned package](https://huggingface.co/vllm-sr/d3) (~52 GB); [media budgets and the 64 MiB total request cap](../recipe/d3/reference.md) |
 
 - **Validated:** covered by the recipe on `main` or by the checks in the linked merged pull request.
 - **Author-run:** recorded by the recipe author on the stated hardware and not independently reproduced; the row names its owner.