From 2a41f7de1fd3b36deacfe9039cc2886e8082fec8 Mon Sep 17 00:00:00 2001 From: linear3735 Date: Fri, 9 Oct 2026 10:33:42 +0800 Subject: [PATCH 01/20] Share bounded image decoding and JEV image geometry --- Cargo.lock | 2 + recipe/jev_vl/export_vision.py | 94 ++++++++++++ src/models/cua_s1/native/src/image_request.rs | 111 +------------- src/models/jev_vl/native/src/executor.rs | 28 ++-- src/models/jev_vl/native/src/images.rs | 46 +++++- src/models/jev_vl/native/src/processing.rs | 22 +-- src/models/qwen3_5/native/Cargo.toml | 6 + src/models/qwen3_5/native/src/image_decode.rs | 120 +++++++++++++++ src/models/qwen3_5/native/src/lib.rs | 1 + tests/jev_vl/images.rs | 26 +++- tests/qwen3_5/image_decode.rs | 139 ++++++++++++++++++ 11 files changed, 458 insertions(+), 137 deletions(-) create mode 100644 recipe/jev_vl/export_vision.py create mode 100644 src/models/qwen3_5/native/src/image_decode.rs create mode 100644 tests/qwen3_5/image_decode.rs diff --git a/Cargo.lock b/Cargo.lock index c43ff226..cb4db489 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1231,7 +1231,9 @@ name = "omni-qwen3-5-native" version = "0.1.0" dependencies = [ "anyhow", + "base64 0.22.1", "half", + "image", "libloading", "memmap2", "safetensors 0.8.0", diff --git a/recipe/jev_vl/export_vision.py b/recipe/jev_vl/export_vision.py new file mode 100644 index 00000000..f6e8abd1 --- /dev/null +++ b/recipe/jev_vl/export_vision.py @@ -0,0 +1,94 @@ +"""Export the pinned JEV vision tower for the shared Rust/CUDA backend (CPU only).""" + +import argparse +import hashlib +import json +import shutil +from pathlib import Path + + +def sha256(path): + digest = hashlib.sha256() + with path.open("rb") as stream: + for block in iter(lambda: stream.read(4 << 20), b""): + digest.update(block) + return digest.hexdigest() + + +def export(model, out): + import torch + from safetensors import safe_open + from safetensors.torch import save_file + + if out.exists(): + raise ValueError("output already exists; choose a new export directory") + config = json.loads((model / "config.json").read_text()) + if (config.get("model_type") != "qwen3_5" + or config.get("text_config", {}).get("hidden_size") != 5120): + raise ValueError("expected the JEV-27B-VL Qwen3.8 backbone") + processor = json.loads((model / "preprocessor_config.json").read_text()) + expected_processor = { + "size": {"longest_edge": 16777216, "shortest_edge": 65536}, + "patch_size": 16, "temporal_patch_size": 2, "merge_size": 2, + "image_mean": [0.5, 0.5, 0.5], "image_std": [0.5, 0.5, 0.5], + "processor_class": "Qwen3VLProcessor", + "image_processor_type": "Qwen2VLImageProcessorFast", + } + if processor != expected_processor: + raise ValueError("unsupported image processor; verify resize and normalization first") + + adapter = model / "adapter_vllm" + with safe_open(adapter / "adapter_model.safetensors", framework="pt") as weights: + for name in weights.keys(): + if not name.startswith(("base_model.model.model.language_model.", + "base_model.model.lm_head.")): + raise ValueError(f"unsupported non-language adapter tensor: {name}") + + index = json.loads((model / "model.safetensors.index.json").read_text())["weight_map"] + names = {name for name in index if name.startswith("model.visual.")} + if not names: + raise ValueError("checkpoint contains no vision tensors") + tensors = {} + for shard in sorted({index[name] for name in names}): + path = (model / shard).resolve() + if not path.is_relative_to(model.resolve()): + raise ValueError("checkpoint shard escapes model directory") + with safe_open(path, framework="pt") as weights: + for name in sorted(names): + if index[name] != shard: + continue + tensor = weights.get_tensor(name) + if tensor.dtype != torch.bfloat16 or not torch.isfinite(tensor).all(): + raise ValueError(f"expected finite BF16 vision weights: {name}") + tensors[name] = tensor.contiguous() + + out.mkdir(parents=True) + save_file(tensors, out / "vision.safetensors") + for name in ("config.json", "preprocessor_config.json"): + shutil.copyfile(model / name, out / name) + manifest = { + "format": "jev-vl-vision/1", + "model_id": "autotrust/JEV-27B-VL", + "pins": { + "model_index_sha256": sha256(model / "model.safetensors.index.json"), + "adapter_config_sha256": sha256(adapter / "adapter_config.json"), + "adapter_model_sha256": sha256(adapter / "adapter_model.safetensors"), + }, + "files": {name: sha256(out / name) for name in + ("config.json", "preprocessor_config.json", "vision.safetensors")}, + } + # The worker requires this manifest, so a partial export is never loadable. + (out / "jev_vl_vision.json").write_text(json.dumps(manifest, indent=2) + "\n") + print(f"exported {len(tensors)} vision tensors -> {out}") + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--model", type=Path, required=True) + parser.add_argument("--out", type=Path, required=True) + args = parser.parse_args() + export(args.model, args.out) + + +if __name__ == "__main__": + main() diff --git a/src/models/cua_s1/native/src/image_request.rs b/src/models/cua_s1/native/src/image_request.rs index 9a626208..ea6d7f71 100644 --- a/src/models/cua_s1/native/src/image_request.rs +++ b/src/models/cua_s1/native/src/image_request.rs @@ -1,12 +1,9 @@ //! Bounded PNG/JPEG screenshot request mapping for the native worker. use crate::contract::Question; use anyhow::{Context, Result, ensure}; -use base64::Engine; use serde_json::{Map, Value}; -use std::io::Cursor; pub const MAX_BODY: usize = 8 * 1024 * 1024; -pub const MAX_IMAGE_BYTES: usize = 4 * 1024 * 1024; pub const MODEL_ID: &str = "cua-ai/cua-s1-4b-0.2@16818868b0cc7813808aae4e87b417657046ab79:multimodal"; @@ -78,114 +75,15 @@ pub fn parse_image_body(body: &Map) -> Result { .get("image") .and_then(Value::as_str) .context("state.image must be a data URL")?; - let (prefix, encoded) = url.split_once(',').context("invalid image data URL")?; - let format = match prefix { - "data:image/png;base64" => image::ImageFormat::Png, - "data:image/jpeg;base64" => image::ImageFormat::Jpeg, - _ => anyhow::bail!("only inline PNG/JPEG images are supported"), - }; - ensure!( - encoded.len() <= 4 * MAX_IMAGE_BYTES.div_ceil(3), - "encoded image exceeds 4 MiB" - ); - let raw = base64::engine::general_purpose::STANDARD - .decode(encoded) - .context("invalid base64 image")?; - ensure!(raw.len() <= MAX_IMAGE_BYTES, "image exceeds 4 MiB"); - ensure!( - image::guess_format(&raw)? == format, - "image format does not match MIME type" - ); - if format == image::ImageFormat::Jpeg { - ensure_single_jpeg(&raw)?; - } - let mut limits = image::Limits::default(); - limits.max_image_width = Some(2048); - limits.max_image_height = Some(2048); - limits.max_alloc = Some(32 * 1024 * 1024); - let mut header = image::ImageReader::with_format(Cursor::new(&raw), format); - header.limits(limits.clone()); - let (width, height) = header.into_dimensions()?; - let (w, h) = (width as usize, height as usize); - ensure!( - w > 0 && h > 0 && w <= 2048 && h <= 2048 && w * h <= 1048576 && w.max(h) <= 200 * w.min(h), - "image dimensions exceed supported limits" - ); - let decoded = if format == image::ImageFormat::Png { - let decoder = image::codecs::png::PngDecoder::with_limits(Cursor::new(&raw), limits)?; - ensure!(!decoder.is_apng()?, "image must be single-frame"); - image::DynamicImage::from_decoder(decoder)? - } else { - let mut reader = image::ImageReader::with_format(Cursor::new(&raw), format); - reader.limits(limits); - reader.decode()? - }; + let decoded = omni_qwen3_5_native::image_decode::decode_data_url(url)?; Ok(ImageRequest { - width: w, - height: h, - rgb: pillow_rgb( - decoded, - format == image::ImageFormat::Png && raw.get(25) == Some(&0), - ), + width: decoded.width, + height: decoded.height, + rgb: decoded.rgb, questions, }) } -fn ensure_single_jpeg(raw: &[u8]) -> Result<()> { - // MPF APP2 identifies an MPO container. Pillow reports it as MPO rather - // than JPEG; decoding it as JPEG would silently select the first picture. - // Walk header segments only, so arbitrary metadata/entropy bytes cannot - // be mistaken for an MPF marker. - let mut offset = 2; // SOI was checked by guess_format. - while offset < raw.len() { - ensure!(raw[offset] == 0xff, "invalid JPEG marker"); - while raw.get(offset) == Some(&0xff) { - offset += 1; - } - let marker = *raw.get(offset).context("truncated JPEG marker")?; - offset += 1; - match marker { - 0xda | 0xd9 => break, // SOS / EOI; the decoder checks the rest. - 0x01 | 0xd0..=0xd8 => continue, // Standalone markers. - _ => {} - } - let length = raw - .get(offset..offset + 2) - .context("truncated JPEG segment")?; - let length = u16::from_be_bytes([length[0], length[1]]) as usize; - ensure!(length >= 2, "invalid JPEG segment length"); - let segment = raw - .get(offset + 2..offset + length) - .context("truncated JPEG segment")?; - ensure!( - marker != 0xe2 || !segment.starts_with(b"MPF\0"), - "image must be single-frame; MPF/MPO containers are unsupported" - ); - offset += length; - } - Ok(()) -} - -fn pillow_rgb(image: image::DynamicImage, png_grayscale: bool) -> Vec { - use image::DynamicImage::*; - match image { - ImageLuma16(p) => p.pixels().flat_map(|v| [v[0].min(255) as u8; 3]).collect(), - ImageLumaA16(p) if png_grayscale => { - p.pixels().flat_map(|v| [v[0].min(255) as u8; 3]).collect() - } - ImageLumaA16(p) => p.pixels().flat_map(|v| [(v[0] >> 8) as u8; 3]).collect(), - ImageRgb16(p) => p - .pixels() - .flat_map(|v| [(v[0] >> 8) as u8, (v[1] >> 8) as u8, (v[2] >> 8) as u8]) - .collect(), - ImageRgba16(p) => p - .pixels() - .flat_map(|v| [(v[0] >> 8) as u8, (v[1] >> 8) as u8, (v[2] >> 8) as u8]) - .collect(), - other => other.to_rgb8().into_raw(), - } -} - fn bounded_key(key: &str) -> Result<()> { ensure!( (1..=256).contains(&key.chars().count()), @@ -221,6 +119,7 @@ fn checked_text(value: &Value) -> Result { #[cfg(test)] mod tests { use super::*; + use base64::Engine; use serde_json::json; fn parse_jpeg(raw: &[u8]) -> Result { diff --git a/src/models/jev_vl/native/src/executor.rs b/src/models/jev_vl/native/src/executor.rs index 461df3aa..d3f8d859 100644 --- a/src/models/jev_vl/native/src/executor.rs +++ b/src/models/jev_vl/native/src/executor.rs @@ -179,38 +179,38 @@ impl LabelHead { } } -/// One image block in the expanded request (adapted rows borrowed from L2). -pub struct ImgBlock { +/// One image block, with CPU input or resolved adapted rows borrowed from L2. +pub struct ImgBlock> { /// Global expanded-id row of the first pad. pub start: usize, /// Global expanded-id row one past the last pad. pub end: usize, - /// Parsed adapter output (grid + rows), shared with the L2 cache. - pub asset: Arc, + /// Image input before resolution, or the completed grid + adapted rows. + pub asset: T, } /// Full expanded ids + absolute positions + image blocks (global coordinates). -pub struct PreparedMm { +pub struct PreparedMm> { pub ids: Vec, pub positions: [Vec; 3], - pub blocks: Vec, + pub blocks: Vec>, } /// The cached-prefix continuation slice of one request. -pub struct ContinueMm { +pub struct ContinueMm> { /// Rows [p, T): (pads_end - p) pads, <|vision_end|>, fresh tail tokens. pub ids: Vec, /// Absolute positions for those rows. pub positions: [Vec; 3], /// The image rows within the slice (the last block's tail), if any. - pub block: Option, + pub block: Option>, /// L3 device state of the prefix. pub state: Arc, } /// The image rows of one continuation slice: an offset range of one asset's rows. -pub struct SuffixBlock { - pub asset: Arc, +pub struct SuffixBlock> { + pub asset: T, /// First adapted-row index feeding the slice (= p - pads_start). pub row_offset: usize, /// Number of rows = number of pads leading the slice. @@ -218,20 +218,20 @@ pub struct SuffixBlock { } /// The inputs and cached state needed to execute one request. -pub enum MmPlan { +pub enum MmPlan> { /// Text-only prompt. Text { ids: Vec }, /// Full multimodal prefill. - Full(PreparedMm), + Full(PreparedMm), /// Structure recorded but its device state is not built yet: run the capture /// phase over rows [0, p) followed by the continuation over [p, T), then /// publish the state so later requests go straight to `Continue`. Populate { - mm: PreparedMm, + mm: PreparedMm, record: Arc, }, /// Cached-prefix hit: only the continuation slice runs. - Continue(ContinueMm), + Continue(ContinueMm), } pub struct Executor { diff --git a/src/models/jev_vl/native/src/images.rs b/src/models/jev_vl/native/src/images.rs index d1089540..1c16e652 100644 --- a/src/models/jev_vl/native/src/images.rs +++ b/src/models/jev_vl/native/src/images.rs @@ -6,6 +6,26 @@ use std::path::Path; use std::sync::Arc; use anyhow::{Context, Result, ensure}; +use omni_qwen3_5_native::image_preprocess::ProcessedImage; + +/// CPU preparation retains pixels until the serialized executor runs vision. +#[derive(Clone)] +pub enum ImageInput { + Ready(Arc), + Inline { + key: String, + pixels: Arc, + }, +} + +impl ImageInput { + pub fn grid_thw(&self) -> [i64; 3] { + match self { + Self::Ready(asset) => asset.grid_thw, + Self::Inline { pixels, .. } => pixels.image_grid_thw.map(|n| n as i64), + } + } +} /// One adapted image: merger output rows in placeholder order + the patch grid. pub struct ImageAsset { @@ -121,16 +141,25 @@ pub fn meshgrid_positions(grid: [i64; 3], base: i64, from: usize, rows: usize) - /// running start position; the next text token continues at /// start + max(grid_h, grid_w) / merge. pub fn expand(text_ids: &[u32], image_pad: u32, assets: &[Arc]) -> Result { + expand_grids( + text_ids, + image_pad, + &assets.iter().map(|a| a.grid_thw).collect::>(), + ) +} + +/// Expand from CPU-validated geometry, before any online vision forward. +pub fn expand_grids(text_ids: &[u32], image_pad: u32, grids: &[[i64; 3]]) -> Result { let marks: Vec = text_ids .iter() .enumerate() .filter_map(|(i, &id)| (id == image_pad).then_some(i)) .collect(); ensure!( - marks.len() == assets.len(), + marks.len() == grids.len(), "prompt has {} image placeholders but the request carries {} images", marks.len(), - assets.len() + grids.len() ); let mut ids: Vec = Vec::new(); let mut positions: [Vec; 3] = [Vec::new(), Vec::new(), Vec::new()]; @@ -146,17 +175,20 @@ pub fn expand(text_ids: &[u32], image_pad: u32, assets: &[Arc]) -> R ids.extend_from_slice(&text_ids[cursor..mark]); current += (mark - cursor) as i64; cursor = mark + 1; - let [gt, gh, gw] = assets[k].grid_thw; + let [gt, gh, gw] = grids[k]; + let tokens = gh.checked_mul(gw).and_then(|n| n.checked_div(4)); ensure!( - gt >= 1 + gt == 1 + && gh > 0 + && gw > 0 && gh % 2 == 0 && gw % 2 == 0 - && gt * gh * gw / 4 == assets[k].n_tokens() as i64, - "grid does not match the adapted rows" + && tokens.is_some_and(|n| n <= 32768), + "expected one image with a bounded, even, positive spatial grid" ); let (lg_t, lg_h, lg_w) = (gt, gh / 2, gw / 2); // temp_merge=1, spatial_merge=2 let start = ids.len(); - for _ in 0..assets[k].n_tokens() { + for _ in 0..tokens.unwrap() { ids.push(image_pad); } blocks.push(ImageBlock { diff --git a/src/models/jev_vl/native/src/processing.rs b/src/models/jev_vl/native/src/processing.rs index 7ac5b557..cd3c8aaa 100644 --- a/src/models/jev_vl/native/src/processing.rs +++ b/src/models/jev_vl/native/src/processing.rs @@ -303,6 +303,17 @@ impl Processor { ) } + fn check_length(&self, t: usize) -> Result<(), Reject> { + if t > self.max_length { + return Err(Reject::bad_request(format!( + "This model's maximum context length is {} tokens. However, you requested 1 output tokens and your prompt contains at least {t} input tokens, for a total of at least {} tokens. Please reduce the length of the input prompt or the number of requested output tokens. (parameter=input_tokens, value={t})", + self.max_length, + t + 1 + ))); + } + Ok(()) + } + fn finish_c( &self, input_tokens: usize, @@ -312,14 +323,7 @@ impl Processor { start: Instant, cache_note: String, ) -> Result { - let t = input_tokens; - if t > self.max_length { - return Err(Reject::bad_request(format!( - "This model's maximum context length is {} tokens. However, you requested 1 output tokens and your prompt contains at least {t} input tokens, for a total of at least {} tokens. Please reduce the length of the input prompt or the number of requested output tokens. (parameter=input_tokens, value={t})", - self.max_length, - t + 1 - ))); - } + self.check_length(input_tokens)?; Ok(PreparedRequest { plan, readout, @@ -327,7 +331,7 @@ impl Processor { context: ResponseContext { kind: compiled.kind, options: compiled.options.clone(), - input_tokens: t, + input_tokens, start, }, }) diff --git a/src/models/qwen3_5/native/Cargo.toml b/src/models/qwen3_5/native/Cargo.toml index 4e9ad660..c127c759 100644 --- a/src/models/qwen3_5/native/Cargo.toml +++ b/src/models/qwen3_5/native/Cargo.toml @@ -7,7 +7,9 @@ description = "Shared Qwen3.5/3.8 prefill implementation for native decision wor [dependencies] anyhow = "1.0.100" +base64 = "0.22.1" half = "2.7.1" +image = { version = "0.25", default-features = false, features = ["png", "jpeg"] } libloading = "0.8" memmap2 = "0.9.9" safetensors = "0.8.0" @@ -21,3 +23,7 @@ path = "../../../../tests/qwen3_5/config.rs" [[test]] name = "kernels" path = "../../../../tests/qwen3_5/kernels.rs" + +[[test]] +name = "image_decode" +path = "../../../../tests/qwen3_5/image_decode.rs" diff --git a/src/models/qwen3_5/native/src/image_decode.rs b/src/models/qwen3_5/native/src/image_decode.rs new file mode 100644 index 00000000..536a0c98 --- /dev/null +++ b/src/models/qwen3_5/native/src/image_decode.rs @@ -0,0 +1,120 @@ +//! Bounded single-frame PNG/JPEG data URL decoding shared by native workers. +use anyhow::{Context, Result, ensure}; +use base64::Engine; +use std::io::Cursor; + +pub const MAX_IMAGE_BYTES: usize = 4 * 1024 * 1024; + +pub struct DecodedImage { + pub width: usize, + pub height: usize, + pub rgb: Vec, +} + +pub fn decode_data_url(url: &str) -> Result { + let (prefix, encoded) = url.split_once(',').context("invalid image data URL")?; + let format = match prefix { + "data:image/png;base64" => image::ImageFormat::Png, + "data:image/jpeg;base64" => image::ImageFormat::Jpeg, + _ => anyhow::bail!("only inline PNG/JPEG images are supported"), + }; + ensure!( + encoded.len() <= 4 * MAX_IMAGE_BYTES.div_ceil(3), + "encoded image exceeds 4 MiB" + ); + let raw = base64::engine::general_purpose::STANDARD + .decode(encoded) + .context("invalid base64 image")?; + ensure!(raw.len() <= MAX_IMAGE_BYTES, "image exceeds 4 MiB"); + ensure!( + image::guess_format(&raw)? == format, + "image format does not match MIME type" + ); + if format == image::ImageFormat::Jpeg { + ensure_single_jpeg(&raw)?; + } + let mut limits = image::Limits::default(); + limits.max_image_width = Some(2048); + limits.max_image_height = Some(2048); + limits.max_alloc = Some(32 * 1024 * 1024); + let mut header = image::ImageReader::with_format(Cursor::new(&raw), format); + header.limits(limits.clone()); + let (width, height) = header.into_dimensions()?; + let (w, h) = (width as usize, height as usize); + ensure!( + w > 0 && h > 0 && w <= 2048 && h <= 2048 && w * h <= 1048576 && w.max(h) <= 200 * w.min(h), + "image dimensions exceed supported limits" + ); + let decoded = if format == image::ImageFormat::Png { + let decoder = image::codecs::png::PngDecoder::with_limits(Cursor::new(&raw), limits)?; + ensure!(!decoder.is_apng()?, "image must be single-frame"); + image::DynamicImage::from_decoder(decoder)? + } else { + let mut reader = image::ImageReader::with_format(Cursor::new(&raw), format); + reader.limits(limits); + reader.decode()? + }; + Ok(DecodedImage { + width: w, + height: h, + rgb: pillow_rgb( + decoded, + format == image::ImageFormat::Png && raw.get(25) == Some(&0), + ), + }) +} + +fn ensure_single_jpeg(raw: &[u8]) -> Result<()> { + // MPF APP2 identifies an MPO container. Pillow reports it as MPO rather + // than JPEG; decoding it as JPEG would silently select the first picture. + // Walk header segments only, so arbitrary metadata/entropy bytes cannot + // be mistaken for an MPF marker. + let mut offset = 2; // SOI was checked by guess_format. + while offset < raw.len() { + ensure!(raw[offset] == 0xff, "invalid JPEG marker"); + while raw.get(offset) == Some(&0xff) { + offset += 1; + } + let marker = *raw.get(offset).context("truncated JPEG marker")?; + offset += 1; + match marker { + 0xda | 0xd9 => break, // SOS / EOI; the decoder checks the rest. + 0x01 | 0xd0..=0xd8 => continue, // Standalone markers. + _ => {} + } + let length = raw + .get(offset..offset + 2) + .context("truncated JPEG segment")?; + let length = u16::from_be_bytes([length[0], length[1]]) as usize; + ensure!(length >= 2, "invalid JPEG segment length"); + let segment = raw + .get(offset + 2..offset + length) + .context("truncated JPEG segment")?; + ensure!( + marker != 0xe2 || !segment.starts_with(b"MPF\0"), + "image must be single-frame; MPF/MPO containers are unsupported" + ); + offset += length; + } + Ok(()) +} + +fn pillow_rgb(image: image::DynamicImage, png_grayscale: bool) -> Vec { + use image::DynamicImage::*; + match image { + ImageLuma16(p) => p.pixels().flat_map(|v| [v[0].min(255) as u8; 3]).collect(), + ImageLumaA16(p) if png_grayscale => { + p.pixels().flat_map(|v| [v[0].min(255) as u8; 3]).collect() + } + ImageLumaA16(p) => p.pixels().flat_map(|v| [(v[0] >> 8) as u8; 3]).collect(), + ImageRgb16(p) => p + .pixels() + .flat_map(|v| [(v[0] >> 8) as u8, (v[1] >> 8) as u8, (v[2] >> 8) as u8]) + .collect(), + ImageRgba16(p) => p + .pixels() + .flat_map(|v| [(v[0] >> 8) as u8, (v[1] >> 8) as u8, (v[2] >> 8) as u8]) + .collect(), + other => other.to_rgb8().into_raw(), + } +} diff --git a/src/models/qwen3_5/native/src/lib.rs b/src/models/qwen3_5/native/src/lib.rs index d63f746a..77f984e3 100644 --- a/src/models/qwen3_5/native/src/lib.rs +++ b/src/models/qwen3_5/native/src/lib.rs @@ -6,5 +6,6 @@ pub mod model; pub mod inputs; +pub mod image_decode; pub mod image_preprocess; pub mod vision; diff --git a/tests/jev_vl/images.rs b/tests/jev_vl/images.rs index da54b544..b1879f06 100644 --- a/tests/jev_vl/images.rs +++ b/tests/jev_vl/images.rs @@ -4,7 +4,7 @@ use std::sync::Arc; -use omni_jev_vl_native::images::{ImageAsset, expand, meshgrid_positions}; +use omni_jev_vl_native::images::{ImageAsset, expand, expand_grids, meshgrid_positions}; const PAD: u32 = 248056; @@ -75,6 +75,30 @@ fn placeholder_count_must_match_images() { assert!(e.is_err()); } +#[test] +fn grid_only_expansion_matches_prepared_assets() { + let text_ids = [300, PAD, 301, PAD, 302]; + let grids = [[1, 4, 6], [1, 6, 4]]; + let ready = expand(&text_ids, PAD, &grids.map(asset)).unwrap(); + let inline = expand_grids(&text_ids, PAD, &grids).unwrap(); + assert_eq!(ready.ids, inline.ids); + assert_eq!(ready.positions, inline.positions); + assert_eq!(ready.blocks.len(), inline.blocks.len()); +} + +#[test] +fn grid_only_expansion_rejects_invalid_and_overflowing_shapes() { + for grid in [ + [1, 0, 2], + [1, -2, 2], + [2, 2, 2], + [1, 3, 2], + [1, i64::MAX - 1, 4], + ] { + assert!(expand_grids(&[PAD], PAD, &[grid]).is_err()); + } +} + #[test] fn image_assets_reject_wrong_sources_and_invalid_grids() { let dir = tempfile::tempdir().unwrap(); diff --git a/tests/qwen3_5/image_decode.rs b/tests/qwen3_5/image_decode.rs new file mode 100644 index 00000000..bc2d88ad --- /dev/null +++ b/tests/qwen3_5/image_decode.rs @@ -0,0 +1,139 @@ +use base64::Engine; +use image::{DynamicImage, ImageBuffer, ImageFormat}; +use omni_qwen3_5_native::image_decode::{MAX_IMAGE_BYTES, decode_data_url}; +use std::io::Cursor; + +fn data_url(mime: &str, raw: &[u8]) -> String { + format!( + "data:image/{mime};base64,{}", + base64::engine::general_purpose::STANDARD.encode(raw) + ) +} + +fn png(image: DynamicImage) -> Vec { + let mut raw = Cursor::new(Vec::new()); + image.write_to(&mut raw, ImageFormat::Png).unwrap(); + raw.into_inner() +} + +fn rejects(url: &str, message: &str) { + let error = decode_data_url(url).err().expect("image must be rejected"); + assert!(error.to_string().contains(message), "{error}"); +} + +#[test] +fn decodes_png_and_jpeg_to_rgb() { + let raw = png(DynamicImage::ImageRgba8( + ImageBuffer::from_raw(2, 1, vec![20, 40, 60, 0, 80, 100, 120, 128]).unwrap(), + )); + let decoded = decode_data_url(&data_url("png", &raw)).unwrap(); + assert_eq!((decoded.width, decoded.height), (2, 1)); + assert_eq!(decoded.rgb, [20, 40, 60, 80, 100, 120]); + + let mut raw = Vec::new(); + image::codecs::jpeg::JpegEncoder::new(&mut raw) + .encode(&[20, 40, 60], 1, 1, image::ExtendedColorType::Rgb8) + .unwrap(); + let decoded = decode_data_url(&data_url("jpeg", &raw)).unwrap(); + assert_eq!((decoded.width, decoded.height), (1, 1)); + assert_eq!(decoded.rgb.len(), 3); +} + +#[test] +fn rejects_non_inline_urls_invalid_base64_and_mime_mismatch() { + rejects("https://example.com/image.png", "invalid image data URL"); + rejects("data:image/webp;base64,AAAA", "only inline PNG/JPEG"); + rejects("data:image/png;base64,%%%", "invalid base64 image"); + let raw = png(DynamicImage::new_rgb8(1, 1)); + rejects(&data_url("jpeg", &raw), "does not match MIME type"); +} + +#[test] +fn enforces_encoded_and_decoded_four_mib_limits() { + let mut raw = png(DynamicImage::new_rgb8(1, 1)); + raw.resize(MAX_IMAGE_BYTES, 0); + assert!(decode_data_url(&data_url("png", &raw)).is_ok()); + // Both lengths encode to the same base64 length, exercising the raw limit. + raw.push(0); + rejects(&data_url("png", &raw), "image exceeds 4 MiB"); + rejects( + &format!( + "data:image/png;base64,{}", + "A".repeat(4 * MAX_IMAGE_BYTES.div_ceil(3) + 4) + ), + "encoded image exceeds 4 MiB", + ); +} + +#[test] +fn enforces_source_dimensions_area_and_aspect_ratio() { + for (width, height, accepted) in [ + (2048, 512, true), + (2049, 400, false), + (400, 2049, false), + (1024, 1024, true), + (1024, 1025, false), + (200, 1, true), + (201, 1, false), + ] { + let raw = png(DynamicImage::new_rgb8(width, height)); + assert_eq!( + decode_data_url(&data_url("png", &raw)).is_ok(), + accepted, + "{width}x{height}" + ); + } +} + +#[test] +fn rejects_apng_instead_of_silently_selecting_a_frame() { + // Pillow: RGB 1x1 red.save(format="PNG", save_all=True, + // append_images=[RGB 1x1 blue], duration=100, loop=0). + rejects( + concat!( + "data:image/png;base64,", + "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAIAAACQd1PeAAAACGFjVEwAAAACAAAAAPONk3", + "AAAAAaZmNUTAAAAAAAAAABAAAAAQAAAAAAAAAAAAEACgAAWn8w0AAAAAxJREFUeJxj+M/A", + "AAADAQEAyf6S7wAAABpmY1RMAAAAAQAAAAEAAAABAAAAAAAAAAAAAQAKAADBDNoEAAAAEG", + "ZkQVQAAAACeJxjYGD4DwABAwEAaL/t9AAAAABJRU5ErkJggg==" + ), + "single-frame", + ); +} + +#[test] +fn preserves_pillow_rgb_conversion_for_sixteen_bit_png() { + for (image, expected) in [ + ( + DynamicImage::ImageLuma16(ImageBuffer::from_raw(3, 1, vec![128, 256, 65535]).unwrap()), + vec![128, 128, 128, 255, 255, 255, 255, 255, 255], + ), + ( + DynamicImage::ImageLumaA16(ImageBuffer::from_raw(1, 1, vec![0x1234, 0]).unwrap()), + vec![0x12; 3], + ), + ( + DynamicImage::ImageRgb16( + ImageBuffer::from_raw(1, 1, vec![0x1234, 0x5678, 0x9abc]).unwrap(), + ), + vec![0x12, 0x56, 0x9a], + ), + ( + DynamicImage::ImageRgba16( + ImageBuffer::from_raw(1, 1, vec![0x1234, 0x5678, 0x9abc, 0]).unwrap(), + ), + vec![0x12, 0x56, 0x9a], + ), + ] { + let decoded = decode_data_url(&data_url("png", &png(image))).unwrap(); + assert_eq!(decoded.rgb, expected); + } + // Pillow I;16 with tRNS keeps grayscale's clamp-to-255 conversion. + let decoded = decode_data_url(concat!( + "data:image/png;base64,", + "iVBORw0KGgoAAAANSUhEUgAAAAMAAAABEAAAAABuG5crAAAAAnRSTlMBAG+I/HkAAAAP", + "SURBVHicY2BoYGT4/x8ABYgCgNJVtjQAAAAASUVORK5CYII=" + )) + .unwrap(); + assert_eq!(decoded.rgb, [128, 128, 128, 255, 255, 255, 255, 255, 255]); +} From c47a90916e899898aabb2bf001f341832d236642 Mon Sep 17 00:00:00 2001 From: linear3735 Date: Fri, 9 Oct 2026 10:33:42 +0800 Subject: [PATCH 02/20] Enable online vision in the JEV-VL worker --- recipe/jev_vl/README.md | 42 +++- recipe/jev_vl/preencode.py | 2 +- src/models/jev_vl/README.md | 18 +- src/models/jev_vl/native/src/caches.rs | 5 +- src/models/jev_vl/native/src/engine.rs | 25 +- src/models/jev_vl/native/src/executor.rs | 131 ++++++++++- src/models/jev_vl/native/src/images.rs | 7 + src/models/jev_vl/native/src/lib.rs | 1 + src/models/jev_vl/native/src/processing.rs | 134 +++++++---- src/models/jev_vl/native/src/vision.rs | 176 ++++++++++++++ tests/jev_vl/processing.rs | 257 ++++++++++++++++++++- tests/jev_vl/vision.rs | 155 +++++++++++++ 12 files changed, 877 insertions(+), 76 deletions(-) create mode 100644 src/models/jev_vl/native/src/vision.rs create mode 100644 tests/jev_vl/vision.rs diff --git a/recipe/jev_vl/README.md b/recipe/jev_vl/README.md index 6304c364..96264686 100644 --- a/recipe/jev_vl/README.md +++ b/recipe/jev_vl/README.md @@ -1,7 +1,8 @@ # JEV-27B-VL experimental native recipe This recipe serves `autotrust/JEV-27B-VL` System-1 decisions with Rust/CUDA. -Text runs natively; images must first be encoded offline by Transformers. +Text runs natively. Images can use the shared native vision backend or assets +encoded offline by Transformers. The [model contract](../../src/models/jev_vl/README.md) explains the single-question API, verbalizer head and cache ownership. The reviewed candidate passed a [bounded H800 validation](validation.md#historical-h800-validation); @@ -81,10 +82,41 @@ checks. The frontend routes `/health` and `/v1/systemone`; it does not proxy generation routes such as `/v1/chat/completions`. An unknown route returns the frontend's own 404 rather than this worker's error envelope. -## Prepare images before serving +## Online images -The worker does not decode images, fetch URLs or invoke a vision tower on cache -miss. First prepare a JSONL manifest with one object per line containing a +Export the vision tower separately from the same pinned source checkpoint. +The CPU exporter preserves BF16 vision weights and pins the processor and source +files. It does not rewrite the language export: + +```sh +CUDA_VISIBLE_DEVICES='' .venv-jev-vl/bin/python recipe/jev_vl/export_vision.py \ + --model weights/JEV-27B-VL --out weights/jev-vl-vision +JEV_VL_MODEL=weights/jev-vl-merged JEV_VL_VISION=weights/jev-vl-vision \ + JEV_VL_CUDA_LIB=$PWD/target/release/libqwen3_5_cuda.so \ + JEV_VL_CACHE=1 target/release/omni-jev-vl-native +``` + +Online mode accepts single-frame inline PNG/JPEG data URIs. Each image is bounded +to 4 MiB of decoded file bytes, 2048 pixels per side, 1,048,576 source pixels, +and 4608 vision patches. The full HTTP body is limited to 4 MiB, including base64 +and text. Remote URLs, animations and video are unsupported. Oversized inputs +are rejected; resource limits do not change the model's resize policy. + +`JEV_VL_VISION` and `JEV_VL_IMGCACHE` are mutually exclusive. Online mode also +ignores the default `imgcache/` directory. The source and processor stay fixed +for the worker lifetime. A new image runs decode, preprocessing and vision +inside the request; repeated images reuse the bounded L2 cache. Vision and +language execution share the existing serial scheduler. Rebuild the CUDA library +to include the shared 27B vision entry points. + +Historical prepared-image measurements below do not validate this online path +or include its vision cost. Compare fresh images and repeated-image questions +separately against the official model before reporting online performance. + +## Prepared images + +Without `JEV_VL_VISION`, the worker reads prepared assets on cache miss. +First prepare a JSONL manifest with one object per line containing a `request` whose `state` includes `{"image":"data:image/png;base64,..."}`. The preencoder accepts base64 data URIs; a fresh URL or changed image needs a new asset. It runs on a GPU and should finish before starting the language worker @@ -111,7 +143,7 @@ not an end-to-end live screenshot service. | Variable | Default | Meaning | | --- | --- | --- | -| `JEV_VL_CACHE` | `1` | Master enable; `0` uses a full language forward and rereads prepared image assets. | +| `JEV_VL_CACHE` | `1` | Master enable; `0` uses a full language forward and recomputes online vision or rereads prepared assets. | | `JEV_VL_L1`, `JEV_VL_L2`, `JEV_VL_L3` | `1` | Processor records, parsed image assets, and language-prefix state. | | `JEV_VL_L1_MAX` | `256` | Maximum resident processor records. | | `JEV_VL_L2_BYTES` | `1073741824` | Parsed image-asset cache budget. | diff --git a/recipe/jev_vl/preencode.py b/recipe/jev_vl/preencode.py index 807f98bc..2a9725fe 100644 --- a/recipe/jev_vl/preencode.py +++ b/recipe/jev_vl/preencode.py @@ -5,7 +5,7 @@ ``//{emb.safetensors,grid.json}`` with the adapted rows (``model.model.visual(pixel_values, grid_thw).pooler_output``, shape [n = prod(grid)/merge^2, 5120], bfloat16) plus the patch grid. The L2 cache holds -these assets; the native worker never decodes images. +these assets in prepared-image mode. Online mode uses its own native vision tower. """ import argparse import hashlib diff --git a/src/models/jev_vl/README.md b/src/models/jev_vl/README.md index c6b3c703..66a692a2 100644 --- a/src/models/jev_vl/README.md +++ b/src/models/jev_vl/README.md @@ -1,7 +1,8 @@ # JEV-27B-VL experimental native worker This integration targets **autotrust/JEV-27B-VL**, with a native Rust/CUDA -language backbone and an offline Hugging Face vision encoder. The reviewed +language backbone and the shared native Qwen vision encoder. Prepared embeddings +from the offline Hugging Face encoder remain available as a separate mode. The reviewed integrated candidate passed the frozen-corpus H800 checks; it remains experimental and has not completed broad quality or clean-install validation. See the [recipe](../../../recipe/jev_vl/README.md) and [evidence and limitations](../../../recipe/jev_vl/validation.md). @@ -23,8 +24,8 @@ This checkpoint's verbalizer head and prompt differ from ZefanCai/Open-Jev-27B-v1.1's independent-candidate scalar head. It is also distinct from [openjev/openjev issue #95](https://github.com/ThinkFlowLab/system1-omni/issues/95). [Cua-S1 PR #64](https://github.com/ThinkFlowLab/system1-omni/pull/64) supplies -upstream native vision infrastructure; this JEV worker currently consumes -preencoded embeddings and does not call that native vision tower. +upstream native vision infrastructure. [Shared vision PR #117](https://github.com/ThinkFlowLab/system1-omni/pull/117) +provides the Qwen vision backend used by this worker's online mode. [Shared-observation RFC #85](https://github.com/ThinkFlowLab/system1-omni/issues/85) is related design context, not this model's acceptance specification. @@ -62,7 +63,8 @@ list parts concatenate without an inserted separator. Each image part inserts The default exported limit is 16,384 expanded tokens. Export permits 1–32,768; larger limits have not been benchmarked. HTTP bodies are limited to 4 MiB. `thinking=auto/on`, tournament and permutation strategies, generation, audio, -video, dynamic batching and unprepared images are unsupported. `strategy=auto` +video and dynamic batching are unsupported. Online mode accepts inline PNG/JPEG +images; prepared mode requires exported image assets. `strategy=auto` uses the single-pass path in this worker. The historical corpus covers only 2–16 choice options and one image, so the wider accepted range and multiple images do not have full-checkpoint parity evidence. @@ -70,7 +72,9 @@ images do not have full-checkpoint parity evidence. ## Execution, layouts and lifetimes `processing.rs` validates and tokenizes one question and prepares either text -IDs, full multimodal inputs, or a cached-prefix continuation. Image assets hold +IDs, full multimodal inputs, or a cached-prefix continuation. Online image pixels +are encoded by the vision model under the same scheduler admission as the language +forward. Image assets hold BF16 `[image_tokens, 5120]` adapted embeddings and their patch grid. The processor constructs expanded token IDs and three position axes; image rows replace placeholder-token embeddings. All vectors describe one unpadded prompt, @@ -92,10 +96,10 @@ finishes. Restart the worker and use a fresh asset directory when changing weights, tokenizer or preprocessing; caches are scoped to a loaded worker. Three configurable caches retain processor prefix geometry (L1), parsed -preencoded image assets (L2), and language-prefix device state (L3). The current +image embeddings (L2), and language-prefix device state (L3). The current L3 path targets a compatible single-image prefix aligned to a 64-token chunk; other inputs use a full forward. L2 avoids file parsing and copying of prepared -assets; it does not run or cache a live vision encoder. Cache sizes and controls +assets in prepared mode and vision encoding in online mode. Cache sizes and controls are listed in the [recipe](../../../recipe/jev_vl/README.md#cache-controls). Tests are registered under [`tests/jev_vl/`](../../../tests/jev_vl/) and diff --git a/src/models/jev_vl/native/src/caches.rs b/src/models/jev_vl/native/src/caches.rs index 99b0b4a6..521eb628 100644 --- a/src/models/jev_vl/native/src/caches.rs +++ b/src/models/jev_vl/native/src/caches.rs @@ -1,4 +1,4 @@ -//! Process-local caches for prepared images and reusable language prefixes. +//! Process-local caches for image embeddings and reusable language prefixes. //! //! - L1 processor: a per-prefix-structure record. Key = sha256 over the exact //! request structure (kind + every state part: text verbatim / image url). @@ -8,8 +8,7 @@ //! (`<|vision_end|>` onward) is always tokenized fresh per question. //! - L2 vision: parsed image assets (adapter output rows + grid) keyed by //! sha256(url) — the same key form as the offline imgcache directory. A hit -//! skips the disk read, safetensors parsing and validation. Vision encoding -//! runs offline and is not part of this cache lookup. +//! skips disk loading in prepared mode or vision encoding in online mode. //! - L3 KV prefix: the device-side prefix state (per-full-attention-layer //! post-prep K and raw V rows; per-GDN-layer float32 recurrent state + conv //! tail). Holders are `Arc`; a continuation reads it while the diff --git a/src/models/jev_vl/native/src/engine.rs b/src/models/jev_vl/native/src/engine.rs index 04d5da75..be7b5ea7 100644 --- a/src/models/jev_vl/native/src/engine.rs +++ b/src/models/jev_vl/native/src/engine.rs @@ -10,6 +10,7 @@ use serde_json::Value; use crate::caches::{CacheCfg, Caches}; use crate::executor::{Executor, LabelHead}; use crate::processing::Processor; +use crate::vision::{self, ImageSource}; pub struct Engine { pub manifest: Value, @@ -24,6 +25,11 @@ pub struct Engine { impl Engine { pub async fn load(dir: &Path, library: &Path) -> Result { + let source = ImageSource::from_paths( + dir, + std::env::var_os("JEV_VL_VISION").map(Into::into), + std::env::var_os("JEV_VL_IMGCACHE").map(Into::into), + )?; let manifest: Value = serde_json::from_slice( &std::fs::read(dir.join("jev_vl_export.json")) .context("export the merged checkpoint; see recipe/jev_vl/export_merged.py")?, @@ -64,14 +70,29 @@ impl Engine { (1..=32768).contains(&max_length), "max_length must be within 1..=32768" ); + let vision_dir = match &source { + ImageSource::Online(vision_dir) => { + vision::validate_export(vision_dir, dir, &manifest)?; + Some(vision_dir.clone()) + } + ImageSource::Prepared(_) => None, + }; let head = Arc::new(LabelHead::load(dir, &manifest)?); let caches = Caches::new(CacheCfg::from_env()); let model_index_hash = manifest["pins"]["model_index_sha256"] .as_str() .context("model_index_sha256")? .to_owned(); - let processor = Processor::load(dir, labels, max_length, model_index_hash, caches.clone())?; - let executor = Executor::load(dir, library, head.clone(), caches.clone()).await?; + let processor = Processor::load( + dir, + labels, + max_length, + model_index_hash, + caches.clone(), + source, + )?; + let executor = + Executor::load(dir, library, head.clone(), caches.clone(), vision_dir).await?; Ok(Self { manifest, head, diff --git a/src/models/jev_vl/native/src/executor.rs b/src/models/jev_vl/native/src/executor.rs index d3f8d859..dc5ad517 100644 --- a/src/models/jev_vl/native/src/executor.rs +++ b/src/models/jev_vl/native/src/executor.rs @@ -8,7 +8,7 @@ //! dot product per candidate token instead of a vocabulary GEMM. use std::collections::HashMap; -use std::path::Path; +use std::path::{Path, PathBuf}; use std::sync::{Arc, Mutex}; use omni_qwen3_5_native::inputs::MultimodalInput; @@ -18,7 +18,8 @@ use omni_qwen3_5_native::model::{Config, Model}; use omni_runtime::SerialScheduler; use serde_json::Value; -use crate::images::ImageAsset; +use crate::images::{ImageAsset, ImageInput}; +use omni_qwen3_5_native::vision::VisionModel; /// Per-request readout setup: which label tokens, with which bias, divided by /// which temperature, matching decision_head.json's slots + calibration.json. @@ -234,8 +235,13 @@ pub enum MmPlan> { Continue(ContinueMm), } +struct Models { + language: Model, + vision: Option, +} + pub struct Executor { - model: Arc>, + model: Arc>, head: Arc, caches: Arc, } @@ -246,9 +252,17 @@ impl Executor { library: &Path, head: Arc, caches: Arc, + vision_dir: Option, ) -> Result { let (d, lib) = (dir.to_path_buf(), library.to_path_buf()); - let model = tokio::task::spawn_blocking(move || Model::load(&d, &lib)).await??; + let model = tokio::task::spawn_blocking(move || -> Result { + let vision = vision_dir + .map(|dir| VisionModel::load(dir, &lib)) + .transpose()?; + let language = Model::load(&d, &lib)?; + Ok(Models { language, vision }) + }) + .await??; Ok(Self { model: Arc::new(Mutex::new(model)), head, @@ -261,7 +275,7 @@ impl Executor { pub async fn execute( &self, scheduler: &SerialScheduler, - plan: MmPlan, + plan: MmPlan, readout: Readout, ) -> Result> { let model = self.model.clone(); @@ -269,11 +283,48 @@ impl Executor { let caches = self.caches.clone(); scheduler .run(move || { - let mut model = model + let mut models = model .lock() .map_err(|_| anyhow::anyhow!("poisoned model"))?; + let Models { + language: model, + vision, + } = &mut *models; + let mut pending_images: Vec<(String, Arc)> = Vec::new(); + let mut pending_prefix = None; + let mut resolved_plan = None; let result = (|| { - let last = match &plan { + // Requests may have waited behind another request for this image. + // Pending assets also share duplicate references within this request + // when L2 retention is disabled or the asset exceeds its budget. + resolved_plan = Some(resolve_images(plan, |input| match input { + ImageInput::Ready(asset) => Ok(asset), + ImageInput::Inline { key, pixels } => { + if let Some(hit) = caches.l2_get(&key) { + return Ok(hit); + } + if let Some((_, asset)) = pending_images.iter().find(|(k, _)| k == &key) + { + return Ok(asset.clone()); + } + let embeddings = vision + .as_mut() + .context("online vision is not loaded")? + .forward(&pixels)?; + ensure!( + embeddings.len() == pixels.image_tokens() * 5120 + && embeddings.iter().all(|value| value.is_finite()), + "vision output does not match the processed image" + ); + let asset = Arc::new(ImageAsset { + grid_thw: pixels.image_grid_thw.map(|n| n as i64), + embeddings, + }); + pending_images.push((key, asset.clone())); + Ok(asset) + } + })?); + let last = match resolved_plan.as_ref().unwrap() { MmPlan::Text { ids } => model.forward(ids)?, MmPlan::Full(mm) => { let one = slice_mm(mm, 0, mm.ids.len()); @@ -300,8 +351,7 @@ impl Executor { }; match populated { Ok(last) => { - caches.record_publish_state(record.key, state.unwrap()); - caches.l3_populate(); + pending_prefix = Some((record.key, state.unwrap())); last } Err(e) => { @@ -343,13 +393,72 @@ impl Executor { head.probabilities(&last, &readout) })(); // Keep admission until all queued device work has drained, including errors. - model.synchronize()?; - result + let language_drain = model.synchronize(); + let vision_drain = vision.as_ref().map(VisionModel::synchronize).transpose(); + // Both drain attempts must happen even if either one fails. + language_drain?; + vision_drain?; + let probabilities = result?; + for (key, asset) in pending_images { + caches.l2_insert(key, asset); + } + if let Some((key, state)) = pending_prefix { + caches.record_publish_state(key, state); + caches.l3_populate(); + } + Ok(probabilities) }) .await } } +/// Resolve typed image inputs once, then use the existing language slicing path. +fn resolve_images( + plan: MmPlan, + mut resolve: impl FnMut(ImageInput) -> Result>, +) -> Result { + let mut full = |mm: PreparedMm| -> Result { + Ok(PreparedMm { + ids: mm.ids, + positions: mm.positions, + blocks: mm + .blocks + .into_iter() + .map(|b| { + Ok(ImgBlock { + start: b.start, + end: b.end, + asset: resolve(b.asset)?, + }) + }) + .collect::>()?, + }) + }; + Ok(match plan { + MmPlan::Text { ids } => MmPlan::Text { ids }, + MmPlan::Full(mm) => MmPlan::Full(full(mm)?), + MmPlan::Populate { mm, record } => MmPlan::Populate { + mm: full(mm)?, + record, + }, + MmPlan::Continue(c) => MmPlan::Continue(ContinueMm { + ids: c.ids, + positions: c.positions, + state: c.state, + block: c + .block + .map(|b| { + Ok::<_, anyhow::Error>(SuffixBlock { + asset: resolve(b.asset)?, + row_offset: b.row_offset, + rows: b.rows, + }) + }) + .transpose()?, + }), + }) +} + /// A slice of one request's expanded data for rows [from, to) in slice-local /// coordinates: borrowed ids/positions, local pad indices, and the adapted rows /// those pads need — borrowed straight out of the L2 asset when they form one diff --git a/src/models/jev_vl/native/src/images.rs b/src/models/jev_vl/native/src/images.rs index 1c16e652..83ee9386 100644 --- a/src/models/jev_vl/native/src/images.rs +++ b/src/models/jev_vl/native/src/images.rs @@ -19,6 +19,13 @@ pub enum ImageInput { } impl ImageInput { + pub(crate) fn n_tokens(&self) -> usize { + match self { + Self::Ready(asset) => asset.n_tokens(), + Self::Inline { pixels, .. } => pixels.image_tokens(), + } + } + pub fn grid_thw(&self) -> [i64; 3] { match self { Self::Ready(asset) => asset.grid_thw, diff --git a/src/models/jev_vl/native/src/lib.rs b/src/models/jev_vl/native/src/lib.rs index f34f4f88..79212428 100644 --- a/src/models/jev_vl/native/src/lib.rs +++ b/src/models/jev_vl/native/src/lib.rs @@ -9,3 +9,4 @@ pub mod engine; pub mod executor; pub mod images; pub mod processing; +mod vision; diff --git a/src/models/jev_vl/native/src/processing.rs b/src/models/jev_vl/native/src/processing.rs index cd3c8aaa..6918786b 100644 --- a/src/models/jev_vl/native/src/processing.rs +++ b/src/models/jev_vl/native/src/processing.rs @@ -1,6 +1,7 @@ //! Request validation, tokenization, and response assembly. -use std::path::{Path, PathBuf}; +use std::collections::HashMap; +use std::path::Path; use std::sync::Arc; use std::time::Instant; @@ -11,13 +12,18 @@ use tokenizers::Tokenizer; use crate::caches::{Caches, L1Meta, structure_key}; use crate::contract::{self, Compiled, Kind, Reject}; use crate::executor::{ContinueMm, ImgBlock, MmPlan, PreparedMm, Readout, SuffixBlock}; -use crate::images::{self, ImageAsset}; +use crate::images::{self, ImageAsset, ImageInput}; +use crate::vision::ImageSource; +use omni_qwen3_5_native::{ + image_decode::decode_data_url, + image_preprocess::{ImageLimits, preprocess_rgb8_with_limits}, +}; pub struct Processor { tokenizer: Tokenizer, labels: Vec, max_length: usize, - imgcache: Option, + source: ImageSource, model_index_hash: String, image_pad: u32, vision_end: u32, @@ -25,7 +31,7 @@ pub struct Processor { } pub struct PreparedRequest { - pub plan: MmPlan, + pub plan: MmPlan, pub readout: Readout, pub context: ResponseContext, /// x-jev-cache response-header marker: l1/l2/l3 hit flags + prefix length. @@ -47,6 +53,7 @@ impl Processor { max_length: usize, model_index_hash: String, caches: Arc, + source: ImageSource, ) -> Result { let tokenizer = Tokenizer::from_file(dir.join("tokenizer.json")).map_err(anyhow::Error::msg)?; @@ -56,16 +63,11 @@ impl Processor { let vision_end = tokenizer .token_to_id("<|vision_end|>") .context("tokenizer has no <|vision_end|>")?; - let hub = std::env::var("JEV_VL_IMGCACHE") - .ok() - .map(PathBuf::from) - .or_else(|| Some(dir.join("imgcache"))) - .filter(|p| p.is_dir()); Ok(Self { tokenizer, labels, max_length, - imgcache: hub, + source, model_index_hash, image_pad, vision_end, @@ -81,25 +83,44 @@ impl Processor { .collect() } - /// Load a prepared image asset, reusing the bounded L2 cache when enabled. - /// With caches disabled, each request reads and parses the asset again. - fn load_asset(&self, url: &str) -> Result, Reject> { + /// CPU image preparation checks the full prompt budget before GPU admission. + fn load_image(&self, url: &str) -> Result { let key = Self::url_key(url); if let Some(hit) = self.caches.l2_get(&key) { - return Ok(hit); + return Ok(ImageInput::Ready(hit)); } - let dir = match &self.imgcache { - Some(d) => d.join(&key), - None => { - return Err(Reject::bad_request( - "image input: no imgcache is configured for this worker", - )); + match &self.source { + ImageSource::Online(_) => { + let decoded = decode_data_url(url) + .map_err(|e| Reject::bad_request(format!("invalid image: {e:#}")))?; + let pixels = preprocess_rgb8_with_limits( + decoded.width, + decoded.height, + &decoded.rgb, + &ImageLimits::cua(), + ) + .map_err(|e| Reject::bad_request(format!("invalid image: {e:#}")))?; + Ok(ImageInput::Inline { + key, + pixels: Arc::new(pixels), + }) + } + ImageSource::Prepared(hub) => { + let dir = hub + .as_ref() + .ok_or_else(|| { + Reject::bad_request( + "image input: no imgcache is configured for this worker", + ) + })? + .join(&key); + let asset = ImageAsset::load(&dir, &key, &self.model_index_hash).map_err(|e| { + Reject::bad_request(format!("image input is not preencoded: {e}")) + })?; + self.caches.l2_insert(key, asset.clone()); + Ok(ImageInput::Ready(asset)) } - }; - let asset = ImageAsset::load(&dir, &key, &self.model_index_hash) - .map_err(|e| Reject::bad_request(format!("image input is not preencoded: {e}")))?; - self.caches.l2_insert(key, asset.clone()); - Ok(asset) + } } fn tokenize(&self, text: &str) -> Result, Reject> { @@ -111,6 +132,26 @@ impl Processor { .to_vec()) } + fn load_images(&self, text_tokens: usize, urls: &[String]) -> Result, Reject> { + let mut input_tokens = text_tokens; + self.check_length(input_tokens)?; + let mut seen: HashMap<&str, ImageInput> = HashMap::new(); + let mut assets = Vec::new(); + for url in urls { + let asset = match seen.get(url.as_str()) { + Some(asset) => asset.clone(), + None => self.load_image(url)?, + }; + // Each image replaces one placeholder. Count every reference, even + // when its pixels are shared, and stop before preparing another image. + input_tokens += asset.n_tokens() - 1; + self.check_length(input_tokens)?; + seen.entry(url).or_insert_with(|| asset.clone()); + assets.push(asset); + } + Ok(assets) + } + /// Validate, compile the raw prompt, and tokenize; add_special_tokens=false, /// like the official raw System-1 readout. Never truncates: oversize prompts /// get the upstream context-length rejection. @@ -136,13 +177,13 @@ impl Processor { "l1=-,l3=-,p=-".to_string(), ) } else { - let assets: Vec> = compiled - .images - .iter() - .map(|url| self.load_asset(url)) - .collect::, Reject>>()?; - let e = images::expand(&text_ids, self.image_pad, &assets) - .map_err(|e| Reject::bad_request(format!("invalid image prompt: {e:#}")))?; + let assets = self.load_images(text_ids.len(), &compiled.images)?; + let e = images::expand_grids( + &text_ids, + self.image_pad, + &assets.iter().map(ImageInput::grid_thw).collect::>(), + ) + .map_err(|e| Reject::bad_request(format!("invalid image prompt: {e:#}")))?; let n = e.ids.len(); ( MmPlan::Full(expanded_mm(e, &assets)), @@ -184,7 +225,6 @@ impl Processor { meta.base_pad, meta.advance, ); - let asset = self.load_asset(&compiled.images[0])?; // Suffix piece: pads(E-P) + <|vision_end|> + fresh tail tokenization. let tail = tail.unwrap(); let tail_ids = self.tokenize(&tail)?; @@ -194,19 +234,21 @@ impl Processor { )); } let suffix_pads = pads_end - p; + let input_tokens = p + suffix_pads + 1 + tail_ids.len(); + self.check_length(input_tokens)?; + let asset = self.load_image(&compiled.images[0])?; let mut ids = Vec::with_capacity(suffix_pads + 1 + tail_ids.len()); ids.extend(std::iter::repeat_n(self.image_pad, suffix_pads)); ids.push(self.vision_end); ids.extend_from_slice(&tail_ids); let mut pos = - images::meshgrid_positions(asset.grid_thw, base_pad, p - pads_start, suffix_pads); + images::meshgrid_positions(asset.grid_thw(), base_pad, p - pads_start, suffix_pads); let after = base_pad + advance; for k in 0..(ids.len() - suffix_pads) as i64 { pos[0].push(after + k); pos[1].push(after + k); pos[2].push(after + k); } - let input_tokens = p + ids.len(); match (cfg.l3, record.state()) { (true, Some(state)) => { self.caches.l3_hit(); @@ -256,13 +298,13 @@ impl Processor { } // L1 miss: full expansion (and the cache record when the shape qualifies). let text_ids = self.tokenize(&compiled.prompt)?; - let assets: Vec> = compiled - .images - .iter() - .map(|url| self.load_asset(url)) - .collect::, Reject>>()?; - let e = images::expand(&text_ids, self.image_pad, &assets) - .map_err(|e| Reject::bad_request(format!("invalid image prompt: {e:#}")))?; + let assets = self.load_images(text_ids.len(), &compiled.images)?; + let e = images::expand_grids( + &text_ids, + self.image_pad, + &assets.iter().map(ImageInput::grid_thw).collect::>(), + ) + .map_err(|e| Reject::bad_request(format!("invalid image prompt: {e:#}")))?; let input_tokens = e.ids.len(); let meta = (single_image && tail.is_some() && cfg.l1) .then(|| prefix_meta(&e, input_tokens)) @@ -317,7 +359,7 @@ impl Processor { fn finish_c( &self, input_tokens: usize, - plan: MmPlan, + plan: MmPlan, readout: Readout, compiled: &Compiled, start: Instant, @@ -363,7 +405,7 @@ fn prefix_meta(e: &images::Expanded, input_tokens: usize) -> Option { } /// Full expanded ids/positions with image blocks borrowed from their assets. -fn expanded_mm(e: images::Expanded, assets: &[Arc]) -> PreparedMm { +fn expanded_mm(e: images::Expanded, assets: &[ImageInput]) -> PreparedMm { let blocks = e .blocks .iter() @@ -386,8 +428,8 @@ fn rebuild_full( ids_suffix: &[u32], pos_suffix: &[Vec; 3], meta: &L1Meta, - asset: &Arc, -) -> PreparedMm { + asset: &ImageInput, +) -> PreparedMm { let mut ids = meta.ids_prefix.clone(); ids.extend_from_slice(ids_suffix); let mut positions = meta.positions_prefix.clone(); diff --git a/src/models/jev_vl/native/src/vision.rs b/src/models/jev_vl/native/src/vision.rs new file mode 100644 index 00000000..5f64d260 --- /dev/null +++ b/src/models/jev_vl/native/src/vision.rs @@ -0,0 +1,176 @@ +//! CPU validation of the pinned online vision export and its processor contract. + +use std::fs::File; +use std::io::Read; +use std::path::{Path, PathBuf}; + +use anyhow::{Context, Result, ensure}; +use omni_qwen3_5_native::vision::{VisionCheckpoint, VisionConfig}; +use serde_json::{Value, json}; +use sha2::{Digest, Sha256}; + +/// The image source is fixed for the lifetime of a worker. +pub(crate) enum ImageSource { + Prepared(Option), + Online(PathBuf), +} + +impl ImageSource { + pub(crate) fn from_paths( + dir: &Path, + vision: Option, + imgcache: Option, + ) -> Result { + ensure!( + vision.is_none() || imgcache.is_none(), + "JEV_VL_VISION and JEV_VL_IMGCACHE are mutually exclusive" + ); + if let Some(dir) = vision { + ensure!(dir.is_dir(), "JEV_VL_VISION must be an existing directory"); + Ok(Self::Online(dir)) + } else { + Ok(Self::Prepared( + imgcache + .or_else(|| Some(dir.join("imgcache"))) + .filter(|p| p.is_dir()), + )) + } + } +} + +fn sha256(path: &Path) -> Result { + let mut file = File::open(path).with_context(|| format!("open {}", path.display()))?; + let mut hash = Sha256::new(); + let mut buffer = [0u8; 64 * 1024]; + loop { + let n = file.read(&mut buffer)?; + if n == 0 { + break; + } + hash.update(&buffer[..n]); + } + Ok(hash + .finalize() + .iter() + .map(|byte| format!("{byte:02x}")) + .collect()) +} + +fn hash_value(value: &Value) -> Result<&str> { + let hash = value.as_str().context("expected SHA256 string")?; + ensure!( + hash.len() == 64 + && hash + .bytes() + .all(|b| b.is_ascii_hexdigit() && !b.is_ascii_uppercase()), + "expected lowercase SHA256" + ); + Ok(hash) +} + +fn validate_processor(processor: &Value) -> Result<()> { + // Frozen upstream metadata. New fields can change preprocessing semantics; + // accept them only after the shared processor implements those semantics. + ensure!( + processor + == &json!({ + "size": {"longest_edge": 16777216, "shortest_edge": 65536}, + "patch_size": 16, "temporal_patch_size": 2, "merge_size": 2, + "image_mean": [0.5, 0.5, 0.5], "image_std": [0.5, 0.5, 0.5], + "processor_class": "Qwen3VLProcessor", + "image_processor_type": "Qwen2VLImageProcessorFast" + }), + "unsupported JEV image processor metadata" + ); + Ok(()) +} + +fn validate_config(config: &Value, language: &Value) -> Result<()> { + ensure!( + config["model_type"] == "qwen3_5" + && language["model_type"] == "qwen3_5" + && config["vision_config"] == language["vision_config"] + && config["text_config"] == language["text_config"], + "vision and language configurations do not match" + ); + let vision = VisionConfig::from_value(config["vision_config"].clone())?; + let supported = serde_json::to_value(&vision)?; + for key in config["vision_config"] + .as_object() + .context("vision_config")? + .keys() + { + ensure!( + supported.get(key).is_some() || key == "initializer_range", + "unsupported vision configuration field: {key}" + ); + } + ensure!( + vision.hidden_size == 1152 && vision.out_hidden_size == 5120, + "online JEV vision requires the 27B layout" + ); + ensure!( + config["text_config"]["model_type"] == "qwen3_5_text" + && config["text_config"]["hidden_size"] == 5120 + && config["text_config"]["intermediate_size"] == 17408 + && config["text_config"]["num_hidden_layers"] == 64 + && config["text_config"]["num_attention_heads"] == 24 + && config["text_config"]["num_key_value_heads"] == 4 + && config["image_token_id"].as_u64().is_some() + && config["image_token_id"] == language["image_token_id"], + "online JEV vision requires the matching 27B text configuration" + ); + Ok(()) +} + +pub(crate) fn validate_export(dir: &Path, language_dir: &Path, language: &Value) -> Result<()> { + let manifest: Value = serde_json::from_slice(&std::fs::read(dir.join("jev_vl_vision.json"))?)?; + ensure!( + manifest["format"] == "jev-vl-vision/1" + && manifest["model_id"] == crate::contract::MODEL_ID, + "expected a pinned JEV-27B-VL vision export" + ); + for pin in [ + "model_index_sha256", + "adapter_config_sha256", + "adapter_model_sha256", + ] { + ensure!( + hash_value(&manifest["pins"][pin])? == hash_value(&language["pins"][pin])?, + "vision export source pin mismatch: {pin}" + ); + } + let files = manifest["files"] + .as_object() + .context("vision export files")?; + let expected = [ + "config.json", + "preprocessor_config.json", + "vision.safetensors", + ]; + ensure!( + files.len() == expected.len() && expected.iter().all(|name| files.contains_key(*name)), + "vision export must pin exactly config, processor and weights" + ); + for name in expected { + ensure!( + hash_value(&files[name])? == sha256(&dir.join(name))?, + "vision export file hash mismatch: {name}" + ); + } + let config = serde_json::from_slice(&std::fs::read(dir.join("config.json"))?)?; + let language_config = + serde_json::from_slice(&std::fs::read(language_dir.join("config.json"))?)?; + validate_config(&config, &language_config)?; + validate_processor(&serde_json::from_slice(&std::fs::read( + dir.join("preprocessor_config.json"), + )?)?)?; + // The shared loader validates the complete tensor inventory on CPU before + // either language or vision weights can be uploaded. + VisionCheckpoint::load(dir)?; + Ok(()) +} + +#[cfg(test)] +#[path = "../../../../../tests/jev_vl/vision.rs"] +mod tests; diff --git a/tests/jev_vl/processing.rs b/tests/jev_vl/processing.rs index c2453504..ed8627e6 100644 --- a/tests/jev_vl/processing.rs +++ b/tests/jev_vl/processing.rs @@ -41,7 +41,7 @@ fn processor() -> Processor { tokenizer, labels: vec!["A".into(), "B".into()], max_length: 256, - imgcache: None, + source: ImageSource::Prepared(None), model_index_hash: "test".into(), caches, } @@ -93,3 +93,258 @@ fn image_placeholders_are_rejected_before_and_after_l1_warmup() { ); } } + +const RED: &str = "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAIAAACQd1PeAAAADElEQVR4nGP4z8AAAAMBAQDJ/pLvAAAAAElFTkSuQmCC"; +const GREEN: &str = "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAIAAACQd1PeAAAADElEQVR4nGNg+M8AAAICAQB7CYF4AAAAAElFTkSuQmCC"; + +fn image_request(images: &[&str]) -> Value { + serde_json::json!({ + "kind": "choice", "state": images.iter().map(|url| serde_json::json!({"image": url})).collect::>(), + "question": "Pick one.", "options": ["yes", "no"], + }) +} + +fn prepare_request(processor: &Processor, raw: &Value) -> Result { + use crate::executor::LabelHead; + use safetensors::{Dtype, tensor::TensorView}; + use std::sync::OnceLock; + + // Exercise the public cache-on/off dispatch with a CPU-only label-head export. + static HEAD: OnceLock<(LabelHead, Value)> = OnceLock::new(); + let (head, manifest) = HEAD.get_or_init(|| { + let dir = tempfile::tempdir().unwrap(); + std::fs::write( + dir.path().join("config.json"), + include_str!("../qwen3_5/data/config.json"), + ) + .unwrap(); + let rows = vec![0u8; 2 * 5120 * 4]; + let ids: Vec<_> = [0i64, 1].into_iter().flat_map(i64::to_le_bytes).collect(); + safetensors::serialize_to_file( + [ + ( + "rows", + TensorView::new(Dtype::F32, vec![2, 5120], &rows).unwrap(), + ), + ("ids", TensorView::new(Dtype::I64, vec![2], &ids).unwrap()), + ], + None, + &dir.path().join("head.safetensors"), + ) + .unwrap(); + let manifest = serde_json::json!({ + "label_head": {"file": "head.safetensors", "ids": [0, 1]}, + "label_ids": [0, 1], "verbalizer_ids": vec![0; 24], + "verbalizer_bias": vec![0.0; 24], "temperatures": {"choice": 1.0}, + }); + (LabelHead::load(dir.path(), &manifest).unwrap(), manifest) + }); + processor.prepare(head, manifest, &serde_json::to_vec(raw).unwrap()) +} + +fn prepare_online(processor: &Processor, images: &[&str]) -> Result { + prepare_request(processor, &image_request(images)) +} + +fn online_processor(enabled: bool, l2: bool) -> Processor { + let mut processor = processor(); + processor.source = ImageSource::Online("unused-by-cpu".into()); + processor.caches = Caches::new(CacheCfg { + enabled, + l1: true, + l2, + l3: false, + l1_max: 8, + l2_bytes: 1 << 20, + l3_bytes: 0, + }); + processor +} + +#[test] +fn online_preparation_has_pixels_not_empty_embeddings_and_preserves_l1_geometry() { + let mut processor = processor(); + processor.source = ImageSource::Online("unused-by-cpu".into()); + let cold = prepare_online(&processor, &[RED]).unwrap(); + let warm = prepare_online(&processor, &[RED]).unwrap(); + assert!(cold.cache_note.contains("l1=miss")); + assert!(warm.cache_note.contains("l1=hit")); + let (MmPlan::Full(cold), MmPlan::Full(warm)) = (cold.plan, warm.plan) else { + panic!("expected full plans") + }; + assert_eq!(cold.ids, warm.ids); + assert_eq!(cold.positions, warm.positions); + let ImageInput::Inline { pixels, .. } = &warm.blocks[0].asset else { + panic!("CPU must not claim ready embeddings") + }; + assert_eq!(pixels.image_grid_thw, [1, 16, 16]); + assert_eq!(pixels.image_tokens(), 64); + assert_eq!(warm.blocks[0].end - warm.blocks[0].start, 64); +} + +#[test] +fn same_size_images_keep_distinct_keys_and_repeated_references_keep_all_blocks() { + let mut processor = processor(); + processor.source = ImageSource::Online("unused-by-cpu".into()); + let plan = prepare_online(&processor, &[RED, GREEN, RED]).unwrap().plan; + let MmPlan::Full(mm) = plan else { + panic!("expected full plan") + }; + let keys: Vec<_> = mm + .blocks + .iter() + .map(|b| match &b.asset { + ImageInput::Inline { key, .. } => key, + _ => panic!("expected inline image"), + }) + .collect(); + assert_eq!(keys.len(), 3); + assert_eq!(keys[0], keys[2]); + assert_ne!(keys[0], keys[1]); + assert_eq!( + mm.blocks.iter().map(|b| b.end - b.start).sum::(), + 192 + ); +} + +#[test] +fn online_rejects_unsupported_urls_and_full_context_overflow_before_admission() { + let mut processor = processor(); + processor.source = ImageSource::Online("unused-by-cpu".into()); + assert!( + processor + .load_image("https://example.com/image.png") + .is_err() + ); + assert!(processor.load_image("prepared://unknown").is_err()); + processor.max_length = 63; + assert_eq!( + prepare_online(&processor, &[RED]).err().unwrap().status, + 400 + ); + // A rejected cold request must not create an L1 record. + assert_eq!(processor.caches.snapshot().l1_records, 0); + assert_eq!( + prepare_online(&processor, &[RED]).err().unwrap().status, + 400 + ); +} + +#[test] +fn request_budget_stops_before_later_images_with_caches_on_or_off() { + for (enabled, l2) in [(true, true), (true, false), (false, false)] { + for second in [RED, GREEN] { + let mut processor = online_processor(enabled, l2); + let raw = image_request(&[RED, second, "data:image/png;base64,%%%"]); + let compiled = + contract::compile(&serde_json::to_vec(&raw).unwrap(), &processor.labels).unwrap(); + let text_tokens = processor.tokenize(&compiled.prompt).unwrap().len(); + // Exactly one expanded image fits; the second reference must reject + // before the third image's deliberately invalid bytes are decoded. + processor.max_length = text_tokens + 63; + let error = prepare_request(&processor, &raw).err().unwrap(); + let message = error.body.to_string(); + assert_eq!(error.status, 400); + assert!(message.contains("maximum context length"), "{message}"); + assert!( + message.contains(&format!("value={}", text_tokens + 126)), + "{message}" + ); + assert!(!message.contains("invalid base64"), "{message}"); + assert_eq!(processor.caches.snapshot().l2_records, 0); + } + } +} + +#[test] +fn repeated_references_share_pixels_without_losing_blocks_or_positions() { + for (enabled, l2) in [(true, true), (true, false), (false, false)] { + let processor = online_processor(enabled, l2); + let raw = image_request(&[RED, GREEN, RED]); + let MmPlan::Full(mm) = prepare_request(&processor, &raw).unwrap().plan else { + panic!("expected full plan") + }; + let pixels: Vec<_> = mm + .blocks + .iter() + .map(|b| match &b.asset { + ImageInput::Inline { pixels, .. } => pixels, + _ => panic!("expected inline pixels"), + }) + .collect(); + assert_eq!(pixels.len(), 3); + assert!(Arc::ptr_eq(pixels[0], pixels[2])); + assert!(!Arc::ptr_eq(pixels[0], pixels[1])); + let compiled = + contract::compile(&serde_json::to_vec(&raw).unwrap(), &processor.labels).unwrap(); + let text_ids = processor.tokenize(&compiled.prompt).unwrap(); + let expected = + images::expand_grids(&text_ids, processor.image_pad, &[[1, 16, 16]; 3]).unwrap(); + assert_eq!(mm.ids, expected.ids); + assert_eq!(mm.positions, expected.positions); + assert_eq!( + mm.blocks + .iter() + .map(|b| (b.start, b.end)) + .collect::>(), + expected + .blocks + .iter() + .map(|b| (b.start, b.end)) + .collect::>() + ); + } +} + +#[test] +fn text_over_budget_is_rejected_before_any_image_decode() { + for enabled in [true, false] { + let mut processor = online_processor(enabled, false); + processor.max_length = 1; + let error = prepare_online(&processor, &["data:image/png;base64,%%%"]) + .err() + .unwrap(); + assert!(error.body.to_string().contains("maximum context length")); + } +} + +#[test] +fn warm_single_image_checks_full_length_before_reloading_pixels() { + let mut processor = online_processor(true, false); + let accepted = prepare_online(&processor, &[RED]).unwrap(); + assert_eq!(processor.caches.snapshot().l1_records, 1); + processor.max_length = accepted.context.input_tokens - 1; + // Preserve the warm prefix record but make image loading fail if reached. + processor.source = ImageSource::Prepared(None); + let error = prepare_online(&processor, &[RED]).err().unwrap(); + let message = error.body.to_string(); + assert!(message.contains("maximum context length"), "{message}"); + assert!(!message.contains("no imgcache")); +} + +#[test] +fn disabling_l2_retains_typed_images_and_does_not_publish_embeddings_on_cpu() { + let mut processor = processor(); + processor.source = ImageSource::Online("unused-by-cpu".into()); + processor.caches = Caches::new(CacheCfg { + enabled: true, + l1: true, + l2: false, + l3: false, + l1_max: 8, + l2_bytes: 1 << 20, + l3_bytes: 0, + }); + for _ in 0..2 { + let MmPlan::Full(mm) = prepare_online(&processor, &[RED, RED]).unwrap().plan else { + panic!("expected full plan") + }; + assert_eq!(mm.blocks.len(), 2); + assert!( + mm.blocks + .iter() + .all(|b| matches!(b.asset, ImageInput::Inline { .. })) + ); + assert_eq!(processor.caches.snapshot().l2_records, 0); + } +} diff --git a/tests/jev_vl/vision.rs b/tests/jev_vl/vision.rs new file mode 100644 index 00000000..3717379c --- /dev/null +++ b/tests/jev_vl/vision.rs @@ -0,0 +1,155 @@ +use super::*; + +fn processor() -> Value { + json!({ + "size": {"longest_edge": 16777216, "shortest_edge": 65536}, + "patch_size": 16, "temporal_patch_size": 2, "merge_size": 2, + "image_mean": [0.5, 0.5, 0.5], "image_std": [0.5, 0.5, 0.5], + "processor_class": "Qwen3VLProcessor", + "image_processor_type": "Qwen2VLImageProcessorFast" + }) +} + +#[test] +fn source_selection_checks_presence_and_never_falls_back_from_online() { + let dir = tempfile::tempdir().unwrap(); + let missing = dir.path().join("missing"); + assert!(ImageSource::from_paths(dir.path(), Some(missing.clone()), None).is_err()); + assert!(ImageSource::from_paths(dir.path(), Some(dir.path().into()), Some(missing)).is_err()); + assert!(matches!( + ImageSource::from_paths(dir.path(), Some(dir.path().into()), None).unwrap(), + ImageSource::Online(_) + )); + assert!(matches!( + ImageSource::from_paths(dir.path(), None, None).unwrap(), + ImageSource::Prepared(None) + )); +} + +#[test] +fn processor_requires_the_frozen_resize_and_normalization_contract() { + let valid = processor(); + validate_processor(&valid).unwrap(); + for (key, value) in [ + ("merge_size", json!(1)), + ("temporal_patch_size", json!(1)), + ( + "size", + json!({"longest_edge": 1048576, "shortest_edge": 65536}), + ), + ("image_mean", json!([0.0, 0.0, 0.0])), + ("do_resize", json!(false)), + ("do_rescale", json!(false)), + ("resample", json!(2)), + ] { + let mut changed = valid.clone(); + changed[key] = value; + assert!(validate_processor(&changed).is_err(), "accepted {key}"); + } +} + +#[test] +fn config_rejects_other_models_mismatches_and_unknown_vision_semantics() { + let valid: Value = serde_json::from_str(include_str!("../qwen3_5/data/config.json")).unwrap(); + validate_config(&valid, &valid).unwrap(); + let mut wrong = valid.clone(); + wrong["model_type"] = json!("qwen3_5_moe"); + assert!(validate_config(&wrong, &wrong).is_err()); + let mut wrong = valid.clone(); + wrong["vision_config"]["out_hidden_size"] = json!(2560); + assert!(validate_config(&wrong, &valid).is_err()); + let mut wrong = valid.clone(); + wrong["text_config"]["model_type"] = json!("another_text_model"); + assert!(validate_config(&wrong, &wrong).is_err()); + let mut wrong = valid.clone(); + wrong["vision_config"]["window_size"] = json!(112); + assert!(validate_config(&wrong, &wrong).is_err()); +} + +fn export() -> (tempfile::TempDir, Value, Value) { + let dir = tempfile::tempdir().unwrap(); + let pins = json!({ + "model_index_sha256": "a".repeat(64), + "adapter_config_sha256": "b".repeat(64), + "adapter_model_sha256": "c".repeat(64) + }); + let language = json!({"pins": pins}); + let mut manifest = json!({ + "format": "jev-vl-vision/1", "model_id": crate::contract::MODEL_ID, + "pins": pins, "files": {} + }); + std::fs::write( + dir.path().join("config.json"), + include_str!("../qwen3_5/data/config.json"), + ) + .unwrap(); + std::fs::write( + dir.path().join("preprocessor_config.json"), + serde_json::to_vec(&processor()).unwrap(), + ) + .unwrap(); + // An empty tensor file proves that metadata reaches CPU inventory validation; + // these tests never need a large checkpoint or CUDA. + std::fs::write( + dir.path().join("vision.safetensors"), + [2u64.to_le_bytes().as_slice(), b"{}"].concat(), + ) + .unwrap(); + for name in [ + "config.json", + "preprocessor_config.json", + "vision.safetensors", + ] { + manifest["files"][name] = json!(sha256(&dir.path().join(name)).unwrap()); + } + (dir, language, manifest) +} + +fn validate_fixture(dir: &Path, language: &Value, manifest: &Value) -> String { + std::fs::write( + dir.join("jev_vl_vision.json"), + serde_json::to_vec(manifest).unwrap(), + ) + .unwrap(); + format!("{:#}", validate_export(dir, dir, language).unwrap_err()) +} + +#[test] +fn export_checks_all_source_pins_and_exact_files_before_weights() { + let (dir, language, manifest) = export(); + assert!(validate_fixture(dir.path(), &language, &manifest).contains("missing visual tensor")); + for pin in [ + "model_index_sha256", + "adapter_config_sha256", + "adapter_model_sha256", + ] { + let mut changed = manifest.clone(); + changed["pins"][pin] = json!("d".repeat(64)); + assert!(validate_fixture(dir.path(), &language, &changed).contains("source pin mismatch")); + } + let mut changed = manifest.clone(); + changed["files"]["other.json"] = json!("d".repeat(64)); + assert!(validate_fixture(dir.path(), &language, &changed).contains("pin exactly")); + for name in [ + "config.json", + "preprocessor_config.json", + "vision.safetensors", + ] { + let mut changed = manifest.clone(); + changed["files"][name] = json!("d".repeat(64)); + assert!( + validate_fixture(dir.path(), &language, &changed) + .contains(&format!("file hash mismatch: {name}")) + ); + } +} + +#[test] +fn streaming_hash_matches_known_sha256() { + let file = tempfile::NamedTempFile::new().unwrap(); + std::fs::write(file.path(), b"abc").unwrap(); + assert_eq!( + sha256(file.path()).unwrap(), + "ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad" + ); +} From 635c389ba9c6a1ba71369faf6e208044d6a3bb68 Mon Sep 17 00:00:00 2001 From: linear3735 Date: Fri, 9 Oct 2026 10:33:42 +0800 Subject: [PATCH 03/20] Warm online vision and skip empty image suffixes --- src/models/jev_vl/native/src/main.rs | 7 ++++ src/models/jev_vl/native/src/processing.rs | 27 ++++++++++-- tests/jev_vl/http.rs | 16 +++++++ tests/jev_vl/images.rs | 2 +- tests/jev_vl/processing.rs | 49 ++++++++++++++++++++++ 5 files changed, 96 insertions(+), 5 deletions(-) diff --git a/src/models/jev_vl/native/src/main.rs b/src/models/jev_vl/native/src/main.rs index 09117f37..7a52e42f 100644 --- a/src/models/jev_vl/native/src/main.rs +++ b/src/models/jev_vl/native/src/main.rs @@ -17,6 +17,7 @@ use omni_qwen3_5_native::cuda; use serde_json::{Value, json}; const WARMUP: &[u8] = br#"{"kind":"choice","state":"Dialog: Update installed.","question":"Close it?","options":["OK","Wait"]}"#; +const ONLINE_VISION_WARMUP: &[u8] = br#"{"kind":"choice","state":[{"image":"data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAIAAACQd1PeAAAADElEQVR4nGP4z8AAAAMBAQDJ/pLvAAAAAElFTkSuQmCC"}],"question":"Is the image loaded?","options":["OK","Wait"]}"#; fn reject(r: Reject) -> Response { ( @@ -141,6 +142,12 @@ async fn main() -> Result<()> { decide(&engine, WARMUP).await.status() == StatusCode::OK, "warmup failed" ); + if std::env::var_os("JEV_VL_VISION").is_some() { + ensure!( + decide(&engine, ONLINE_VISION_WARMUP).await.status() == StatusCode::OK, + "online vision warmup failed" + ); + } let host = std::env::var("JEV_VL_HOST").unwrap_or_else(|_| "127.0.0.1".into()); let port: u16 = std::env::var("JEV_VL_PORT") .map_or(Ok(8001), |v| v.parse()) diff --git a/src/models/jev_vl/native/src/processing.rs b/src/models/jev_vl/native/src/processing.rs index 6918786b..8215047b 100644 --- a/src/models/jev_vl/native/src/processing.rs +++ b/src/models/jev_vl/native/src/processing.rs @@ -236,13 +236,24 @@ impl Processor { let suffix_pads = pads_end - p; let input_tokens = p + suffix_pads + 1 + tail_ids.len(); self.check_length(input_tokens)?; - let asset = self.load_image(&compiled.images[0])?; let mut ids = Vec::with_capacity(suffix_pads + 1 + tail_ids.len()); ids.extend(std::iter::repeat_n(self.image_pad, suffix_pads)); ids.push(self.vision_end); ids.extend_from_slice(&tail_ids); - let mut pos = - images::meshgrid_positions(asset.grid_thw(), base_pad, p - pads_start, suffix_pads); + let suffix_asset = if suffix_pads > 0 { + Some(self.load_image(&compiled.images[0])?) + } else { + None + }; + let mut pos = match &suffix_asset { + Some(asset) => images::meshgrid_positions( + asset.grid_thw(), + base_pad, + p - pads_start, + suffix_pads, + ), + None => [Vec::new(), Vec::new(), Vec::new()], + }; let after = base_pad + advance; for k in 0..(ids.len() - suffix_pads) as i64 { pos[0].push(after + k); @@ -252,7 +263,7 @@ impl Processor { match (cfg.l3, record.state()) { (true, Some(state)) => { self.caches.l3_hit(); - let block = Some(SuffixBlock { + let block = suffix_asset.map(|asset| SuffixBlock { asset, row_offset: p - pads_start, rows: suffix_pads, @@ -273,6 +284,10 @@ impl Processor { } (true, None) => { self.caches.l3_miss(); + let asset = match suffix_asset { + Some(asset) => asset, + None => self.load_image(&compiled.images[0])?, + }; let mm = rebuild_full(&ids, &pos, &record.meta, &asset); return self.finish_c( mm.ids.len(), @@ -284,6 +299,10 @@ impl Processor { ); } (false, _) => { + let asset = match suffix_asset { + Some(asset) => asset, + None => self.load_image(&compiled.images[0])?, + }; let mm = rebuild_full(&ids, &pos, &record.meta, &asset); return self.finish_c( mm.ids.len(), diff --git a/tests/jev_vl/http.rs b/tests/jev_vl/http.rs index 6ed2c38f..5ba25c78 100644 --- a/tests/jev_vl/http.rs +++ b/tests/jev_vl/http.rs @@ -6,6 +6,22 @@ use std::time::Duration; use super::*; +#[test] +fn online_vision_startup_warmup_is_inline_image_decision() { + let labels = vec!["OK".to_owned(), "Wait".to_owned()]; + let compiled = omni_jev_vl_native::contract::compile(ONLINE_VISION_WARMUP, &labels).unwrap(); + assert_eq!(compiled.images.len(), 1); + let image = omni_qwen3_5_native::image_decode::decode_data_url(&compiled.images[0]).unwrap(); + let pixels = omni_qwen3_5_native::image_preprocess::preprocess_rgb8( + image.width, + image.height, + &image.rgb, + ) + .unwrap(); + assert_eq!(pixels.image_grid_thw, [1, 16, 16]); + assert_eq!(pixels.image_tokens(), 64); +} + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn unknown_model_chat_probe_returns_canonical_worker_error() { let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); diff --git a/tests/jev_vl/images.rs b/tests/jev_vl/images.rs index b1879f06..7f2c8070 100644 --- a/tests/jev_vl/images.rs +++ b/tests/jev_vl/images.rs @@ -61,7 +61,7 @@ fn meshgrid_helper_matches_expand_rows() { let text_ids: Vec = vec![300, 301, PAD, 302]; let e = expand(&text_ids, PAD, &[asset([1, 4, 6])]).unwrap(); let b = &e.blocks[0]; - for from in 0..6usize { + for from in 0..=6usize { let got = meshgrid_positions([1, 4, 6], b.base, from, 6 - from); for (want, have) in e.positions.iter().zip(&got) { assert_eq!(&want[b.start + from..b.end], &have[..], "slice from {from}"); diff --git a/tests/jev_vl/processing.rs b/tests/jev_vl/processing.rs index ed8627e6..9b4640d4 100644 --- a/tests/jev_vl/processing.rs +++ b/tests/jev_vl/processing.rs @@ -348,3 +348,52 @@ fn disabling_l2_retains_typed_images_and_does_not_publish_embeddings_on_cpu() { assert_eq!(processor.caches.snapshot().l2_records, 0); } } + +#[test] +fn aligned_image_end_still_loads_images_without_a_live_prefix() { + for l3 in [false, true] { + let mut processor = online_processor(true, false); + processor.caches = Caches::new(CacheCfg { + enabled: true, + l1: true, + l2: false, + l3, + l1_max: 8, + l2_bytes: 0, + l3_bytes: 1 << 20, + }); + let request = (0..64) + .map(|padding| { + let mut request = image_request(&[RED]); + request["state"] = serde_json::json!(["align ".repeat(padding), {"image": RED}]); + request + }) + .find(|request| { + let compiled = + contract::compile(&serde_json::to_vec(request).unwrap(), &processor.labels) + .unwrap(); + let ids = processor.tokenize(&compiled.prompt).unwrap(); + (ids.iter() + .position(|&id| id == processor.image_pad) + .unwrap() + + 64) + % 64 + == 0 + }) + .unwrap(); + let cold = prepare_request(&processor, &request).unwrap(); + let before = processor.caches.snapshot(); + let warm = prepare_request(&processor, &request).unwrap(); + assert!(warm.cache_note.contains("l1=hit")); + assert!(processor.caches.snapshot().l2_miss > before.l2_miss); + let full = |plan| match plan { + MmPlan::Full(mm) | MmPlan::Populate { mm, .. } => mm, + _ => panic!("no live GPU prefix exists"), + }; + let (cold, warm) = (full(cold.plan), full(warm.plan)); + assert_eq!(cold.ids, warm.ids); + assert_eq!(cold.positions, warm.positions); + assert_eq!(warm.blocks[0].end % 64, 0); + assert_eq!(warm.blocks[0].asset.n_tokens(), 64); + } +} From 0a3f38e6593d41e278394d227858eeaff2d60d25 Mon Sep 17 00:00:00 2001 From: linear3735 Date: Mon, 12 Oct 2026 17:10:15 +0800 Subject: [PATCH 04/20] fix: enforce native buffer and vision export bounds --- .github/workflows/ci.yml | 1 + recipe/jev_vl/export_vision.py | 10 ++-- src/models/qwen3_5/native/Cargo.toml | 4 ++ src/models/qwen3_5/native/src/cuda.rs | 2 +- .../qwen3_5/native/src/image_preprocess.rs | 2 +- tests/jev_vl/test_export_vision.py | 50 +++++++++++++++++++ tests/qwen3_5/device_buffer.rs | 14 ++++++ 7 files changed, 77 insertions(+), 6 deletions(-) create mode 100644 tests/jev_vl/test_export_vision.py create mode 100644 tests/qwen3_5/device_buffer.rs diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 594ef0f0..f01f033e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -42,3 +42,4 @@ jobs: - run: python -m pip install -r benchmarks/requirements.txt - run: python benchmarks/bench.py validate benchmarks/smoke.jsonl - run: python -m unittest discover -s tests/benchmarks -p 'test_*.py' -v + - run: python -m unittest discover -s tests/jev_vl -p 'test_export_vision.py' -v diff --git a/recipe/jev_vl/export_vision.py b/recipe/jev_vl/export_vision.py index f6e8abd1..8a0eb413 100644 --- a/recipe/jev_vl/export_vision.py +++ b/recipe/jev_vl/export_vision.py @@ -16,14 +16,12 @@ def sha256(path): def export(model, out): - import torch - from safetensors import safe_open - from safetensors.torch import save_file - if out.exists(): raise ValueError("output already exists; choose a new export directory") config = json.loads((model / "config.json").read_text()) if (config.get("model_type") != "qwen3_5" + or config.get("vision_config", {}).get("hidden_size") != 1152 + or config.get("vision_config", {}).get("out_hidden_size") != 5120 or config.get("text_config", {}).get("hidden_size") != 5120): raise ValueError("expected the JEV-27B-VL Qwen3.8 backbone") processor = json.loads((model / "preprocessor_config.json").read_text()) @@ -37,6 +35,10 @@ def export(model, out): if processor != expected_processor: raise ValueError("unsupported image processor; verify resize and normalization first") + import torch + from safetensors import safe_open + from safetensors.torch import save_file + adapter = model / "adapter_vllm" with safe_open(adapter / "adapter_model.safetensors", framework="pt") as weights: for name in weights.keys(): diff --git a/src/models/qwen3_5/native/Cargo.toml b/src/models/qwen3_5/native/Cargo.toml index c127c759..7604c234 100644 --- a/src/models/qwen3_5/native/Cargo.toml +++ b/src/models/qwen3_5/native/Cargo.toml @@ -27,3 +27,7 @@ path = "../../../../tests/qwen3_5/kernels.rs" [[test]] name = "image_decode" path = "../../../../tests/qwen3_5/image_decode.rs" + +[[test]] +name = "device_buffer" +path = "../../../../tests/qwen3_5/device_buffer.rs" diff --git a/src/models/qwen3_5/native/src/cuda.rs b/src/models/qwen3_5/native/src/cuda.rs index ece939b5..f6a5b0ab 100644 --- a/src/models/qwen3_5/native/src/cuda.rs +++ b/src/models/qwen3_5/native/src/cuda.rs @@ -264,7 +264,7 @@ impl DeviceBuffer { /// The device address `offset` bytes into the buffer. pub fn at(&self, offset: usize) -> *mut c_void { - debug_assert!(offset <= self.bytes); + assert!(offset <= self.bytes); self.ptr.wrapping_byte_add(offset) } } diff --git a/src/models/qwen3_5/native/src/image_preprocess.rs b/src/models/qwen3_5/native/src/image_preprocess.rs index 48b63625..0e50f644 100644 --- a/src/models/qwen3_5/native/src/image_preprocess.rs +++ b/src/models/qwen3_5/native/src/image_preprocess.rs @@ -55,7 +55,7 @@ impl ProcessedImage { } } -/// Preprocess a decoded RGB8 image using the fixed 4B processor settings. +/// Preprocess a decoded RGB8 image with CUA source and patch limits. /// /// Rejects empty dimensions, sides over 2048, area over 1,048,576 pixels, /// aspect ratios over 200, and buffers whose length is not `width * height * 3`. diff --git a/tests/jev_vl/test_export_vision.py b/tests/jev_vl/test_export_vision.py new file mode 100644 index 00000000..6312d24c --- /dev/null +++ b/tests/jev_vl/test_export_vision.py @@ -0,0 +1,50 @@ +"""Reject unsupported vision exports before loading weights or dependencies.""" + +import copy +import importlib.util +import json +from pathlib import Path +import sys +import tempfile +import unittest +from unittest import mock + +REPO = Path(__file__).resolve().parents[2] +SPEC = importlib.util.spec_from_file_location( + "jev_vl_export_vision", REPO / "recipe/jev_vl/export_vision.py" +) +export_vision = importlib.util.module_from_spec(SPEC) +SPEC.loader.exec_module(export_vision) + + +class ExportVisionTests(unittest.TestCase): + def test_rejects_wrong_dimensions_before_weights_and_output(self): + valid = json.loads((REPO / "tests/qwen3_5/data/config.json").read_text()) + read_weights = mock.Mock() + save_weights = mock.Mock() + with tempfile.TemporaryDirectory() as temporary, mock.patch.dict(sys.modules, { + "torch": None, + "safetensors": mock.Mock(safe_open=read_weights), + "safetensors.torch": mock.Mock(save_file=save_weights), + }): + model = Path(temporary) / "model" + model.mkdir() + out = Path(temporary) / "export" + for section, field, value in ( + ("vision_config", "hidden_size", 1024), + ("vision_config", "out_hidden_size", 2560), + ("text_config", "hidden_size", 2560), + ): + with self.subTest(section=section, field=field): + config = copy.deepcopy(valid) + config[section][field] = value + (model / "config.json").write_text(json.dumps(config)) + with self.assertRaisesRegex(ValueError, "expected the JEV-27B-VL"): + export_vision.export(model, out) + read_weights.assert_not_called() + save_weights.assert_not_called() + self.assertFalse(out.exists()) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/qwen3_5/device_buffer.rs b/tests/qwen3_5/device_buffer.rs new file mode 100644 index 00000000..1566fd66 --- /dev/null +++ b/tests/qwen3_5/device_buffer.rs @@ -0,0 +1,14 @@ +use omni_qwen3_5_native::cuda::DeviceBuffer; + +#[test] +fn zero_byte_buffer_accepts_zero_offset_without_cuda() { + let buffer = DeviceBuffer::new(0).unwrap(); + assert!(buffer.at(0).is_null()); +} + +#[test] +#[should_panic(expected = "offset <= self.bytes")] +fn zero_byte_buffer_rejects_out_of_bounds_offset_without_cuda() { + let buffer = DeviceBuffer::new(0).unwrap(); + buffer.at(1); +} From 953b6914fbb2a667a7939dd2ad832136750a11ad Mon Sep 17 00:00:00 2001 From: linear3735 Date: Mon, 12 Oct 2026 19:31:01 +0800 Subject: [PATCH 05/20] refactor: remove duplicate online vision preparation --- src/models/jev_vl/native/src/processing.rs | 51 ++++++------------ src/models/jev_vl/native/src/vision.rs | 5 +- tests/jev_vl/processing.rs | 62 +++++++--------------- tests/jev_vl/vision.rs | 29 ++++++++-- 4 files changed, 61 insertions(+), 86 deletions(-) diff --git a/src/models/jev_vl/native/src/processing.rs b/src/models/jev_vl/native/src/processing.rs index 8215047b..317c75f9 100644 --- a/src/models/jev_vl/native/src/processing.rs +++ b/src/models/jev_vl/native/src/processing.rs @@ -14,10 +14,7 @@ use crate::contract::{self, Compiled, Kind, Reject}; use crate::executor::{ContinueMm, ImgBlock, MmPlan, PreparedMm, Readout, SuffixBlock}; use crate::images::{self, ImageAsset, ImageInput}; use crate::vision::ImageSource; -use omni_qwen3_5_native::{ - image_decode::decode_data_url, - image_preprocess::{ImageLimits, preprocess_rgb8_with_limits}, -}; +use omni_qwen3_5_native::{image_decode::decode_data_url, image_preprocess::preprocess_rgb8}; pub struct Processor { tokenizer: Tokenizer, @@ -93,13 +90,8 @@ impl Processor { ImageSource::Online(_) => { let decoded = decode_data_url(url) .map_err(|e| Reject::bad_request(format!("invalid image: {e:#}")))?; - let pixels = preprocess_rgb8_with_limits( - decoded.width, - decoded.height, - &decoded.rgb, - &ImageLimits::cua(), - ) - .map_err(|e| Reject::bad_request(format!("invalid image: {e:#}")))?; + let pixels = preprocess_rgb8(decoded.width, decoded.height, &decoded.rgb) + .map_err(|e| Reject::bad_request(format!("invalid image: {e:#}")))?; Ok(ImageInput::Inline { key, pixels: Arc::new(pixels), @@ -282,36 +274,25 @@ impl Processor { format!("l1=hit,l3=hit,p={p}"), ); } - (true, None) => { - self.caches.l3_miss(); + (l3, _) => { + if l3 { + self.caches.l3_miss(); + } let asset = match suffix_asset { Some(asset) => asset, None => self.load_image(&compiled.images[0])?, }; let mm = rebuild_full(&ids, &pos, &record.meta, &asset); - return self.finish_c( - mm.ids.len(), - MmPlan::Populate { mm, record }, - readout, - compiled, - start, - format!("l1=hit,l3=miss,p={p}"), - ); - } - (false, _) => { - let asset = match suffix_asset { - Some(asset) => asset, - None => self.load_image(&compiled.images[0])?, + let input_tokens = mm.ids.len(); + let (plan, note) = if l3 { + ( + MmPlan::Populate { mm, record }, + format!("l1=hit,l3=miss,p={p}"), + ) + } else { + (MmPlan::Full(mm), "l1=hit,l3=off,p=-".to_string()) }; - let mm = rebuild_full(&ids, &pos, &record.meta, &asset); - return self.finish_c( - mm.ids.len(), - MmPlan::Full(mm), - readout, - compiled, - start, - "l1=hit,l3=off,p=-".to_string(), - ); + return self.finish_c(input_tokens, plan, readout, compiled, start, note); } } } diff --git a/src/models/jev_vl/native/src/vision.rs b/src/models/jev_vl/native/src/vision.rs index 5f64d260..28baff7b 100644 --- a/src/models/jev_vl/native/src/vision.rs +++ b/src/models/jev_vl/native/src/vision.rs @@ -5,7 +5,7 @@ use std::io::Read; use std::path::{Path, PathBuf}; use anyhow::{Context, Result, ensure}; -use omni_qwen3_5_native::vision::{VisionCheckpoint, VisionConfig}; +use omni_qwen3_5_native::vision::VisionConfig; use serde_json::{Value, json}; use sha2::{Digest, Sha256}; @@ -165,9 +165,6 @@ pub(crate) fn validate_export(dir: &Path, language_dir: &Path, language: &Value) validate_processor(&serde_json::from_slice(&std::fs::read( dir.join("preprocessor_config.json"), )?)?)?; - // The shared loader validates the complete tensor inventory on CPU before - // either language or vision weights can be uploaded. - VisionCheckpoint::load(dir)?; Ok(()) } diff --git a/tests/jev_vl/processing.rs b/tests/jev_vl/processing.rs index 9b4640d4..672c2514 100644 --- a/tests/jev_vl/processing.rs +++ b/tests/jev_vl/processing.rs @@ -182,31 +182,6 @@ fn online_preparation_has_pixels_not_empty_embeddings_and_preserves_l1_geometry( assert_eq!(warm.blocks[0].end - warm.blocks[0].start, 64); } -#[test] -fn same_size_images_keep_distinct_keys_and_repeated_references_keep_all_blocks() { - let mut processor = processor(); - processor.source = ImageSource::Online("unused-by-cpu".into()); - let plan = prepare_online(&processor, &[RED, GREEN, RED]).unwrap().plan; - let MmPlan::Full(mm) = plan else { - panic!("expected full plan") - }; - let keys: Vec<_> = mm - .blocks - .iter() - .map(|b| match &b.asset { - ImageInput::Inline { key, .. } => key, - _ => panic!("expected inline image"), - }) - .collect(); - assert_eq!(keys.len(), 3); - assert_eq!(keys[0], keys[2]); - assert_ne!(keys[0], keys[1]); - assert_eq!( - mm.blocks.iter().map(|b| b.end - b.start).sum::(), - 192 - ); -} - #[test] fn online_rejects_unsupported_urls_and_full_context_overflow_before_admission() { let mut processor = processor(); @@ -264,17 +239,23 @@ fn repeated_references_share_pixels_without_losing_blocks_or_positions() { let MmPlan::Full(mm) = prepare_request(&processor, &raw).unwrap().plan else { panic!("expected full plan") }; - let pixels: Vec<_> = mm + let images: Vec<_> = mm .blocks .iter() .map(|b| match &b.asset { - ImageInput::Inline { pixels, .. } => pixels, + ImageInput::Inline { key, pixels } => (key, pixels), _ => panic!("expected inline pixels"), }) .collect(); - assert_eq!(pixels.len(), 3); - assert!(Arc::ptr_eq(pixels[0], pixels[2])); - assert!(!Arc::ptr_eq(pixels[0], pixels[1])); + assert_eq!(images.len(), 3); + assert_eq!(images[0].0, images[2].0); + assert_ne!(images[0].0, images[1].0); + assert!(Arc::ptr_eq(images[0].1, images[2].1)); + assert!(!Arc::ptr_eq(images[0].1, images[1].1)); + assert_eq!( + mm.blocks.iter().map(|b| b.end - b.start).sum::(), + 192 + ); let compiled = contract::compile(&serde_json::to_vec(&raw).unwrap(), &processor.labels).unwrap(); let text_ids = processor.tokenize(&compiled.prompt).unwrap(); @@ -324,17 +305,7 @@ fn warm_single_image_checks_full_length_before_reloading_pixels() { #[test] fn disabling_l2_retains_typed_images_and_does_not_publish_embeddings_on_cpu() { - let mut processor = processor(); - processor.source = ImageSource::Online("unused-by-cpu".into()); - processor.caches = Caches::new(CacheCfg { - enabled: true, - l1: true, - l2: false, - l3: false, - l1_max: 8, - l2_bytes: 1 << 20, - l3_bytes: 0, - }); + let processor = online_processor(true, false); for _ in 0..2 { let MmPlan::Full(mm) = prepare_online(&processor, &[RED, RED]).unwrap().plan else { panic!("expected full plan") @@ -385,7 +356,14 @@ fn aligned_image_end_still_loads_images_without_a_live_prefix() { let before = processor.caches.snapshot(); let warm = prepare_request(&processor, &request).unwrap(); assert!(warm.cache_note.contains("l1=hit")); - assert!(processor.caches.snapshot().l2_miss > before.l2_miss); + let after = processor.caches.snapshot(); + assert!(after.l2_miss > before.l2_miss); + assert_eq!(after.l3_miss, before.l3_miss + u64::from(l3)); + assert_eq!(matches!(warm.plan, MmPlan::Populate { .. }), l3); + assert!( + warm.cache_note + .contains(if l3 { "l3=miss" } else { "l3=off" }) + ); let full = |plan| match plan { MmPlan::Full(mm) | MmPlan::Populate { mm, .. } => mm, _ => panic!("no live GPU prefix exists"), diff --git a/tests/jev_vl/vision.rs b/tests/jev_vl/vision.rs index 3717379c..aa446dd4 100644 --- a/tests/jev_vl/vision.rs +++ b/tests/jev_vl/vision.rs @@ -105,19 +105,26 @@ fn export() -> (tempfile::TempDir, Value, Value) { (dir, language, manifest) } -fn validate_fixture(dir: &Path, language: &Value, manifest: &Value) -> String { +fn validate_fixture(dir: &Path, language: &Value, manifest: &Value) -> Result<()> { std::fs::write( dir.join("jev_vl_vision.json"), serde_json::to_vec(manifest).unwrap(), ) .unwrap(); - format!("{:#}", validate_export(dir, dir, language).unwrap_err()) + validate_export(dir, dir, language) } #[test] fn export_checks_all_source_pins_and_exact_files_before_weights() { let (dir, language, manifest) = export(); - assert!(validate_fixture(dir.path(), &language, &manifest).contains("missing visual tensor")); + validate_fixture(dir.path(), &language, &manifest).unwrap(); + let error = omni_qwen3_5_native::vision::VisionModel::load( + dir.path(), + &dir.path().join("missing-cuda-library"), + ) + .err() + .expect("empty inventory must fail before loading CUDA"); + assert!(format!("{error:#}").contains("missing visual tensor")); for pin in [ "model_index_sha256", "adapter_config_sha256", @@ -125,11 +132,21 @@ fn export_checks_all_source_pins_and_exact_files_before_weights() { ] { let mut changed = manifest.clone(); changed["pins"][pin] = json!("d".repeat(64)); - assert!(validate_fixture(dir.path(), &language, &changed).contains("source pin mismatch")); + assert!( + validate_fixture(dir.path(), &language, &changed) + .unwrap_err() + .to_string() + .contains("source pin mismatch") + ); } let mut changed = manifest.clone(); changed["files"]["other.json"] = json!("d".repeat(64)); - assert!(validate_fixture(dir.path(), &language, &changed).contains("pin exactly")); + assert!( + validate_fixture(dir.path(), &language, &changed) + .unwrap_err() + .to_string() + .contains("pin exactly") + ); for name in [ "config.json", "preprocessor_config.json", @@ -139,6 +156,8 @@ fn export_checks_all_source_pins_and_exact_files_before_weights() { changed["files"][name] = json!("d".repeat(64)); assert!( validate_fixture(dir.path(), &language, &changed) + .unwrap_err() + .to_string() .contains(&format!("file hash mismatch: {name}")) ); } From d5246e22d1ddce760ae315a6a9f2e76cf04c43d7 Mon Sep 17 00:00:00 2001 From: linear3735 Date: Mon, 12 Oct 2026 20:56:47 +0800 Subject: [PATCH 06/20] docs: record online image support, warmup readiness, and per-request bounds Address PR #139 review: the architecture page still claimed image features must be prepared offline; the recipe documented neither the online vision warmup that now gates readiness nor the transient host memory a many-image online request holds outside the cache budgets. Co-Authored-By: Claude Code --- README.md | 2 +- docs/architecture.md | 9 +++++---- recipe/jev_vl/README.md | 18 +++++++++++++++++- 3 files changed, 23 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index aa5450e9..1c6d71ed 100644 --- a/README.md +++ b/README.md @@ -172,7 +172,7 @@ v1.1 has its checkpoint export and CPU contract, with no worker yet: | Valen-Preview-0923 | [Python reference worker](recipe/valen/reference.md); text state or one inline PNG/JPEG image, `choice`; author-run on RTX 5060 Laptop GPU (sm_120, WSL2), owner: community | | Open-Jev-27B-v1.1 | [Native Rust/CUDA worker](recipe/open_jev/native.md); eager independent text candidates; [H200 validation](recipe/open_jev/validation.md) | | Open-Jev-9B | The same [native Rust/CUDA worker](recipe/open_jev/native.md); eager independent text candidates; [reference comparison on sm_89](recipe/open_jev/validation-9b.md) | -| autotrust/JEV-27B-VL | [Experimental Rust/CUDA worker](recipe/jev_vl/README.md); single-question text and offline-preencoded image inputs; [bounded H800 validation and limits](recipe/jev_vl/validation.md) | +| autotrust/JEV-27B-VL | [Experimental Rust/CUDA worker](recipe/jev_vl/README.md); single-question text, online inline PNG/JPEG and offline-preencoded image inputs; [validation scope and limits](recipe/jev_vl/validation.md) | | CLM-v0.1-8B | [External worker with a CPU stub encoder](recipe/clm/README.md); contract checks only, real Qwen3-8B decisions unverified by this recipe | | LFM2.5-350M | [Transformers worker](recipe/lfm2/README.md); text state and independent choice questions | | OmniJev-4B v1.1 | [Checkpoint export and CPU contract](recipe/omnijev/README.md) only; the native worker is in progress ([#114](https://github.com/ThinkFlowLab/system1-omni/issues/114)) | diff --git a/docs/architecture.md b/docs/architecture.md index 9fb69e11..1463073a 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -63,10 +63,11 @@ warmup before readiness remain model-specific and unchanged. The experimental [JEV-VL worker](../recipe/jev_vl/README.md) also uses the shared Qwen executor. It admits one official single-decision request at a time, reads -selected LM-head rows, and can retain image-prefix state across requests. Image -features must be prepared offline. Its `{kind, state, question, options}` contract -is distinct from the existing `{model, state, questions}` envelope; frontend -transport alone does not adapt those schemas. +selected LM-head rows, and can retain image-prefix state across requests. Images +are either prepared offline by Transformers or supplied inline and encoded +inside the worker by the shared native vision backend. Its `{kind, state, +question, options}` contract is distinct from the existing `{model, state, +questions}` envelope; frontend transport alone does not adapt those schemas. ## Layer ownership and implementation language diff --git a/recipe/jev_vl/README.md b/recipe/jev_vl/README.md index 96264686..932e9a0d 100644 --- a/recipe/jev_vl/README.md +++ b/recipe/jev_vl/README.md @@ -53,7 +53,8 @@ JEV_VL_MODEL=weights/jev-vl-merged \ `JEV_VL_HOST` defaults to `127.0.0.1`, `JEV_VL_PORT` to `8001`, and the CUDA library defaults to the file beside the executable. The worker uses visible CUDA device 0. `/health` becomes available after model loading and a successful -text warmup; that health check does not validate image assets. +text warmup. With `JEV_VL_VISION`, readiness also requires a successful inline-image +warmup; it does not validate every user image or offline-preencoded asset. In separate terminals, start the frontend and send the same request directly and through it: @@ -96,12 +97,27 @@ JEV_VL_MODEL=weights/jev-vl-merged JEV_VL_VISION=weights/jev-vl-vision \ JEV_VL_CACHE=1 target/release/omni-jev-vl-native ``` +With `JEV_VL_VISION` set, startup runs an online vision warmup after the text +warmup: one full inline-image decision, a 1×1 PNG through decode, preprocessing +and the vision forward. The worker does not bind its port unless that warmup +succeeds, so in online mode `/health` implies the inline image path executed +end to end once at startup. It still does not validate user images; each +request image is decoded and checked on arrival. + Online mode accepts single-frame inline PNG/JPEG data URIs. Each image is bounded to 4 MiB of decoded file bytes, 2048 pixels per side, 1,048,576 source pixels, and 4608 vision patches. The full HTTP body is limited to 4 MiB, including base64 and text. Remote URLs, animations and video are unsupported. Oversized inputs are rejected; resource limits do not change the model's resize policy. +No per-request image-count limit applies beyond the body and token budgets. +A request's resized pixel values and post-vision embeddings stay in host memory +until the request completes, and cache budgets exclude these in-flight inputs. +Each minimum-size image holds about 1.5 MiB of pixels and 1.25 MiB of +embeddings, so a body full of tiny images (roughly 250 within the 4 MiB body +and 16,384-token budgets) transiently adds up to about 0.7 GiB on top of +weights, scratch and caches; size online workloads accordingly on shared hosts. + `JEV_VL_VISION` and `JEV_VL_IMGCACHE` are mutually exclusive. Online mode also ignores the default `imgcache/` directory. The source and processor stay fixed for the worker lifetime. A new image runs decode, preprocessing and vision From ceadfd7d3bc2d4189a44c0a43280a20cd2959f08 Mon Sep 17 00:00:00 2001 From: linear3735 Date: Mon, 12 Oct 2026 22:50:33 +0800 Subject: [PATCH 07/20] docs: correct online image memory and cache semantics --- recipe/jev_vl/README.md | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/recipe/jev_vl/README.md b/recipe/jev_vl/README.md index 932e9a0d..996c1816 100644 --- a/recipe/jev_vl/README.md +++ b/recipe/jev_vl/README.md @@ -101,8 +101,9 @@ With `JEV_VL_VISION` set, startup runs an online vision warmup after the text warmup: one full inline-image decision, a 1×1 PNG through decode, preprocessing and the vision forward. The worker does not bind its port unless that warmup succeeds, so in online mode `/health` implies the inline image path executed -end to end once at startup. It still does not validate user images; each -request image is decoded and checked on arrival. +end to end once at startup. It does not validate future user images. New image +inputs are decoded and checked when prepared; cache hits may reuse validated +assets or retained image-prefix state. Online mode accepts single-frame inline PNG/JPEG data URIs. Each image is bounded to 4 MiB of decoded file bytes, 2048 pixels per side, 1,048,576 source pixels, @@ -111,12 +112,12 @@ and text. Remote URLs, animations and video are unsupported. Oversized inputs are rejected; resource limits do not change the model's resize policy. No per-request image-count limit applies beyond the body and token budgets. -A request's resized pixel values and post-vision embeddings stay in host memory -until the request completes, and cache budgets exclude these in-flight inputs. -Each minimum-size image holds about 1.5 MiB of pixels and 1.25 MiB of -embeddings, so a body full of tiny images (roughly 250 within the 4 MiB body -and 16,384-token budgets) transiently adds up to about 0.7 GiB on top of -weights, scratch and caches; size online workloads accordingly on shared hosts. +A minimum-size image uses 1.5 MiB of FP32 processed pixels and produces +0.625 MiB of BF16 embeddings. Pixel buffers are released as image inputs are +consumed during vision resolution; embeddings remain for language execution. +Cache budgets exclude in-flight inputs. Hundreds of tiny images can therefore +require hundreds of MiB of host memory, plus temporary copies, weights, scratch +and caches. These array sizes are not a measured peak-memory bound. `JEV_VL_VISION` and `JEV_VL_IMGCACHE` are mutually exclusive. Online mode also ignores the default `imgcache/` directory. The source and processor stay fixed From f0febb089a47e1cbc0fc6e2e7e85801a1943840a Mon Sep 17 00:00:00 2001 From: linear3735 Date: Tue, 13 Oct 2026 00:05:05 +0800 Subject: [PATCH 08/20] tests: pin cache-lookup counting and share vision checkpoint regressions cua_s1 consumes the shared TensorStore header/offset/path checks instead of a private duplicate (27B layouts stay rejected for the 4B worker); the JEV cache controls document l2_hit/l2_miss as lookup counters and pin the LRU-refresh on admission rechecks; the recipe pins explicit eager vision (CUA_S1_VISION_GRAPH=0) for the documented path. Adds the byte-level vision checkpoint regression suite, the non-UTF8 JEV_VL_IMGCACHE fallback test, and GPU-gated 27B-layout v2 position/rope/attention kernel checks. (Prepared as tree 60a8fbe9 by the codex jev session; landed after independent review.) Co-Authored-By: Claude Code --- recipe/jev_vl/README.md | 17 +- src/models/cua_s1/native/src/vision/mod.rs | 182 +-------- src/models/jev_vl/README.md | 3 + src/models/jev_vl/native/Cargo.toml | 4 + src/models/jev_vl/native/src/caches.rs | 4 + src/models/jev_vl/native/src/engine.rs | 2 +- src/models/jev_vl/native/src/executor.rs | 1 + src/models/jev_vl/native/src/vision.rs | 7 +- src/models/qwen3_5/native/src/vision/mod.rs | 20 +- tests/cua_s1/vision_checkpoint.rs | 404 ++++++++++++++++++++ tests/jev_vl/caches.rs | 73 ++++ tests/jev_vl/vision.rs | 29 +- tests/qwen3_5/kernels.rs | 234 ++++++++++++ 13 files changed, 796 insertions(+), 184 deletions(-) create mode 100644 tests/cua_s1/vision_checkpoint.rs create mode 100644 tests/jev_vl/caches.rs diff --git a/recipe/jev_vl/README.md b/recipe/jev_vl/README.md index 996c1816..d5234511 100644 --- a/recipe/jev_vl/README.md +++ b/recipe/jev_vl/README.md @@ -94,7 +94,7 @@ CUDA_VISIBLE_DEVICES='' .venv-jev-vl/bin/python recipe/jev_vl/export_vision.py \ --model weights/JEV-27B-VL --out weights/jev-vl-vision JEV_VL_MODEL=weights/jev-vl-merged JEV_VL_VISION=weights/jev-vl-vision \ JEV_VL_CUDA_LIB=$PWD/target/release/libqwen3_5_cuda.so \ - JEV_VL_CACHE=1 target/release/omni-jev-vl-native + JEV_VL_CACHE=1 CUA_S1_VISION_GRAPH=0 target/release/omni-jev-vl-native ``` With `JEV_VL_VISION` set, startup runs an online vision warmup after the text @@ -119,6 +119,13 @@ Cache budgets exclude in-flight inputs. Hundreds of tiny images can therefore require hundreds of MiB of host memory, plus temporary copies, weights, scratch and caches. These array sizes are not a measured peak-memory bound. +The shared vision backend also reads `CUA_S1_VISION_GRAPH` and +`CUA_S1_GRAPH_TRACE`, including when called by JEV. They enable capture and +capture diagnostics respectively only when set to `1`; both default to off. +Set `CUA_S1_VISION_GRAPH=0` explicitly as above to prevent an inherited Cua +setting from changing the JEV execution path. Current JEV validation uses +eager vision and does not cover the opt-in graph path. + `JEV_VL_VISION` and `JEV_VL_IMGCACHE` are mutually exclusive. Online mode also ignores the default `imgcache/` directory. The source and processor stay fixed for the worker lifetime. A new image runs decode, preprocessing and vision @@ -168,8 +175,12 @@ not an end-to-end live screenshot service. Cache budgets do not include weights, model scratch, in-flight request inputs or total process memory. `GET /v1/cache/stats` exposes counters and resident cache -accounting. Use those counters to assess L2 hits; `x-jev-cache` does not report -an L2 hit field. `POST /v1/cache/reset` resets counters only; restart the worker for +accounting. `l2_hit` and `l2_miss` count lookups, not unique images or requests. +Cold online images are checked during preparation and again after queueing; +prepared mode normally checks once. Retained-prefix continuations may skip L2 +entirely. Compare these counters within the same execution path rather than as +cross-mode per-image hit rates. `x-jev-cache` does not report an L2 hit field. +`POST /v1/cache/reset` resets counters only; restart the worker for a cold cache. Request preparation may overlap, while GPU execution stays serial. Use cache modes as experimental controls within the documented validation scope. diff --git a/src/models/cua_s1/native/src/vision/mod.rs b/src/models/cua_s1/native/src/vision/mod.rs index 942e1086..d0230542 100644 --- a/src/models/cua_s1/native/src/vision/mod.rs +++ b/src/models/cua_s1/native/src/vision/mod.rs @@ -6,21 +6,14 @@ //! verify upstream hashes or tensor values. use anyhow::{Context, Result, ensure}; -use memmap2::Mmap; -use safetensors::{ - Dtype, SafeTensors, - tensor::{TensorInfo, TensorView}, -}; -use serde::{ - Deserialize, - de::{self, MapAccess, Visitor}, -}; +use omni_qwen3_5_native::vision::{TensorStore, UniqueMap, inside, is_visual}; +use safetensors::{Dtype, tensor::TensorView}; +use serde::Deserialize; use serde_json::Value; use std::{ collections::{BTreeMap, BTreeSet}, - fmt, - fs::{self, File}, - path::{Component, Path, PathBuf}, + fs, + path::Path, }; const BASE: &str = "model.visual."; @@ -307,10 +300,10 @@ impl VisionCheckpoint { &self.adapter } pub fn base_names(&self) -> impl Iterator { - self.base.tensors.keys().map(String::as_str) + self.base.names() } pub fn adapter_names(&self) -> impl Iterator { - self.lora.tensors.keys().map(String::as_str) + self.lora.names() } pub fn base_tensor(&self, name: &str) -> Result> { self.base.tensor(name) @@ -320,162 +313,9 @@ impl VisionCheckpoint { } } -struct TensorStore { - maps: Vec, - tensors: BTreeMap, -} -impl TensorStore { - fn load( - dir: &Path, - files: BTreeSet, - expected: &Inventory, - dtype: Dtype, - index: Option<&BTreeMap>, - ) -> Result { - let mut store = Self { - maps: Vec::new(), - tensors: BTreeMap::new(), - }; - for filename in files { - let path = inside(dir, &filename)?; - let file = File::open(&path).with_context(|| format!("open {}", path.display()))?; - // SAFETY: callers must not mutate or truncate checkpoint files while mapped. - let map = unsafe { Mmap::map(&file) } - .with_context(|| format!("mmap safetensors {}", path.display()))?; - validate_header(&map) - .with_context(|| format!("safetensors header {}", path.display()))?; - let (header_len, metadata) = SafeTensors::read_metadata(&map) - .with_context(|| format!("safetensors {}", path.display()))?; - for (name, info) in metadata.tensors() { - if !is_visual(&name) { - continue; - } - let shape = expected - .get(&name) - .with_context(|| format!("unexpected visual tensor {name} in {filename}"))?; - if let Some(index) = index { - ensure!( - index.get(&name) == Some(&filename), - "index mismatch for {name} in {filename}" - ); - } - ensure!( - &info.shape == shape, - "shape mismatch for {name}: {:?}, expected {shape:?}", - info.shape - ); - ensure!( - info.dtype == dtype, - "dtype mismatch for {name}: {:?}, expected {dtype:?}", - info.dtype - ); - let mut info = info.clone(); - info.data_offsets.0 += 8 + header_len; - info.data_offsets.1 += 8 + header_len; - ensure!( - store - .tensors - .insert(name.clone(), (store.maps.len(), info)) - .is_none(), - "duplicate visual tensor {name}" - ); - } - store.maps.push(map); - } - for name in expected.keys() { - ensure!( - store.tensors.contains_key(name), - "missing visual tensor {name}" - ); - } - Ok(store) - } - fn tensor(&self, name: &str) -> Result> { - let (shard, info) = self - .tensors - .get(name) - .with_context(|| format!("unknown visual tensor {name}"))?; - Ok(TensorView::new( - info.dtype, - info.shape.clone(), - &self.maps[*shard][info.data_offsets.0..info.data_offsets.1], - )?) - } -} -// Bound all offsets before safetensors 0.8 adds payload size to header size: -// its final length check uses unchecked addition, even for ignored language tensors. -fn validate_header(bytes: &[u8]) -> Result<()> { - let length_bytes = bytes.get(..8).context("missing header length")?; - let header_len = usize::try_from(u64::from_le_bytes(length_bytes.try_into()?))?; - // Match safetensors 0.8's header allocation limit. - ensure!(header_len <= 100_000_000, "header too large"); - let data_start = header_len - .checked_add(8) - .context("header length overflow")?; - let header = bytes.get(8..data_start).context("truncated header")?; - let payload_len = bytes.len() - data_start; - let entries: UniqueMap = serde_json::from_slice(header)?; - for (name, entry) in entries.0 { - if name == "__metadata__" { - continue; - } - let (start, end): (usize, usize) = serde_json::from_value(entry["data_offsets"].clone()) - .with_context(|| format!("invalid offsets for {name}"))?; - ensure!( - start <= end && end <= payload_len, - "tensor offsets exceed payload for {name}" - ); - } - Ok(()) -} -fn is_visual(name: &str) -> bool { - name.split('.').any(|part| part == "visual") -} -fn inside(dir: &Path, name: &str) -> Result { - ensure!( - !name.is_empty() - && Path::new(name) - .components() - .all(|c| matches!(c, Component::Normal(_))), - "path must remain inside checkpoint directory: {name}" - ); - let resolved = - fs::canonicalize(dir.join(name)).with_context(|| format!("checkpoint file {name}"))?; - ensure!( - resolved.starts_with(dir), - "path escapes checkpoint directory: {name}" - ); - Ok(resolved) -} - -// serde_json's ordinary maps overwrite repeated keys. Reject ambiguous headers/indexes. -struct UniqueMap(BTreeMap); -impl<'de, T: Deserialize<'de>> Deserialize<'de> for UniqueMap { - fn deserialize>( - deserializer: D, - ) -> std::result::Result { - struct UniqueVisitor(std::marker::PhantomData); - impl<'de, T: Deserialize<'de>> Visitor<'de> for UniqueVisitor { - type Value = UniqueMap; - fn expecting(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - f.write_str("an object with unique names") - } - fn visit_map>( - self, - mut map: M, - ) -> std::result::Result { - let mut entries = BTreeMap::new(); - while let Some((key, value)) = map.next_entry::()? { - if entries.insert(key.clone(), value).is_some() { - return Err(de::Error::custom(format!("duplicate JSON key: {key}"))); - } - } - Ok(UniqueMap(entries)) - } - } - deserializer.deserialize_map(UniqueVisitor(std::marker::PhantomData)) - } -} - mod model; pub use model::VisionModel; + +#[cfg(test)] +#[path = "../../../../../../tests/cua_s1/vision_checkpoint.rs"] +mod tests; diff --git a/src/models/jev_vl/README.md b/src/models/jev_vl/README.md index 66a692a2..b9581950 100644 --- a/src/models/jev_vl/README.md +++ b/src/models/jev_vl/README.md @@ -101,6 +101,9 @@ L3 path targets a compatible single-image prefix aligned to a 64-token chunk; other inputs use a full forward. L2 avoids file parsing and copying of prepared assets in prepared mode and vision encoding in online mode. Cache sizes and controls are listed in the [recipe](../../../recipe/jev_vl/README.md#cache-controls). +The shared vision backend's `CUA_S1_VISION_GRAPH` and `CUA_S1_GRAPH_TRACE` controls +also apply here; see [online images](../../../recipe/jev_vl/README.md#online-images) +for explicit eager settings and their validation scope. Tests are registered under [`tests/jev_vl/`](../../../tests/jev_vl/) and [`tests/qwen3_5/`](../../../tests/qwen3_5/). CPU tests establish contract and diff --git a/src/models/jev_vl/native/Cargo.toml b/src/models/jev_vl/native/Cargo.toml index 84bd8c8f..cb7add85 100644 --- a/src/models/jev_vl/native/Cargo.toml +++ b/src/models/jev_vl/native/Cargo.toml @@ -29,5 +29,9 @@ path = "../../../../tests/jev_vl/images.rs" name = "jev_vl_prefix" path = "../../../../tests/jev_vl/prefix.rs" +[[test]] +name = "jev_vl_caches" +path = "../../../../tests/jev_vl/caches.rs" + [dev-dependencies] tempfile = "3" diff --git a/src/models/jev_vl/native/src/caches.rs b/src/models/jev_vl/native/src/caches.rs index 521eb628..2fb61b8e 100644 --- a/src/models/jev_vl/native/src/caches.rs +++ b/src/models/jev_vl/native/src/caches.rs @@ -67,7 +67,10 @@ impl CacheCfg { pub struct CacheStats { pub l1_hit: u64, pub l1_miss: u64, + /// L2 lookup hits, including online executor rechecks after admission. pub l2_hit: u64, + /// L2 lookup misses, not unique images or encoding attempts. A cold single-image + /// request checks once in prepared mode and twice online. pub l2_miss: u64, pub l3_hit: u64, pub l3_miss: u64, @@ -194,6 +197,7 @@ impl Caches { // ---- L2: parsed image assets ---- + /// Count every lookup, including disabled-cache misses, and refresh hit LRU. pub fn l2_get(&self, key: &str) -> Option> { if !(self.cfg.enabled && self.cfg.l2) { self.bump(|s| s.l2_miss += 1); diff --git a/src/models/jev_vl/native/src/engine.rs b/src/models/jev_vl/native/src/engine.rs index be7b5ea7..19b63b7f 100644 --- a/src/models/jev_vl/native/src/engine.rs +++ b/src/models/jev_vl/native/src/engine.rs @@ -28,7 +28,7 @@ impl Engine { let source = ImageSource::from_paths( dir, std::env::var_os("JEV_VL_VISION").map(Into::into), - std::env::var_os("JEV_VL_IMGCACHE").map(Into::into), + std::env::var_os("JEV_VL_IMGCACHE"), )?; let manifest: Value = serde_json::from_slice( &std::fs::read(dir.join("jev_vl_export.json")) diff --git a/src/models/jev_vl/native/src/executor.rs b/src/models/jev_vl/native/src/executor.rs index dc5ad517..36b4331c 100644 --- a/src/models/jev_vl/native/src/executor.rs +++ b/src/models/jev_vl/native/src/executor.rs @@ -295,6 +295,7 @@ impl Executor { let mut resolved_plan = None; let result = (|| { // Requests may have waited behind another request for this image. + // The L2 recheck counts a lookup hit/miss independently of preparation. // Pending assets also share duplicate references within this request // when L2 retention is disabled or the asset exceeds its budget. resolved_plan = Some(resolve_images(plan, |input| match input { diff --git a/src/models/jev_vl/native/src/vision.rs b/src/models/jev_vl/native/src/vision.rs index 28baff7b..7f033d4e 100644 --- a/src/models/jev_vl/native/src/vision.rs +++ b/src/models/jev_vl/native/src/vision.rs @@ -1,5 +1,6 @@ //! CPU validation of the pinned online vision export and its processor contract. +use std::ffi::OsString; use std::fs::File; use std::io::Read; use std::path::{Path, PathBuf}; @@ -19,8 +20,12 @@ impl ImageSource { pub(crate) fn from_paths( dir: &Path, vision: Option, - imgcache: Option, + imgcache: Option, ) -> Result { + // Preserve the prepared worker's env::var fallback for non-UTF8 values. + let imgcache = imgcache + .and_then(|p| p.into_string().ok()) + .map(PathBuf::from); ensure!( vision.is_none() || imgcache.is_none(), "JEV_VL_VISION and JEV_VL_IMGCACHE are mutually exclusive" diff --git a/src/models/qwen3_5/native/src/vision/mod.rs b/src/models/qwen3_5/native/src/vision/mod.rs index c0e09f84..d9647c4e 100644 --- a/src/models/qwen3_5/native/src/vision/mod.rs +++ b/src/models/qwen3_5/native/src/vision/mod.rs @@ -152,18 +152,20 @@ impl VisionCheckpoint { &self.config } pub fn base_names(&self) -> impl Iterator { - self.base.tensors.keys().map(String::as_str) + self.base.names() } pub fn base_tensor(&self, name: &str) -> Result> { self.base.tensor(name) } } -struct TensorStore { +/// Checked visual tensors borrowed from immutable checkpoint mappings. +pub struct TensorStore { maps: Vec, tensors: BTreeMap, } impl TensorStore { - fn load( + /// `dir` must be canonical; files must remain immutable while the store is alive. + pub fn load( dir: &Path, files: BTreeSet, expected: &Inventory, @@ -228,7 +230,10 @@ impl TensorStore { } Ok(store) } - fn tensor(&self, name: &str) -> Result> { + pub fn names(&self) -> impl Iterator { + self.tensors.keys().map(String::as_str) + } + pub fn tensor(&self, name: &str) -> Result> { let (shard, info) = self .tensors .get(name) @@ -266,10 +271,11 @@ fn validate_header(bytes: &[u8]) -> Result<()> { } Ok(()) } -fn is_visual(name: &str) -> bool { +pub fn is_visual(name: &str) -> bool { name.split('.').any(|part| part == "visual") } -fn inside(dir: &Path, name: &str) -> Result { +/// Resolve a checkpoint file inside its canonical directory, including symlinks. +pub fn inside(dir: &Path, name: &str) -> Result { ensure!( !name.is_empty() && Path::new(name) @@ -287,7 +293,7 @@ fn inside(dir: &Path, name: &str) -> Result { } // serde_json's ordinary maps overwrite repeated keys. Reject ambiguous headers/indexes. -struct UniqueMap(BTreeMap); +pub struct UniqueMap(pub BTreeMap); impl<'de, T: Deserialize<'de>> Deserialize<'de> for UniqueMap { fn deserialize>( deserializer: D, diff --git a/tests/cua_s1/vision_checkpoint.rs b/tests/cua_s1/vision_checkpoint.rs new file mode 100644 index 00000000..53dbcba9 --- /dev/null +++ b/tests/cua_s1/vision_checkpoint.rs @@ -0,0 +1,404 @@ +use super::*; +use omni_qwen3_5_native::vision::VisionCheckpoint as QwenCheckpoint; +use safetensors::tensor::TensorInfo; +use serde_json::json; +use std::{ + io::{Read, Seek, SeekFrom, Write}, + path::PathBuf, + sync::atomic::{AtomicUsize, Ordering}, +}; + +type Tensors = BTreeMap)>; +static NEXT_DIR: AtomicUsize = AtomicUsize::new(0); + +struct Fixture { + root: PathBuf, + base: PathBuf, + adapter: PathBuf, + tensors: Tensors, + lora: Tensors, +} +impl Fixture { + fn new() -> Self { + let root = std::env::temp_dir().join(format!( + "omni-vision-checkpoint-{}-{}", + std::process::id(), + NEXT_DIR.fetch_add(1, Ordering::Relaxed) + )); + let base = root.join("base"); + let adapter = root.join("adapter"); + fs::create_dir_all(&base).unwrap(); + fs::create_dir_all(&adapter).unwrap(); + let root = fs::canonicalize(root).unwrap(); + let base = fs::canonicalize(base).unwrap(); + let adapter = fs::canonicalize(adapter).unwrap(); + let vision = json!({ + "depth": 24, "hidden_size": 1024, "intermediate_size": 4096, + "num_heads": 16, "num_position_embeddings": 2304, "out_hidden_size": 2560, + "in_channels": 3, "patch_size": 16, "temporal_patch_size": 2, + "spatial_merge_size": 2, "hidden_act": "gelu_pytorch_tanh", + "deepstack_visual_indexes": [], "model_type": "qwen3_5" + }); + fs::write( + base.join("config.json"), + json!({"model_type": "qwen3_5", "text_config": {"hidden_size": 2560}, + "vision_config": vision}) + .to_string(), + ) + .unwrap(); + fs::write( + adapter.join("adapter_config.json"), + json!({"r": 16, "lora_alpha": 32, "peft_type": "LORA", "bias": "none", + "target_modules": ["up_proj", "k_proj", "linear_fc1", "q_proj", "linear_fc2", + "down_proj", "gate_proj", "o_proj", "v_proj"]}) + .to_string(), + ) + .unwrap(); + let inventory = VisionConfig::load(&base).unwrap().inventory(); + let mut tensors = tensor_specs(&inventory, Dtype::BF16); + tensors.insert("model.language.weight".into(), (Dtype::U8, vec![1])); + let mut lora = tensor_specs( + &AdapterConfig::load(&adapter).unwrap().inventory(&inventory), + Dtype::F32, + ); + lora.insert( + "base_model.model.model.language.lora_A.weight".into(), + (Dtype::U8, vec![1]), + ); + write_tensors(&base.join("model.safetensors"), &tensors); + fs::hard_link( + base.join("model.safetensors"), + base.join("vision.safetensors"), + ) + .unwrap(); + write_tensors(&adapter.join("adapter_model.safetensors"), &lora); + Self { + root, + base, + adapter, + tensors, + lora, + } + } + fn both_reject(&self, message: &str) { + assert_error(QwenCheckpoint::load(&self.base), message); + self.cua_reject(message); + } + fn cua_reject(&self, message: &str) { + assert_error(VisionCheckpoint::load(&self.base, &self.adapter), message); + } + fn index(&self, map: &BTreeMap) { + fs::write( + self.base.join("model.safetensors.index.json"), + json!({"weight_map": map}).to_string(), + ) + .unwrap(); + } + fn weight_map(&self) -> BTreeMap { + self.tensors + .keys() + .filter(|name| is_visual(name)) + .map(|name| (name.clone(), "model.safetensors".into())) + .collect() + } +} +impl Drop for Fixture { + fn drop(&mut self) { + fs::remove_dir_all(&self.root).unwrap(); + } +} +fn tensor_specs(inventory: &Inventory, dtype: Dtype) -> Tensors { + inventory + .iter() + .map(|(name, shape)| (name.clone(), (dtype, shape.clone()))) + .collect() +} +fn write_tensors(path: &Path, tensors: &Tensors) { + let mut offset = 0; + let header: BTreeMap<_, _> = tensors + .iter() + .map(|(name, (dtype, shape))| { + let start = offset; + offset += shape.iter().product::() * dtype.bitsize() / 8; + ( + name, + TensorInfo { + dtype: *dtype, + shape: shape.clone(), + data_offsets: (start, offset), + }, + ) + }) + .collect(); + write_header(path, &serde_json::to_string(&header).unwrap(), offset); +} +fn write_header(path: &Path, header: &str, payload_len: usize) { + let mut file = fs::File::create(path).unwrap(); + file.write_all(&(header.len() as u64).to_le_bytes()) + .unwrap(); + file.write_all(header.as_bytes()).unwrap(); + // Sparse zero payloads exercise full inventories without allocating model weights. + file.set_len((8 + header.len() + payload_len) as u64) + .unwrap(); +} +fn write_marker(path: &Path, name: &str, marker: &[u8]) { + let mut file = fs::OpenOptions::new() + .read(true) + .write(true) + .open(path) + .unwrap(); + let mut length = [0; 8]; + file.read_exact(&mut length).unwrap(); + let mut header = vec![0; u64::from_le_bytes(length) as usize]; + file.read_exact(&mut header).unwrap(); + let tensors: BTreeMap = serde_json::from_slice(&header).unwrap(); + file.seek(SeekFrom::Start( + (8 + header.len() + tensors[name].data_offsets.0) as u64, + )) + .unwrap(); + file.write_all(marker).unwrap(); +} +fn assert_error(result: Result, message: &str) { + let error = match result { + Ok(_) => panic!("expected {message}"), + Err(error) => format!("{error:#}"), + }; + assert!(error.contains(message), "expected {message:?}, got {error}"); +} + +#[test] +fn both_checkpoint_entries_keep_base_and_visual_lora_views_separate() { + let f = Fixture::new(); + let name = "model.visual.blocks.0.mlp.linear_fc1.weight"; + let second_name = "model.visual.patch_embed.proj.bias"; + let lora_name = "base_model.model.model.visual.blocks.0.mlp.linear_fc1.lora_A.weight"; + write_marker(&f.base.join("model.safetensors"), name, &[1, 2]); + write_marker(&f.base.join("model.safetensors"), second_name, &[3, 4]); + write_marker( + &f.adapter.join("adapter_model.safetensors"), + lora_name, + &[5, 6, 7, 8], + ); + let qwen = QwenCheckpoint::load(&f.base).unwrap(); + let unindexed = VisionCheckpoint::load(&f.base, &f.adapter).unwrap(); + let mut second = f.tensors.clone(); + let first = BTreeMap::from([(name.into(), second.remove(name).unwrap())]); + write_tensors(&f.base.join("first.safetensors"), &first); + write_tensors(&f.base.join("second.safetensors"), &second); + write_marker(&f.base.join("first.safetensors"), name, &[1, 2]); + write_marker(&f.base.join("second.safetensors"), second_name, &[3, 4]); + let map = f + .weight_map() + .into_keys() + .map(|tensor| { + let shard = if tensor == name { + "first.safetensors" + } else { + "second.safetensors" + }; + (tensor, shard.into()) + }) + .collect(); + f.index(&map); + let cua = VisionCheckpoint::load(&f.base, &f.adapter).unwrap(); + assert_eq!( + qwen.base_names().collect::>(), + cua.base_names().collect::>() + ); + assert_eq!(cua.base_names().count(), 297); + assert_eq!(cua.adapter_names().count(), 100); + for view in [ + qwen.base_tensor(name).unwrap(), + unindexed.base_tensor(name).unwrap(), + cua.base_tensor(name).unwrap(), + ] { + assert_eq!(view.dtype(), Dtype::BF16); + assert_eq!(view.shape(), [4096, 1024]); + assert_eq!(&view.data()[..2], [1, 2]); + } + assert_eq!(&cua.base_tensor(second_name).unwrap().data()[..2], [3, 4]); + let view = cua.adapter_tensor(lora_name).unwrap(); + assert_eq!(view.dtype(), Dtype::F32); + assert_eq!(view.shape(), [16, 1024]); + assert_eq!(&view.data()[..4], [5, 6, 7, 8]); + assert_eq!(cua.adapter().scale(), 2.0); + assert_error( + qwen.base_tensor("model.language.weight"), + "unknown visual tensor", + ); + assert_error( + cua.adapter_tensor("base_model.model.model.language.lora_A.weight"), + "unknown visual tensor", + ); +} + +#[test] +fn common_header_checks_reject_malformed_lengths_and_json() { + let f = Fixture::new(); + for (bytes, message) in [ + (vec![0; 7], "missing header length"), + (100_000_001_u64.to_le_bytes().to_vec(), "header too large"), + (9_u64.to_le_bytes().to_vec(), "truncated header"), + ] { + fs::write(f.base.join("model.safetensors"), bytes).unwrap(); + f.both_reject(message); + } + write_header(&f.base.join("model.safetensors"), "{", 0); + f.both_reject("EOF while parsing"); + write_header( + &f.base.join("model.safetensors"), + r#"{"language":{"data_offsets":[0,0]},"language":{"data_offsets":[0,0]}}"#, + 0, + ); + f.both_reject("duplicate JSON key: language"); +} + +#[test] +fn ignored_language_offsets_are_checked_before_safetensors_arithmetic() { + let f = Fixture::new(); + for offsets in [[1, 0], [0, 2], [0, usize::MAX]] { + let header = json!({"model.language.weight": {"dtype": "U8", "shape": [1], "data_offsets": offsets}}); + write_header(&f.base.join("model.safetensors"), &header.to_string(), 1); + f.both_reject("tensor offsets exceed payload for model.language.weight"); + } + write_header(&f.adapter.join("adapter_model.safetensors"), + &json!({"base_model.model.model.language.lora_A.weight": {"data_offsets": [0, usize::MAX]}}).to_string(), 0); + write_tensors(&f.base.join("model.safetensors"), &f.tensors); + f.cua_reject("tensor offsets exceed payload"); +} + +#[test] +fn both_base_entries_require_exact_dtype_shape_and_visual_inventory() { + let f = Fixture::new(); + let name = "model.visual.patch_embed.proj.bias"; + let mut wrong = f.tensors.clone(); + wrong.get_mut(name).unwrap().0 = Dtype::F32; + write_tensors(&f.base.join("model.safetensors"), &wrong); + f.both_reject("dtype mismatch"); + wrong = f.tensors.clone(); + wrong.get_mut(name).unwrap().1 = vec![1023]; + write_tensors(&f.base.join("model.safetensors"), &wrong); + f.both_reject("shape mismatch"); + wrong = f.tensors.clone(); + wrong.remove(name); + write_tensors(&f.base.join("model.safetensors"), &wrong); + f.both_reject("missing visual tensor"); + wrong = f.tensors.clone(); + wrong.insert("other.visual.weight".into(), (Dtype::BF16, vec![1])); + write_tensors(&f.base.join("model.safetensors"), &wrong); + f.both_reject("unexpected visual tensor"); +} + +#[test] +fn cua_lora_requires_exact_f32_shape_and_visual_inventory() { + let f = Fixture::new(); + let name = "base_model.model.model.visual.blocks.0.mlp.linear_fc1.lora_A.weight"; + let mut wrong = f.lora.clone(); + wrong.get_mut(name).unwrap().0 = Dtype::BF16; + write_tensors(&f.adapter.join("adapter_model.safetensors"), &wrong); + f.cua_reject("dtype mismatch"); + wrong = f.lora.clone(); + wrong.get_mut(name).unwrap().1 = vec![16, 1023]; + write_tensors(&f.adapter.join("adapter_model.safetensors"), &wrong); + f.cua_reject("shape mismatch"); + wrong = f.lora.clone(); + wrong.remove(name); + write_tensors(&f.adapter.join("adapter_model.safetensors"), &wrong); + f.cua_reject("missing visual tensor"); + wrong = f.lora.clone(); + wrong.insert("other.visual.lora_A.weight".into(), (Dtype::F32, vec![1])); + write_tensors(&f.adapter.join("adapter_model.safetensors"), &wrong); + f.cua_reject("unexpected visual tensor"); +} + +#[test] +fn cua_index_rejects_duplicates_inventory_mismatch_and_escaping_shards() { + let f = Fixture::new(); + let name = "model.visual.patch_embed.proj.bias"; + fs::write( + f.base.join("model.safetensors.index.json"), + format!( + r#"{{"weight_map":{{"{name}":"model.safetensors","{name}":"other.safetensors"}}}}"# + ), + ) + .unwrap(); + f.cua_reject("duplicate JSON key"); + let mut map = f.weight_map(); + map.remove(name); + f.index(&map); + f.cua_reject("missing visual tensor in index"); + map = f.weight_map(); + map.insert("other.visual.weight".into(), "model.safetensors".into()); + f.index(&map); + f.cua_reject("unexpected visual tensor in index"); + map = f.weight_map(); + map.insert(name.into(), "other.safetensors".into()); + write_tensors(&f.base.join("other.safetensors"), &f.tensors); + f.index(&map); + f.cua_reject("index mismatch"); + for filename in ["../outside.safetensors", "/outside.safetensors"] { + let map = f + .weight_map() + .into_keys() + .map(|name| (name, filename.into())) + .collect(); + f.index(&map); + f.cua_reject("path must remain inside checkpoint directory"); + } +} + +#[cfg(unix)] +#[test] +fn both_entries_reject_config_and_tensor_symlinks_outside_checkpoint() { + use std::os::unix::fs::symlink; + for config in [true, false] { + let f = Fixture::new(); + let filename = if config { + "config.json" + } else { + "model.safetensors" + }; + let outside = f.root.join("outside"); + fs::rename(f.base.join(filename), &outside).unwrap(); + symlink(&outside, f.base.join(filename)).unwrap(); + if !config { + fs::remove_file(f.base.join("vision.safetensors")).unwrap(); + symlink(&outside, f.base.join("vision.safetensors")).unwrap(); + } + f.both_reject("path escapes checkpoint directory"); + } + let f = Fixture::new(); + let outside = f.root.join("outside"); + fs::rename(f.adapter.join("adapter_model.safetensors"), &outside).unwrap(); + symlink(&outside, f.adapter.join("adapter_model.safetensors")).unwrap(); + f.cua_reject("path escapes checkpoint directory"); +} + +#[test] +fn qwen_27b_layout_does_not_expand_cua_4b_support() { + let f = Fixture::new(); + let config_path = f.base.join("config.json"); + let mut config: Value = serde_json::from_slice(&fs::read(&config_path).unwrap()).unwrap(); + for (key, value) in [ + ("depth", 27), + ("hidden_size", 1152), + ("intermediate_size", 4304), + ("out_hidden_size", 5120), + ] { + config["vision_config"][key] = json!(value); + } + config["text_config"]["hidden_size"] = json!(5120); + let vision = + omni_qwen3_5_native::vision::VisionConfig::from_value(config["vision_config"].clone()) + .unwrap(); + fs::write(config_path, config.to_string()).unwrap(); + write_tensors( + &f.base.join("vision.safetensors"), + &tensor_specs(&vision.inventory(), Dtype::BF16), + ); + assert_eq!( + QwenCheckpoint::load(&f.base).unwrap().config().head_dim(), + 72 + ); + f.cua_reject("expected pinned Qwen3.5-4B layout"); +} diff --git a/tests/jev_vl/caches.rs b/tests/jev_vl/caches.rs new file mode 100644 index 00000000..85e57576 --- /dev/null +++ b/tests/jev_vl/caches.rs @@ -0,0 +1,73 @@ +use std::sync::Arc; + +use omni_jev_vl_native::caches::{CacheCfg, Caches}; +use omni_jev_vl_native::images::ImageAsset; + +const ASSET_BYTES: usize = 5120 * 2 + 64; + +fn asset() -> Arc { + Arc::new(ImageAsset { + grid_thw: [1, 2, 2], + embeddings: vec![half::bf16::ONE; 5120], + }) +} + +fn caches(enabled: bool, l2: bool, budget: usize) -> Arc { + Caches::new(CacheCfg { + enabled, + l1: false, + l2, + l3: false, + l1_max: 0, + l2_bytes: budget, + l3_bytes: 0, + }) +} + +#[test] +fn l2_cold_rechecks_count_each_lookup() { + let caches = caches(true, true, ASSET_BYTES); + assert!(caches.l2_get("image").is_none()); + assert!(caches.l2_get("image").is_none()); + let stats = caches.snapshot(); + assert_eq!(stats.l2_hit, 0); + assert_eq!(stats.l2_miss, 2); +} + +#[test] +fn l2_recheck_sees_a_queued_publication_and_refreshes_lru() { + let caches = caches(true, true, ASSET_BYTES * 2); + // Preparation misses; another admitted request then publishes this image. + assert!(caches.l2_get("queued").is_none()); + let queued = asset(); + caches.l2_insert("queued".into(), queued.clone()); + caches.l2_insert("older".into(), asset()); + assert!(Arc::ptr_eq(&caches.l2_get("queued").unwrap(), &queued)); + caches.l2_insert("newer".into(), asset()); + assert!(caches.l2_get("older").is_none()); + assert!(Arc::ptr_eq(&caches.l2_get("queued").unwrap(), &queued)); + let stats = caches.snapshot(); + assert_eq!(stats.l2_hit, 2); + assert_eq!(stats.l2_miss, 2); + assert_eq!(stats.l2_records, 2); + assert_eq!(stats.l2_bytes, (ASSET_BYTES * 2) as u64); +} + +#[test] +fn l2_disabled_or_over_budget_rechecks_remain_counted_misses() { + for (enabled, l2, budget) in [ + (false, true, ASSET_BYTES), + (true, false, ASSET_BYTES), + (true, true, ASSET_BYTES - 1), + ] { + let caches = caches(enabled, l2, budget); + caches.l2_insert("image".into(), asset()); + assert!(caches.l2_get("image").is_none()); + assert!(caches.l2_get("image").is_none()); + let stats = caches.snapshot(); + assert_eq!(stats.l2_hit, 0); + assert_eq!(stats.l2_miss, 2); + assert_eq!(stats.l2_records, 0); + assert_eq!(stats.l2_bytes, 0); + } +} diff --git a/tests/jev_vl/vision.rs b/tests/jev_vl/vision.rs index aa446dd4..2ea06c72 100644 --- a/tests/jev_vl/vision.rs +++ b/tests/jev_vl/vision.rs @@ -15,7 +15,9 @@ fn source_selection_checks_presence_and_never_falls_back_from_online() { let dir = tempfile::tempdir().unwrap(); let missing = dir.path().join("missing"); assert!(ImageSource::from_paths(dir.path(), Some(missing.clone()), None).is_err()); - assert!(ImageSource::from_paths(dir.path(), Some(dir.path().into()), Some(missing)).is_err()); + assert!( + ImageSource::from_paths(dir.path(), Some(dir.path().into()), Some(missing.into())).is_err() + ); assert!(matches!( ImageSource::from_paths(dir.path(), Some(dir.path().into()), None).unwrap(), ImageSource::Online(_) @@ -26,6 +28,31 @@ fn source_selection_checks_presence_and_never_falls_back_from_online() { )); } +#[cfg(unix)] +#[test] +fn non_utf8_imgcache_preserves_default_selection_and_online_mode() { + use std::os::unix::ffi::OsStringExt; + + let dir = tempfile::tempdir().unwrap(); + let default = dir.path().join("imgcache"); + std::fs::create_dir(&default).unwrap(); + let non_utf8 = std::ffi::OsString::from_vec(vec![b'i', 0xff]); + let source = ImageSource::from_paths(dir.path(), None, Some(non_utf8.clone())).unwrap(); + let ImageSource::Prepared(Some(selected)) = source else { + panic!("non-UTF8 imgcache must retain the default directory") + }; + assert_eq!(selected, default); + + assert!(ImageSource::from_paths(dir.path(), Some(dir.path().join(&non_utf8)), None).is_err()); + let vision = dir.path().join("vision"); + std::fs::create_dir(&vision).unwrap(); + let source = ImageSource::from_paths(dir.path(), Some(vision.clone()), Some(non_utf8)).unwrap(); + let ImageSource::Online(selected) = source else { + panic!("non-UTF8 imgcache must not conflict with online vision") + }; + assert_eq!(selected, vision); +} + #[test] fn processor_requires_the_frozen_resize_and_normalization_contract() { let valid = processor(); diff --git a/tests/qwen3_5/kernels.rs b/tests/qwen3_5/kernels.rs index 7a217c81..af68e1f8 100644 --- a/tests/qwen3_5/kernels.rs +++ b/tests/qwen3_5/kernels.rs @@ -473,6 +473,240 @@ fn flash_attention_matches_float64_reference() { } } +#[test] +#[ignore = "needs a GPU and CUA_S1_CUDA_LIB with vision v2 symbols"] +fn vision_v2_head_dim72_position_and_rope_match_rounded_reference() { + let st = setup(); + let v2 = api().vision_v2().unwrap(); + let (hidden, dh) = (1152usize, 72usize); + let table = random(32 * hidden, 11, 0.5); + let table_device = to_device(&table, st); + for n in [1usize, 3, 65] { + let input = random(n * hidden, 12, 1.0); + let indices: Vec = (0..n * 4).map(|i| ((i * 7 + 3) % 32) as i32).collect(); + let weights: Vec = [0.125, 0.375, 0.25, 0.25].repeat(n); + let index_bytes: Vec = indices.iter().flat_map(|v| v.to_le_bytes()).collect(); + let index_device = DeviceBuffer::new(index_bytes.len()).unwrap(); + let weight_device = f32_to_device(&weights, st); + let positioned = to_device(&input, st); + // SAFETY: n complete hidden-width rows, four in-range table indices and + // weights per token, and immutable table/index buffers on this stream. + unsafe { + cuda::upload(index_device.at(0), &index_bytes, st).unwrap(); + check( + (v2.position)( + positioned.at(0), + table_device.at(0), + index_device.at(0).cast(), + weight_device.at(0).cast(), + n as i32, + hidden as i32, + st, + ), + "vision v2 positions", + ) + .unwrap(); + } + let actual = from_device(&positioned, n * hidden, st); + for token in 0..n { + for channel in 0..hidden { + let mut position = 0f32; + for corner in 0..4 { + let index = token * 4 + corner; + position += + table[indices[index] as usize * hidden + channel].to_f32() * weights[index]; + } + let expected = bf16::from_f32( + input[token * hidden + channel].to_f32() + bf16::from_f32(position).to_f32(), + ); + assert_eq!( + actual[token * hidden + channel].to_bits(), + expected.to_f32().to_bits(), + "position n={n}, token={token}, channel={channel}" + ); + } + } + + let qkv = random(n * hidden * 3, 13, 2.0); + let angles: Vec = (0..n * dh / 2).map(|i| i as f32 * 0.03125).collect(); + let cos: Vec = angles.iter().map(|v| v.cos()).collect(); + let sin: Vec = angles.iter().map(|v| v.sin()).collect(); + let qkv_device = to_device(&qkv, st); + let cos_device = f32_to_device(&cos, st); + let sin_device = f32_to_device(&sin, st); + let q = DeviceBuffer::new(n * hidden * 2).unwrap(); + let k = DeviceBuffer::new(n * hidden * 2).unwrap(); + // SAFETY: QKV has n interleaved 3*hidden rows, the FP32 tables have + // n*(dh/2) entries, and Q/K have n*hidden disjoint BF16 entries. + unsafe { + check( + (v2.rope)( + qkv_device.at(0), + cos_device.at(0).cast(), + sin_device.at(0).cast(), + q.at(0), + k.at(0), + n as i32, + hidden as i32, + dh as i32, + st, + ), + "vision v2 rotary", + ) + .unwrap(); + } + for (part, buffer) in [(0, &q), (1, &k)] { + let actual = from_device(buffer, n * hidden, st); + for token in 0..n { + for channel in 0..hidden { + let d = channel % dh; + let half = dh / 2; + let partner = if d < half { + channel + half + } else { + channel - half + }; + let sign = if d < half { -1.0 } else { 1.0 }; + let offset = token * hidden * 3 + part * hidden; + let angle = token * half + d % half; + let expected = bf16::from_f32( + qkv[offset + channel].to_f32() * cos[angle] + + sign * qkv[offset + partner].to_f32() * sin[angle], + ); + assert_eq!( + actual[token * hidden + channel].to_bits(), + expected.to_f32().to_bits(), + "rope n={n}, part={part}, token={token}, channel={channel}" + ); + } + } + } + } + // SAFETY: every operation/download above completed before stream retirement. + check( + unsafe { (api().cs1_stream_destroy)(st) }, + "destroy vision test stream", + ) + .unwrap(); +} + +#[test] +#[ignore = "needs a GPU and CUA_S1_CUDA_LIB with vision v2 symbols"] +fn vision_v2_head_dim72_padding_attention_and_unpack() { + let st = setup(); + let v2 = api().vision_v2().unwrap(); + let (heads, dh, padded_dh) = (16usize, 72usize, 80usize); + let hidden = heads * dh; + for (n, amplitude) in [(1usize, 8.0), (63, 2.0), (64, 0.5), (65, 0.5), (139, 2.0)] { + let q = random(n * hidden, 21, amplitude); + let k = random(n * hidden, 22, amplitude); + let qkv = random(n * hidden * 3, 23, 1.0); + let q_device = to_device(&q, st); + let k_device = to_device(&k, st); + let qkv_device = to_device(&qkv, st); + let output = DeviceBuffer::new(n * hidden * 2).unwrap(); + let count = n * heads * padded_dh; + // Nonzero initial contents expose missing zero writes to padded columns. + let workspace = to_device(&vec![bf16::from_f32(9.0); count * 4], st); + // SAFETY: Q/K are compact, V starts at the third slice of each 3*hidden + // QKV row, and workspace contains four n*heads*80 BF16 buffers. + unsafe { + check( + (v2.attention)( + q_device.at(0), + k_device.at(0), + qkv_device.at(hidden * 2 * 2), + output.at(0), + n as i32, + heads as i32, + dh as i32, + workspace.at(0), + st, + ), + "vision v2 attention", + ) + .unwrap(); + } + let actual = from_device(&output, n * hidden, st); + let padded = from_device(&workspace, count * 4, st); + assert!(actual.iter().all(|v| v.is_finite())); + for token in 0..n { + for head in 0..heads { + for d in 0..padded_dh { + let offset = (token * heads + head) * padded_dh + d; + for (part, source) in [(0, &q), (1, &k), (2, &qkv)] { + let expected = if d < dh { + let index = if part == 2 { + token * hidden * 3 + hidden * 2 + head * dh + d + } else { + (token * heads + head) * dh + d + }; + source[index].to_f32() + } else { + 0.0 + }; + assert_eq!(padded[part * count + offset].to_bits(), expected.to_bits()); + } + if d < dh { + assert_eq!( + actual[(token * heads + head) * dh + d].to_bits(), + padded[3 * count + offset].to_bits(), + "unpack n={n}, token={token}, head={head}, d={d}" + ); + } + } + } + } + let mut worst = 0f64; + // Vision attention is noncausal. Keep the existing FlashAttention test's + // per-row relative error metric and 1.6e-2 budget, including tail rows. + for row in (0..n).step_by(n.div_ceil(64)).chain([n - 1]) { + for head in 0..heads { + let query = &q[(row * heads + head) * dh..][..dh]; + let scores: Vec = (0..n) + .map(|token| { + query + .iter() + .zip(&k[(token * heads + head) * dh..][..dh]) + .map(|(q, k)| q.to_f64() * k.to_f64()) + .sum::() + / (dh as f64).sqrt() + }) + .collect(); + let max = scores.iter().copied().fold(f64::NEG_INFINITY, f64::max); + let weights: Vec = scores.iter().map(|v| (v - max).exp()).collect(); + let sum: f64 = weights.iter().sum(); + let reference: Vec = (0..dh) + .map(|d| { + (0..n) + .map(|token| { + weights[token] + * qkv[token * hidden * 3 + hidden * 2 + head * dh + d].to_f64() + }) + .sum::() + / sum + }) + .collect(); + let difference = actual[(row * heads + head) * dh..][..dh] + .iter() + .zip(&reference) + .map(|(got, want)| (*got as f64 - want).abs()) + .fold(0f64, f64::max); + let scale = reference.iter().map(|v| v.abs()).fold(1e-3, f64::max); + worst = worst.max(difference / scale); + } + } + eprintln!("vision v2 dh=72 n={n}: largest relative difference {worst:.2e}"); + assert!(worst < 1.6e-2, "n={n}: {worst}"); + } + // SAFETY: every operation/download above completed before stream retirement. + check( + unsafe { (api().cs1_stream_destroy)(st) }, + "destroy vision test stream", + ) + .unwrap(); +} + /// Transformers' torch_recurrent_gated_delta_rule in float64, one token at a time, /// with the L2 norms of q and k and q scaled by K^-1/2. #[allow(clippy::too_many_arguments)] From ea48a42c2d12928245c3f16f59567f7183a265dd Mon Sep 17 00:00:00 2001 From: linear3735 Date: Tue, 13 Oct 2026 00:27:33 +0800 Subject: [PATCH 09/20] fix: keep the original request error when post-error drains also fail The sticky drain checks ran after `language_drain?; vision_drain?` would have been fine only when the request succeeded; on a failed request the first `?` on drain results replaced the root-cause error with a drain error and dropped the original context entirely. Merge drain failures into the existing result instead: the request error wins, and a drain failure on an already-failed request is logged rather than substituted. Co-Authored-By: Claude Code --- src/models/jev_vl/native/src/executor.rs | 24 ++++++++++++++++++++---- 1 file changed, 20 insertions(+), 4 deletions(-) diff --git a/src/models/jev_vl/native/src/executor.rs b/src/models/jev_vl/native/src/executor.rs index 36b4331c..9ad46055 100644 --- a/src/models/jev_vl/native/src/executor.rs +++ b/src/models/jev_vl/native/src/executor.rs @@ -396,10 +396,26 @@ impl Executor { // Keep admission until all queued device work has drained, including errors. let language_drain = model.synchronize(); let vision_drain = vision.as_ref().map(VisionModel::synchronize).transpose(); - // Both drain attempts must happen even if either one fails. - language_drain?; - vision_drain?; - let probabilities = result?; + // Both drain attempts must happen even if either one fails. The + // request's own error names the failing stage, so surface it ahead + // of sticky drain failures; a drain error only fails an otherwise + // successful request. + let mut probabilities = result; + if let Err(drain) = language_drain { + if probabilities.is_ok() { + probabilities = Err(drain).context("language drain"); + } else { + eprintln!("language drain also failed after the request error: {drain:#}"); + } + } + if let Err(drain) = vision_drain { + if probabilities.is_ok() { + probabilities = Err(drain).context("vision drain"); + } else { + eprintln!("vision drain also failed after the request error: {drain:#}"); + } + } + let probabilities = probabilities?; for (key, asset) in pending_images { caches.l2_insert(key, asset); } From 79c3b170f25276c7848599275da6f3a6d01f1773 Mon Sep 17 00:00:00 2001 From: linear3735 Date: Tue, 13 Oct 2026 00:27:41 +0800 Subject: [PATCH 10/20] fix: derive the shared vision config from the parsed struct Re-listing all thirteen VisionConfig fields in a json! literal could silently drop any member added later; serialize the parsed struct instead so the shared model configuration always sees the full set. Co-Authored-By: Claude Code --- src/models/cua_s1/native/src/vision/mod.rs | 4 ++-- src/models/cua_s1/native/src/vision/model.rs | 13 ++++++------- 2 files changed, 8 insertions(+), 9 deletions(-) diff --git a/src/models/cua_s1/native/src/vision/mod.rs b/src/models/cua_s1/native/src/vision/mod.rs index d0230542..ea6428f2 100644 --- a/src/models/cua_s1/native/src/vision/mod.rs +++ b/src/models/cua_s1/native/src/vision/mod.rs @@ -8,7 +8,7 @@ use anyhow::{Context, Result, ensure}; use omni_qwen3_5_native::vision::{TensorStore, UniqueMap, inside, is_visual}; use safetensors::{Dtype, tensor::TensorView}; -use serde::Deserialize; +use serde::{Deserialize, Serialize}; use serde_json::Value; use std::{ collections::{BTreeMap, BTreeSet}, @@ -21,7 +21,7 @@ const ADAPTER: &str = "base_model.model.model.visual."; type Inventory = BTreeMap>; /// The supported Qwen3.5-4B vision architecture; all fields are validated at load. -#[derive(Debug, Deserialize)] +#[derive(Debug, Deserialize, Serialize)] pub struct VisionConfig { pub depth: usize, pub hidden_size: usize, diff --git a/src/models/cua_s1/native/src/vision/model.rs b/src/models/cua_s1/native/src/vision/model.rs index 29f84521..5a1a62fe 100644 --- a/src/models/cua_s1/native/src/vision/model.rs +++ b/src/models/cua_s1/native/src/vision/model.rs @@ -9,13 +9,12 @@ impl VisionModel { pub fn load(base: impl AsRef, adapter: impl AsRef, library: &Path) -> Result { let checkpoint = VisionCheckpoint::load(base, adapter)?; let c = checkpoint.config(); - let config = omni_qwen3_5_native::vision::VisionConfig::from_value(serde_json::json!({ - "depth":c.depth,"hidden_size":c.hidden_size,"intermediate_size":c.intermediate_size, - "num_heads":c.num_heads,"num_position_embeddings":c.num_position_embeddings, - "out_hidden_size":c.out_hidden_size,"in_channels":c.in_channels,"patch_size":c.patch_size, - "temporal_patch_size":c.temporal_patch_size,"spatial_merge_size":c.spatial_merge_size, - "hidden_act":c.hidden_act,"deepstack_visual_indexes":c.deepstack_visual_indexes,"model_type":c.model_type - }))?; + // Serialize the parsed struct instead of re-listing its fields: a member + // added to VisionConfig can no longer be silently dropped from the + // shared model configuration. + let config = omni_qwen3_5_native::vision::VisionConfig::from_value( + serde_json::to_value(c)?, + )?; let base = checkpoint .base_names() .map(|n| Ok((n.to_owned(), checkpoint.base_tensor(n)?))) From 08a13924d03e70bc089b3866e79956bce1303e3e Mon Sep 17 00:00:00 2001 From: linear3735 Date: Tue, 13 Oct 2026 00:27:57 +0800 Subject: [PATCH 11/20] test: pin every vision exporter rejection gate with mocks The single existing test covered only three of the four backbone dimension checks. Extend it to the top-level model_type and cover the remaining gates in execution order: pre-existing output, the exact eight-key processor contract (drifted value, extra key, missing key), non-language adapter tensors, an empty vision tensor set, shard paths escaping the model directory, and non-BF16 or non-finite weights; each asserts weights are never saved and no partial output is left behind. A happy-path test pins the manifest: exact format and model id, three output file hashes and the three checkpoint pin hashes recomputed from disk. Co-Authored-By: Claude Code --- tests/jev_vl/test_export_vision.py | 232 ++++++++++++++++++++++++++++- 1 file changed, 228 insertions(+), 4 deletions(-) diff --git a/tests/jev_vl/test_export_vision.py b/tests/jev_vl/test_export_vision.py index 6312d24c..2d021ef1 100644 --- a/tests/jev_vl/test_export_vision.py +++ b/tests/jev_vl/test_export_vision.py @@ -1,6 +1,7 @@ """Reject unsupported vision exports before loading weights or dependencies.""" import copy +import hashlib import importlib.util import json from pathlib import Path @@ -16,10 +17,102 @@ export_vision = importlib.util.module_from_spec(SPEC) SPEC.loader.exec_module(export_vision) +VALID_CONFIG = json.loads((REPO / "tests/qwen3_5/data/config.json").read_text()) + +# The exporter compares the whole file content against this exact contract: +# wrong values, extra keys and missing keys must all reject. +EXPECTED_PROCESSOR = { + "size": {"longest_edge": 16777216, "shortest_edge": 65536}, + "patch_size": 16, "temporal_patch_size": 2, "merge_size": 2, + "image_mean": [0.5, 0.5, 0.5], "image_std": [0.5, 0.5, 0.5], + "processor_class": "Qwen3VLProcessor", + "image_processor_type": "Qwen2VLImageProcessorFast", +} + +LANGUAGE_ADAPTER = { + "base_model.model.model.language_model.layers.0.mlp.gate_proj.lora_A.weight", + "base_model.model.lm_head.lora_A.weight", +} +VISUAL_TENSORS = ( + "model.visual.blocks.0.attn.qkv.weight", + "model.visual.merger.linear_fc2.weight", +) +SHARD = "model-00001-of-00002.safetensors" + + +class _Finite: + def __init__(self, value): + self._value = value + + def all(self): + return self._value + + +class _Weight: + def __init__(self, dtype, finite=True): + self.dtype = dtype + self._finite = finite + + def contiguous(self): + return self + + +class _FakeSafeOpen: + def __init__(self, weights): + self._weights = weights + + def __enter__(self): + return self + + def __exit__(self, *exc): + return False + + def keys(self): + return list(self._weights) + + def get_tensor(self, name): + return self._weights[name] + + +def _sha256(path): + return hashlib.sha256(path.read_bytes()).hexdigest() + class ExportVisionTests(unittest.TestCase): + def _fixtures(self, temporary, config=None, processor=None, index=None): + model = Path(temporary) / "model" + (model / "adapter_vllm").mkdir(parents=True) + (model / "config.json").write_text(json.dumps( + config if config is not None else VALID_CONFIG)) + (model / "preprocessor_config.json").write_text(json.dumps( + processor if processor is not None else EXPECTED_PROCESSOR)) + if index is not None: + (model / "model.safetensors.index.json").write_text( + json.dumps({"weight_map": index})) + (model / "adapter_vllm" / "adapter_config.json").write_text("{}") + (model / "adapter_vllm" / "adapter_model.safetensors").write_bytes(b"adapter") + return model, Path(temporary) / "export" + + def _mock_stack(self, bf16=object(), adapter=None, shard=None, + save_effect=None): + torch = mock.Mock(name="torch") + torch.bfloat16 = bf16 + torch.isfinite = lambda tensor: _Finite(tensor._finite) + files = {} + if adapter is not None: + files["adapter_model.safetensors"] = adapter + if shard is not None: + files[SHARD] = shard + read_weights = mock.Mock(side_effect=lambda path, framework: + _FakeSafeOpen(files[Path(path).name])) + save_weights = mock.Mock(side_effect=save_effect) + return read_weights, save_weights, mock.patch.dict(sys.modules, { + "torch": torch, + "safetensors": mock.Mock(safe_open=read_weights), + "safetensors.torch": mock.Mock(save_file=save_weights), + }) + def test_rejects_wrong_dimensions_before_weights_and_output(self): - valid = json.loads((REPO / "tests/qwen3_5/data/config.json").read_text()) read_weights = mock.Mock() save_weights = mock.Mock() with tempfile.TemporaryDirectory() as temporary, mock.patch.dict(sys.modules, { @@ -31,13 +124,17 @@ def test_rejects_wrong_dimensions_before_weights_and_output(self): model.mkdir() out = Path(temporary) / "export" for section, field, value in ( + ("", "model_type", "qwen2_vl"), ("vision_config", "hidden_size", 1024), ("vision_config", "out_hidden_size", 2560), ("text_config", "hidden_size", 2560), ): - with self.subTest(section=section, field=field): - config = copy.deepcopy(valid) - config[section][field] = value + with self.subTest(section=section or "toplevel", field=field): + config = copy.deepcopy(VALID_CONFIG) + if section: + config[section][field] = value + else: + config[field] = value (model / "config.json").write_text(json.dumps(config)) with self.assertRaisesRegex(ValueError, "expected the JEV-27B-VL"): export_vision.export(model, out) @@ -45,6 +142,133 @@ def test_rejects_wrong_dimensions_before_weights_and_output(self): save_weights.assert_not_called() self.assertFalse(out.exists()) + def test_rejects_existing_output_before_reading_the_model(self): + with tempfile.TemporaryDirectory() as temporary: + model, out = self._fixtures(temporary) + out.mkdir() + with self.assertRaisesRegex(ValueError, "output already exists"): + export_vision.export(model, out) + + def test_rejects_processor_contract_mismatch_before_weights(self): + extra = dict(EXPECTED_PROCESSOR, do_convert_rgb=True) + missing = {key: value for key, value in EXPECTED_PROCESSOR.items() + if key != "merge_size"} + drifted = dict(EXPECTED_PROCESSOR, patch_size=14) + with tempfile.TemporaryDirectory() as temporary: + model, out = self._fixtures(temporary) + for label, processor in ( + ("drifted value", drifted), + ("extra key", extra), + ("missing key", missing), + ): + with self.subTest(label=label): + (model / "preprocessor_config.json").write_text( + json.dumps(processor)) + read_weights, save_weights, patched = self._mock_stack() + with patched, self.assertRaisesRegex( + ValueError, "unsupported image processor"): + export_vision.export(model, out) + read_weights.assert_not_called() + save_weights.assert_not_called() + self.assertFalse(out.exists()) + + def test_rejects_non_language_adapter_tensor(self): + with tempfile.TemporaryDirectory() as temporary: + model, out = self._fixtures(temporary, index={ + name: SHARD for name in VISUAL_TENSORS}) + adapter = set(LANGUAGE_ADAPTER) | { + "base_model.model.model.visual.patch_embed.proj.weight"} + read_weights, save_weights, patched = self._mock_stack( + adapter={name: mock.Mock() for name in adapter}) + with patched, self.assertRaisesRegex( + ValueError, "unsupported non-language adapter tensor"): + export_vision.export(model, out) + save_weights.assert_not_called() + self.assertFalse(out.exists()) + + def test_rejects_checkpoint_without_vision_tensors(self): + with tempfile.TemporaryDirectory() as temporary: + model, out = self._fixtures(temporary, index={ + "model.language_model.layers.0.self_attn.q_proj.weight": SHARD}) + read_weights, save_weights, patched = self._mock_stack( + adapter={name: mock.Mock() for name in LANGUAGE_ADAPTER}) + with patched, self.assertRaisesRegex( + ValueError, "no vision tensors"): + export_vision.export(model, out) + save_weights.assert_not_called() + self.assertFalse(out.exists()) + + def test_rejects_shard_escaping_model_directory(self): + with tempfile.TemporaryDirectory() as temporary: + model, out = self._fixtures( + temporary, index={VISUAL_TENSORS[0]: "../escaped.safetensors"}) + read_weights, save_weights, patched = self._mock_stack( + adapter={name: mock.Mock() for name in LANGUAGE_ADAPTER}) + with patched, self.assertRaisesRegex( + ValueError, "shard escapes model directory"): + export_vision.export(model, out) + save_weights.assert_not_called() + self.assertFalse(out.exists()) + + def test_rejects_non_bf16_or_non_finite_vision_weights(self): + bf16 = object() + with tempfile.TemporaryDirectory() as temporary: + model, out = self._fixtures(temporary, index={ + name: SHARD for name in VISUAL_TENSORS}) + for label, weight in ( + ("wrong dtype", _Weight(object())), + ("non-finite", _Weight(bf16, finite=False)), + ): + with self.subTest(label=label): + read_weights, save_weights, patched = self._mock_stack( + bf16=bf16, + adapter={name: mock.Mock() for name in LANGUAGE_ADAPTER}, + shard={VISUAL_TENSORS[0]: weight, + VISUAL_TENSORS[1]: _Weight(bf16)}) + with patched, self.assertRaisesRegex( + ValueError, "expected finite BF16 vision weights"): + export_vision.export(model, out) + save_weights.assert_not_called() + self.assertFalse(out.exists()) + + def test_export_writes_manifest_pins_for_valid_checkpoint(self): + bf16 = object() + saved = {} + + def record(tensors, path): + saved.update(tensors) + path.write_bytes(b"vision") + + with tempfile.TemporaryDirectory() as temporary: + model, out = self._fixtures(temporary, index={ + name: SHARD for name in VISUAL_TENSORS}) + read_weights, save_weights, patched = self._mock_stack( + bf16=bf16, + adapter={name: mock.Mock() for name in LANGUAGE_ADAPTER}, + shard={name: _Weight(bf16) for name in VISUAL_TENSORS}, + save_effect=record) + with patched: + export_vision.export(model, out) + self.assertEqual(set(saved), set(VISUAL_TENSORS)) + for name in ("config.json", "preprocessor_config.json", + "vision.safetensors", "jev_vl_vision.json"): + self.assertTrue((out / name).exists(), name) + manifest = json.loads((out / "jev_vl_vision.json").read_text()) + self.assertEqual(manifest["format"], "jev-vl-vision/1") + self.assertEqual(manifest["model_id"], "autotrust/JEV-27B-VL") + self.assertEqual(manifest["files"], { + name: _sha256(out / name) + for name in ("config.json", "preprocessor_config.json", + "vision.safetensors")}) + self.assertEqual(manifest["pins"], { + "model_index_sha256": _sha256( + model / "model.safetensors.index.json"), + "adapter_config_sha256": _sha256( + model / "adapter_vllm" / "adapter_config.json"), + "adapter_model_sha256": _sha256( + model / "adapter_vllm" / "adapter_model.safetensors"), + }) + if __name__ == "__main__": unittest.main() From adf7ffc90ca37a194bcf25f5564bba259a593523 Mon Sep 17 00:00:00 2001 From: linear3735 Date: Tue, 13 Oct 2026 11:11:23 +0800 Subject: [PATCH 12/20] style: rustfmt the shared vision config derivation Co-Authored-By: Claude Code --- src/models/cua_s1/native/src/vision/model.rs | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/models/cua_s1/native/src/vision/model.rs b/src/models/cua_s1/native/src/vision/model.rs index 5a1a62fe..7464bd3e 100644 --- a/src/models/cua_s1/native/src/vision/model.rs +++ b/src/models/cua_s1/native/src/vision/model.rs @@ -12,9 +12,8 @@ impl VisionModel { // Serialize the parsed struct instead of re-listing its fields: a member // added to VisionConfig can no longer be silently dropped from the // shared model configuration. - let config = omni_qwen3_5_native::vision::VisionConfig::from_value( - serde_json::to_value(c)?, - )?; + let config = + omni_qwen3_5_native::vision::VisionConfig::from_value(serde_json::to_value(c)?)?; let base = checkpoint .base_names() .map(|n| Ok((n.to_owned(), checkpoint.base_tensor(n)?))) From a5be750f8756d535fd660e66fdd89fd88da816eb Mon Sep 17 00:00:00 2001 From: linear3735 Date: Tue, 13 Oct 2026 11:11:23 +0800 Subject: [PATCH 13/20] fix: let the documented inline-image caps past the HTTP body limit MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The 4 MiB axum DefaultBodyLimit rejected every request carrying even one maximum-size image (4 MiB raw ≈ 5.33 MiB base64) before the structured validators ran, so the advertised per-image caps were unreachable and their BadRequest rejections dead code. Over-limit clients also got axum's plain-text 413 instead of the worker's error envelope. Raise the transport cap to 8 MiB (the number the Cua-S1 vision worker already uses) so at least one maximum-size image fits, and map transport rejections into the official envelope with status 413. Pin both halves with raw-HTTP tests through the production limit layer: an over-limit body returns the 413 envelope, and a 5 MiB body now reaches request validation instead of dying in transport. Co-Authored-By: Claude Code --- recipe/jev_vl/README.md | 8 ++- src/models/jev_vl/native/src/contract.rs | 23 ++++++++ src/models/jev_vl/native/src/main.rs | 37 ++++++++++-- tests/jev_vl/http.rs | 74 +++++++++++++++++++++--- 4 files changed, 125 insertions(+), 17 deletions(-) diff --git a/recipe/jev_vl/README.md b/recipe/jev_vl/README.md index d5234511..b8f703a8 100644 --- a/recipe/jev_vl/README.md +++ b/recipe/jev_vl/README.md @@ -107,9 +107,11 @@ assets or retained image-prefix state. Online mode accepts single-frame inline PNG/JPEG data URIs. Each image is bounded to 4 MiB of decoded file bytes, 2048 pixels per side, 1,048,576 source pixels, -and 4608 vision patches. The full HTTP body is limited to 4 MiB, including base64 -and text. Remote URLs, animations and video are unsupported. Oversized inputs -are rejected; resource limits do not change the model's resize policy. +and 4608 vision patches. The full HTTP body is limited to 8 MiB, including base64 +and text, so at least one maximum-size image fits; requests beyond the body +limit are rejected with status 413 in the same error envelope. Remote URLs, +animations and video are unsupported. Oversized inputs are rejected; resource +limits do not change the model's resize policy. No per-request image-count limit applies beyond the body and token budgets. A minimum-size image uses 1.5 MiB of FP32 processed pixels and produces diff --git a/src/models/jev_vl/native/src/contract.rs b/src/models/jev_vl/native/src/contract.rs index f373213d..2e645087 100644 --- a/src/models/jev_vl/native/src/contract.rs +++ b/src/models/jev_vl/native/src/contract.rs @@ -62,6 +62,29 @@ impl Reject { Some("model"), ) } + + /// Transport-level 413: keeps the envelope instead of axum's plain text. + pub fn payload_too_large() -> Self { + Self::new( + 413, + format!( + "The request body exceeds the {} MiB transport limit; send fewer or smaller inline images.", + MAX_BODY_BYTES >> 20 + ), + "BadRequestError", + Some("body"), + ) + } +} + +/// Transport cap for one POST body. It must hold at least one maximum-size +/// image (4 MiB raw ≈ 5.33 MiB base64) plus JSON framing and text; together +/// with the token budget it is the only per-request image-count limiter. +pub const MAX_BODY_BYTES: usize = 8 << 20; + +/// Lowercase hex rendering shared by cache keys and export pin checks. +pub(crate) fn hex_digest(digest: &[u8]) -> String { + digest.iter().map(|b| format!("{b:02x}")).collect() } #[derive(Clone, Copy, PartialEq, Debug)] diff --git a/src/models/jev_vl/native/src/main.rs b/src/models/jev_vl/native/src/main.rs index 7a52e42f..a5fca186 100644 --- a/src/models/jev_vl/native/src/main.rs +++ b/src/models/jev_vl/native/src/main.rs @@ -6,12 +6,12 @@ use anyhow::{Context, Result, ensure}; use axum::{ Json, Router, body::Bytes, - extract::{DefaultBodyLimit, State}, + extract::{DefaultBodyLimit, State, rejection::BytesRejection}, http::{HeaderMap, StatusCode, Uri}, response::{IntoResponse, Response}, routing::{get, post}, }; -use omni_jev_vl_native::contract::{MODEL_ID, Reject}; +use omni_jev_vl_native::contract::{MAX_BODY_BYTES, MODEL_ID, Reject}; use omni_jev_vl_native::engine::Engine; use omni_qwen3_5_native::cuda; use serde_json::{Value, json}; @@ -68,7 +68,30 @@ async fn decide(engine: &Engine, raw: &[u8]) -> Response { } } -async fn systemone(State(engine): State>, headers: HeaderMap, body: Bytes) -> Response { +/// Body buffering happens before routing, so transport-level rejections would +/// bypass the worker's error envelope unless the handlers map them explicitly. +fn body_bytes(body: Result) -> Result { + match body { + Ok(bytes) => Ok(bytes), + Err(rejection) if rejection.status() == StatusCode::PAYLOAD_TOO_LARGE => { + Err(reject(Reject::payload_too_large())) + } + Err(rejection) => Err(reject(Reject::bad_request(format!( + "could not buffer the request body: {}", + rejection.body_text() + )))), + } +} + +async fn systemone( + State(engine): State>, + headers: HeaderMap, + body: Result, +) -> Response { + let body = match body_bytes(body) { + Ok(body) => body, + Err(response) => return response, + }; // Upstream pydantic validates the raw body even for a non-JSON content type; // mirroring it keeps the wrong-content-type probe's 400 (not 415) semantics. let ctype = headers @@ -91,7 +114,11 @@ async fn systemone(State(engine): State>, headers: HeaderMap, body: decide(&engine, &body).await } -async fn fallback(uri: Uri, body: Bytes) -> Response { +async fn fallback(uri: Uri, body: Result) -> Response { + let body = match body_bytes(body) { + Ok(body) => body, + Err(response) => return response, + }; let model = serde_json::from_slice::(&body) .ok() .and_then(|v| v.get("model").and_then(Value::as_str).map(str::to_owned)); @@ -161,7 +188,7 @@ async fn main() -> Result<()> { .route("/v1/cache/stats", get(cache_stats)) .route("/v1/cache/reset", post(cache_reset)) .fallback(fallback) - .layer(DefaultBodyLimit::max(4 << 20)) + .layer(DefaultBodyLimit::max(MAX_BODY_BYTES)) .with_state(engine); let listener = tokio::net::TcpListener::bind((host.as_str(), port)).await?; println!("listening on {host}:{port}"); diff --git a/tests/jev_vl/http.rs b/tests/jev_vl/http.rs index 5ba25c78..69ee2c8a 100644 --- a/tests/jev_vl/http.rs +++ b/tests/jev_vl/http.rs @@ -22,21 +22,18 @@ fn online_vision_startup_warmup_is_inline_image_decision() { assert_eq!(pixels.image_tokens(), 64); } -#[tokio::test(flavor = "multi_thread", worker_threads = 2)] -async fn unknown_model_chat_probe_returns_canonical_worker_error() { +/// Raw HTTP/1.0 POST against a bound test router; returns the full response. +async fn post_raw(app: Router, route: &'static str, body: String) -> String { let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); let address = listener.local_addr().unwrap(); - // This route needs no model or GPU; exercise the production fallback over HTTP. - let app = Router::new().fallback(fallback); let server = tokio::spawn(async move { axum::serve(listener, app).await.unwrap() }); let response = tokio::task::spawn_blocking(move || { - let body = r#"{"model":"no-such-model","messages":[{"role":"user","content":"hi"}],"max_tokens":1}"#; let mut stream = TcpStream::connect(address).unwrap(); - stream.set_read_timeout(Some(Duration::from_secs(5))).unwrap(); - stream.set_write_timeout(Some(Duration::from_secs(5))).unwrap(); + stream.set_read_timeout(Some(Duration::from_secs(15))).unwrap(); + stream.set_write_timeout(Some(Duration::from_secs(15))).unwrap(); write!( stream, - "POST /v1/chat/completions HTTP/1.0\r\nHost: {address}\r\nContent-Type: application/json\r\nContent-Length: {}\r\n\r\n{body}", + "POST {route} HTTP/1.0\r\nHost: {address}\r\nContent-Type: application/json\r\nContent-Length: {}\r\n\r\n{body}", body.len() ) .unwrap(); @@ -47,7 +44,21 @@ async fn unknown_model_chat_probe_returns_canonical_worker_error() { .await .unwrap(); server.abort(); - let (headers, body) = response.split_once("\r\n\r\n").unwrap(); + response +} + +fn split_response(response: &str) -> (&str, &str) { + response.split_once("\r\n\r\n").unwrap() +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn unknown_model_chat_probe_returns_canonical_worker_error() { + // This route needs no model or GPU; exercise the production fallback over HTTP. + let app = Router::new().fallback(fallback); + let body = + r#"{"model":"no-such-model","messages":[{"role":"user","content":"hi"}],"max_tokens":1}"#; + let response = post_raw(app, "/v1/chat/completions", body.to_owned()).await; + let (headers, body) = split_response(&response); assert_eq!(headers.split_whitespace().nth(1), Some("404")); assert!( headers @@ -64,3 +75,48 @@ async fn unknown_model_chat_probe_returns_canonical_worker_error() { }}) ); } + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn over_limit_body_returns_envelope_413() { + // The production body-cap layer and rejection mapping, minus the Engine: + // an over-limit body must surface the worker's error envelope, not axum's + // plain-text 413. + let app = Router::new() + .fallback(fallback) + .layer(DefaultBodyLimit::max( + omni_jev_vl_native::contract::MAX_BODY_BYTES, + )); + let body = format!( + r#"{{"model":"no-such-model","pad":"{}"}}"#, + "A".repeat(omni_jev_vl_native::contract::MAX_BODY_BYTES) + ); + let response = post_raw(app, "/v1/systemone", body).await; + let (headers, body) = split_response(&response); + assert_eq!(headers.split_whitespace().nth(1), Some("413")); + let error = serde_json::from_str::(body).unwrap(); + let error = error["error"].clone(); + assert_eq!(error["type"], json!("BadRequestError")); + assert_eq!(error["param"], json!("body")); + assert_eq!(error["code"], json!(413)); + assert!(error["message"].as_str().unwrap().contains("8 MiB")); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn body_within_transport_limit_reaches_request_validation() { + // Bodies above the old 4 MiB layer but within the transport cap must reach + // the structured handlers, so the advertised inline-image caps stay usable. + let app = Router::new() + .fallback(fallback) + .layer(DefaultBodyLimit::max( + omni_jev_vl_native::contract::MAX_BODY_BYTES, + )); + let body = format!( + r#"{{"model":"no-such-model","pad":"{}"}}"#, + "A".repeat(5 << 20) + ); + let response = post_raw(app, "/v1/systemone", body).await; + let (headers, body) = split_response(&response); + assert_eq!(headers.split_whitespace().nth(1), Some("404")); + let error = serde_json::from_str::(body).unwrap(); + assert_eq!(error["error"]["code"], json!(404)); +} From 1cde60f920ac80bbee1713af104a542d689dc75e Mon Sep 17 00:00:00 2001 From: linear3735 Date: Tue, 13 Oct 2026 11:11:23 +0800 Subject: [PATCH 14/20] fix: pin-check the vision export off the async runtime Engine::load streamed the full vision.safetensors through sha256 on a Tokio worker thread. Move validate_export into spawn_blocking so the GB-scale startup pass no longer occupies the async runtime; the executor's device upload already re-reads the same file once. Co-Authored-By: Claude Code --- src/models/jev_vl/native/src/engine.rs | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/src/models/jev_vl/native/src/engine.rs b/src/models/jev_vl/native/src/engine.rs index 19b63b7f..93d36335 100644 --- a/src/models/jev_vl/native/src/engine.rs +++ b/src/models/jev_vl/native/src/engine.rs @@ -72,7 +72,16 @@ impl Engine { ); let vision_dir = match &source { ImageSource::Online(vision_dir) => { - vision::validate_export(vision_dir, dir, &manifest)?; + // Pin-checking streams the full vision.safetensors through + // sha256; keep that GB-scale pass off the async runtime. + let language_dir = dir.to_owned(); + let vision_dir_owned = vision_dir.clone(); + let manifest_owned = manifest.clone(); + tokio::task::spawn_blocking(move || { + vision::validate_export(&vision_dir_owned, &language_dir, &manifest_owned) + }) + .await + .context("vision export pin check failed to run")??; Some(vision_dir.clone()) } ImageSource::Prepared(_) => None, From 89559ad7d20e65d3bcf308ed1baeefc5378cb827 Mon Sep 17 00:00:00 2001 From: linear3735 Date: Tue, 13 Oct 2026 11:11:23 +0800 Subject: [PATCH 15/20] refactor: crate-scope the image limits and share the hex digest rendering ImageLimits and preprocess_rgb8_with_limits had public visibility with a single consumer (the cua() preset behind preprocess_rgb8); keeping them crate-private keeps the limit surface single-sourced so a future caller cannot compose caps the model side never validated. url_key and the export pin check each hand-rolled digest-to-lowercase-hex formatting; route both through contract::hex_digest so the L2 keys, imgcache directories and manifest pin hashes cannot drift apart. Co-Authored-By: Claude Code --- src/models/jev_vl/native/src/processing.rs | 5 +---- src/models/jev_vl/native/src/vision.rs | 6 +----- src/models/qwen3_5/native/src/image_preprocess.rs | 7 +++++-- 3 files changed, 7 insertions(+), 11 deletions(-) diff --git a/src/models/jev_vl/native/src/processing.rs b/src/models/jev_vl/native/src/processing.rs index 317c75f9..43199242 100644 --- a/src/models/jev_vl/native/src/processing.rs +++ b/src/models/jev_vl/native/src/processing.rs @@ -74,10 +74,7 @@ impl Processor { fn url_key(url: &str) -> String { use sha2::Digest; - sha2::Sha256::digest(url.as_bytes()) - .iter() - .map(|b| format!("{b:02x}")) - .collect() + crate::contract::hex_digest(&sha2::Sha256::digest(url.as_bytes())) } /// CPU image preparation checks the full prompt budget before GPU admission. diff --git a/src/models/jev_vl/native/src/vision.rs b/src/models/jev_vl/native/src/vision.rs index 7f033d4e..7d03ba60 100644 --- a/src/models/jev_vl/native/src/vision.rs +++ b/src/models/jev_vl/native/src/vision.rs @@ -54,11 +54,7 @@ fn sha256(path: &Path) -> Result { } hash.update(&buffer[..n]); } - Ok(hash - .finalize() - .iter() - .map(|byte| format!("{byte:02x}")) - .collect()) + Ok(crate::contract::hex_digest(&hash.finalize())) } fn hash_value(value: &Value) -> Result<&str> { diff --git a/src/models/qwen3_5/native/src/image_preprocess.rs b/src/models/qwen3_5/native/src/image_preprocess.rs index 0e50f644..ca5692b2 100644 --- a/src/models/qwen3_5/native/src/image_preprocess.rs +++ b/src/models/qwen3_5/native/src/image_preprocess.rs @@ -17,8 +17,11 @@ const MERGE_SIZE: usize = 2; const FACTOR: usize = PATCH_SIZE * MERGE_SIZE; const PATCH_VALUES: usize = 3 * 2 * PATCH_SIZE * PATCH_SIZE; /// Source validation and upstream smart-resize limits. Prompt budgeting belongs to callers. +/// +/// Crate-private until a second consumer exists, so the limit surface stays +/// single-sourced through [`preprocess_rgb8`]. #[derive(Clone, Copy, Debug)] -pub struct ImageLimits { +pub(crate) struct ImageLimits { pub max_source_side: usize, pub max_source_pixels: usize, pub min_pixels: usize, @@ -65,7 +68,7 @@ pub fn preprocess_rgb8(width: usize, height: usize, rgb: &[u8]) -> Result Date: Tue, 13 Oct 2026 11:12:47 +0800 Subject: [PATCH 16/20] fix: keep the true status for non-limit body rejections The body-mapping fallback mapped every non-413 transport rejection to 400; a mid-read connection failure is axum's 500, not a client error. Add Reject::internal and route that branch through it. Co-Authored-By: Claude Code --- src/models/jev_vl/native/src/contract.rs | 5 +++++ src/models/jev_vl/native/src/main.rs | 2 +- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/src/models/jev_vl/native/src/contract.rs b/src/models/jev_vl/native/src/contract.rs index 2e645087..be1624d8 100644 --- a/src/models/jev_vl/native/src/contract.rs +++ b/src/models/jev_vl/native/src/contract.rs @@ -75,6 +75,11 @@ impl Reject { Some("body"), ) } + + /// Unexpected server-side failure (e.g. the body channel broke mid-read). + pub fn internal(message: impl Into) -> Self { + Self::new(500, message.into(), "InternalServerError", None) + } } /// Transport cap for one POST body. It must hold at least one maximum-size diff --git a/src/models/jev_vl/native/src/main.rs b/src/models/jev_vl/native/src/main.rs index a5fca186..9ab2a0dd 100644 --- a/src/models/jev_vl/native/src/main.rs +++ b/src/models/jev_vl/native/src/main.rs @@ -76,7 +76,7 @@ fn body_bytes(body: Result) -> Result { Err(rejection) if rejection.status() == StatusCode::PAYLOAD_TOO_LARGE => { Err(reject(Reject::payload_too_large())) } - Err(rejection) => Err(reject(Reject::bad_request(format!( + Err(rejection) => Err(reject(Reject::internal(format!( "could not buffer the request body: {}", rejection.body_text() )))), From 0f834a95bf1eeb7c6f9430b67a798877fb657770 Mon Sep 17 00:00:00 2001 From: linear3735 Date: Tue, 13 Oct 2026 11:37:25 +0800 Subject: [PATCH 17/20] fix: mirror axum's 400 for non-limit body rejections A truncated upload surfaces as axum's UnknownBodyError, which is 400; mapping it to 500 relabeled an interrupted client delivery as a server fault (and broke the status the un-mapped route already returned). Keep the envelope but mirror axum's statuses: 413 over the cap, 400 for other buffer errors, so Reject::internal is removed again. Returning Reject from the mapper, with each handler rendering the response, also keeps the helper's error type small enough for clippy::result_large_err under -D warnings. Pin the 400 path with a truncated-upload regression. Co-Authored-By: Claude Code --- src/models/jev_vl/native/src/contract.rs | 5 --- src/models/jev_vl/native/src/main.rs | 13 +++---- tests/jev_vl/http.rs | 43 ++++++++++++++++++++++++ 3 files changed, 50 insertions(+), 11 deletions(-) diff --git a/src/models/jev_vl/native/src/contract.rs b/src/models/jev_vl/native/src/contract.rs index be1624d8..2e645087 100644 --- a/src/models/jev_vl/native/src/contract.rs +++ b/src/models/jev_vl/native/src/contract.rs @@ -75,11 +75,6 @@ impl Reject { Some("body"), ) } - - /// Unexpected server-side failure (e.g. the body channel broke mid-read). - pub fn internal(message: impl Into) -> Self { - Self::new(500, message.into(), "InternalServerError", None) - } } /// Transport cap for one POST body. It must hold at least one maximum-size diff --git a/src/models/jev_vl/native/src/main.rs b/src/models/jev_vl/native/src/main.rs index 9ab2a0dd..3c7a7fa8 100644 --- a/src/models/jev_vl/native/src/main.rs +++ b/src/models/jev_vl/native/src/main.rs @@ -70,16 +70,17 @@ async fn decide(engine: &Engine, raw: &[u8]) -> Response { /// Body buffering happens before routing, so transport-level rejections would /// bypass the worker's error envelope unless the handlers map them explicitly. -fn body_bytes(body: Result) -> Result { +/// Status codes mirror axum's: 413 over the cap, 400 for other buffer errors. +fn body_bytes(body: Result) -> Result { match body { Ok(bytes) => Ok(bytes), Err(rejection) if rejection.status() == StatusCode::PAYLOAD_TOO_LARGE => { - Err(reject(Reject::payload_too_large())) + Err(Reject::payload_too_large()) } - Err(rejection) => Err(reject(Reject::internal(format!( + Err(rejection) => Err(Reject::bad_request(format!( "could not buffer the request body: {}", rejection.body_text() - )))), + ))), } } @@ -90,7 +91,7 @@ async fn systemone( ) -> Response { let body = match body_bytes(body) { Ok(body) => body, - Err(response) => return response, + Err(rejection) => return reject(rejection), }; // Upstream pydantic validates the raw body even for a non-JSON content type; // mirroring it keeps the wrong-content-type probe's 400 (not 415) semantics. @@ -117,7 +118,7 @@ async fn systemone( async fn fallback(uri: Uri, body: Result) -> Response { let body = match body_bytes(body) { Ok(body) => body, - Err(response) => return response, + Err(rejection) => return reject(rejection), }; let model = serde_json::from_slice::(&body) .ok() diff --git a/tests/jev_vl/http.rs b/tests/jev_vl/http.rs index 69ee2c8a..faa3e789 100644 --- a/tests/jev_vl/http.rs +++ b/tests/jev_vl/http.rs @@ -120,3 +120,46 @@ async fn body_within_transport_limit_reaches_request_validation() { let error = serde_json::from_str::(body).unwrap(); assert_eq!(error["error"]["code"], json!(404)); } + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn truncated_body_keeps_axums_400_status() { + // A truncated upload (declared length never delivered, then EOF) hits + // axum's UnknownBodyError, which is 400 — mapping it must not relabel the + // client's interrupted delivery as a server fault. + let app = Router::new() + .fallback(fallback) + .layer(DefaultBodyLimit::max( + omni_jev_vl_native::contract::MAX_BODY_BYTES, + )); + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let address = listener.local_addr().unwrap(); + let server = tokio::spawn(async move { axum::serve(listener, app).await.unwrap() }); + let response = tokio::task::spawn_blocking(move || { + let mut stream = TcpStream::connect(address).unwrap(); + stream + .set_read_timeout(Some(Duration::from_secs(15))) + .unwrap(); + stream + .set_write_timeout(Some(Duration::from_secs(15))) + .unwrap(); + write!( + stream, + "POST /v1/systemone HTTP/1.0\r\nHost: {address}\r\nContent-Type: application/json\r\nContent-Length: 100\r\n\r\n" + ) + .unwrap(); + stream.write_all(b"{}").unwrap(); + stream.shutdown(std::net::Shutdown::Write).unwrap(); + let mut response = String::new(); + stream.read_to_string(&mut response).unwrap(); + response + }) + .await + .unwrap(); + server.abort(); + let (headers, body) = split_response(&response); + assert_eq!(headers.split_whitespace().nth(1), Some("400")); + let error = serde_json::from_str::(body).unwrap(); + let error = error["error"].clone(); + assert_eq!(error["type"], json!("BadRequestError")); + assert_eq!(error["code"], json!(400)); +} From 26dc76bf65c544815e7b285866791da1e703816d Mon Sep 17 00:00:00 2001 From: linear3735 Date: Tue, 13 Oct 2026 11:37:25 +0800 Subject: [PATCH 18/20] docs: sync the model contract body cap to 8 MiB The recipe README moved to 8 MiB with the transport fix; the model contract page still said 4 MiB. Co-Authored-By: Claude Code --- src/models/jev_vl/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/models/jev_vl/README.md b/src/models/jev_vl/README.md index b9581950..4826e5ce 100644 --- a/src/models/jev_vl/README.md +++ b/src/models/jev_vl/README.md @@ -61,7 +61,7 @@ list parts concatenate without an inserted separator. Each image part inserts `<|vision_start|><|image_pad|><|vision_end|>` before image-token expansion. The default exported limit is 16,384 expanded tokens. Export permits 1–32,768; -larger limits have not been benchmarked. HTTP bodies are limited to 4 MiB. +larger limits have not been benchmarked. HTTP bodies are limited to 8 MiB. `thinking=auto/on`, tournament and permutation strategies, generation, audio, video and dynamic batching are unsupported. Online mode accepts inline PNG/JPEG images; prepared mode requires exported image assets. `strategy=auto` From 5901747a2081fbdf26ea1254bfd3dde4bdb08d98 Mon Sep 17 00:00:00 2001 From: linear3735 Date: Wed, 21 Oct 2026 22:43:45 +0800 Subject: [PATCH 19/20] Drop CPU-only DeviceBuffer tests superseded by #117 device binding #117 binds every DeviceBuffer to its allocation device: DeviceBuffer::new now reads the current device through the loaded CUDA library before any allocation, so a zero-byte buffer can no longer be constructed on a CUDA-free host and the two CPU-only offset tests cannot run. The assertion they pinned is carried by upstream itself (assert!(offset <= self.bytes) in DeviceBuffer::at), so no product behavior is lost; the test target and file are removed. Co-Authored-By: Claude Code --- src/models/qwen3_5/native/Cargo.toml | 4 ---- tests/qwen3_5/device_buffer.rs | 14 -------------- 2 files changed, 18 deletions(-) delete mode 100644 tests/qwen3_5/device_buffer.rs diff --git a/src/models/qwen3_5/native/Cargo.toml b/src/models/qwen3_5/native/Cargo.toml index 76bc0702..ebb6c9d7 100644 --- a/src/models/qwen3_5/native/Cargo.toml +++ b/src/models/qwen3_5/native/Cargo.toml @@ -28,10 +28,6 @@ path = "../../../../tests/qwen3_5/kernels.rs" name = "image_decode" path = "../../../../tests/qwen3_5/image_decode.rs" -[[test]] -name = "device_buffer" -path = "../../../../tests/qwen3_5/device_buffer.rs" - [[test]] name = "device_features" path = "../../../../tests/qwen3_5/device_features.rs" diff --git a/tests/qwen3_5/device_buffer.rs b/tests/qwen3_5/device_buffer.rs deleted file mode 100644 index 1566fd66..00000000 --- a/tests/qwen3_5/device_buffer.rs +++ /dev/null @@ -1,14 +0,0 @@ -use omni_qwen3_5_native::cuda::DeviceBuffer; - -#[test] -fn zero_byte_buffer_accepts_zero_offset_without_cuda() { - let buffer = DeviceBuffer::new(0).unwrap(); - assert!(buffer.at(0).is_null()); -} - -#[test] -#[should_panic(expected = "offset <= self.bytes")] -fn zero_byte_buffer_rejects_out_of_bounds_offset_without_cuda() { - let buffer = DeviceBuffer::new(0).unwrap(); - buffer.at(1); -} From 034117ed4b75fb2ad5bdb8608509e1295a25f89d Mon Sep 17 00:00:00 2001 From: linear3735 Date: Wed, 21 Oct 2026 23:16:55 +0800 Subject: [PATCH 20/20] Restore public ImageLimits for the merged JEMM consumer The JEMM integration (#155, main 32bdd2a) consumes ImageLimits and preprocess_rgb8_with_limits from omni-jemm-native, so the second consumer the crate-private note was waiting for now exists on main; restore pub for both. No semantics change. Co-Authored-By: Claude Code --- src/models/qwen3_5/native/src/image_preprocess.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/models/qwen3_5/native/src/image_preprocess.rs b/src/models/qwen3_5/native/src/image_preprocess.rs index b0d9806a..52dec59c 100644 --- a/src/models/qwen3_5/native/src/image_preprocess.rs +++ b/src/models/qwen3_5/native/src/image_preprocess.rs @@ -18,10 +18,10 @@ const FACTOR: usize = PATCH_SIZE * MERGE_SIZE; const PATCH_VALUES: usize = 3 * 2 * PATCH_SIZE * PATCH_SIZE; /// Source validation and upstream smart-resize limits. Prompt budgeting belongs to callers. /// -/// Crate-private until a second consumer exists, so the limit surface stays +/// Shared by the cua-s1 and jemm consumers, so the limit surface stays /// single-sourced through [`preprocess_rgb8`]. #[derive(Clone, Copy, Debug)] -pub(crate) struct ImageLimits { +pub struct ImageLimits { pub max_source_side: usize, pub max_source_pixels: usize, pub min_pixels: usize, @@ -68,7 +68,7 @@ pub fn preprocess_rgb8(width: usize, height: usize, rgb: &[u8]) -> Result