From ef1fd634ebdf4aeff15b946fae732e1ae126b65d Mon Sep 17 00:00:00 2001 From: levius <2114377220@qq.com> Date: Thu, 8 Oct 2026 11:21:03 +0800 Subject: [PATCH 01/21] jemm: add native decision worker and pinned weight export --- Cargo.lock | 35 + Cargo.toml | 2 +- recipe/jemm/export.py | 338 +++ recipe/jemm/pinned_inventory.json | 2253 +++++++++++++++++ src/models/jemm/native/Cargo.toml | 18 + src/models/jemm/native/LICENSE.jemm | 201 ++ src/models/jemm/native/LICENSE.unicode | 39 + src/models/jemm/native/src/artifacts.rs | 97 + src/models/jemm/native/src/contract.rs | 411 +++ src/models/jemm/native/src/executor.rs | 287 +++ src/models/jemm/native/src/lib.rs | 6 + src/models/jemm/native/src/main.rs | 4 + src/models/jemm/native/src/processing.rs | 326 +++ src/models/jemm/native/src/server.rs | 131 + src/models/jemm/native/src/unicode.rs | 21 + .../jemm/native/src/unicode15_printable.bin | Bin 0 -> 5688 bytes 16 files changed, 4168 insertions(+), 1 deletion(-) create mode 100644 recipe/jemm/export.py create mode 100644 recipe/jemm/pinned_inventory.json create mode 100644 src/models/jemm/native/Cargo.toml create mode 100644 src/models/jemm/native/LICENSE.jemm create mode 100644 src/models/jemm/native/LICENSE.unicode create mode 100644 src/models/jemm/native/src/artifacts.rs create mode 100644 src/models/jemm/native/src/contract.rs create mode 100644 src/models/jemm/native/src/executor.rs create mode 100644 src/models/jemm/native/src/lib.rs create mode 100644 src/models/jemm/native/src/main.rs create mode 100644 src/models/jemm/native/src/processing.rs create mode 100644 src/models/jemm/native/src/server.rs create mode 100644 src/models/jemm/native/src/unicode.rs create mode 100644 src/models/jemm/native/src/unicode15_printable.bin diff --git a/Cargo.lock b/Cargo.lock index 2bb95d2f..f637bd76 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -862,6 +862,7 @@ checksum = "85ab80394333c02fe689eaf900ab500fbd0c2213da414687ebf995a65d5a6104" dependencies = [ "bytemuck", "byteorder-lite", + "image-webp", "moxcms", "num-traits", "png", @@ -869,6 +870,16 @@ dependencies = [ "zune-jpeg", ] +[[package]] +name = "image-webp" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "525e9ff3e1a4be2fbea1fdf0e98686a6d98b4d8f937e1bf7402245af1909e8c3" +dependencies = [ + "byteorder-lite", + "quick-error", +] + [[package]] name = "indexmap" version = "2.14.2" @@ -1123,6 +1134,24 @@ dependencies = [ "libloading", ] +[[package]] +name = "omni-jemm-native" +version = "0.1.0" +dependencies = [ + "anyhow", + "axum", + "base64 0.22.1", + "half", + "image", + "omni-qwen3-5-native", + "omni-runtime", + "safetensors 0.8.0", + "serde_json", + "sha2", + "tokenizers 0.22.2", + "tokio", +] + [[package]] name = "omni-jev" version = "0.1.0" @@ -1290,6 +1319,12 @@ version = "0.1.30" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d55d956fa96f5ec02be2e13af0e20391a5aa83d6a074e3ad368959d0fab299ea" +[[package]] +name = "quick-error" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a993555f31e5a609f617c12db6250dedcac1b0a85076912c436e6fc9b2c8e6a3" + [[package]] name = "quinn" version = "0.11.12" diff --git a/Cargo.toml b/Cargo.toml index 242d51ca..f8003d5d 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,3 +1,3 @@ [workspace] -members = ["src/frontend", "src/runtime", "src/models/clm", "src/models/cua_s1/native", "src/models/qwen3_5/native", "src/models/open_jev/native", "src/models/laya", "src/backends/cuda"] +members = ["src/frontend", "src/runtime", "src/models/clm", "src/models/cua_s1/native", "src/models/qwen3_5/native", "src/models/open_jev/native", "src/models/jemm/native", "src/models/laya", "src/backends/cuda"] resolver = "3" diff --git a/recipe/jemm/export.py b/recipe/jemm/export.py new file mode 100644 index 00000000..54cc23a1 --- /dev/null +++ b/recipe/jemm/export.py @@ -0,0 +1,338 @@ +"""Offline CPU export of pinned JEMM, one safetensors shard at a time. + +No full Transformers model is instantiated. Raw inputs remain untouched. The +completion marker is written last; restart with the same arguments to verify +and reuse atomically completed shards under the same producer script/runtime. +Changed producers require a new export directory. Needs roughly one base shard plus a +bounded FP32 LoRA delta in RAM, rather than two copies of the full model. +""" +import argparse +import gc +import hashlib +import importlib.metadata +import json +import os +import platform +import shutil +from pathlib import Path + +import torch +from safetensors import safe_open +from safetensors.torch import save_file + +BASE_MODEL_ID = "Qwen/Qwen3.8-27B" +PINS = { + "base_model_id": BASE_MODEL_ID, + "base_revision": "1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0", + "checkpoint_revision": "76e3c209e8441fa658221c7ba2725bad2f811176", + "source_revision": "6822fe0fd53c5e6670af6ba99fb2c857a661e532", +} +LABELS = "ABCDEFGHIJKLMNOPQRSTUVWXYZ012345" +SYSTEM = ("Choose the best available candidate for the question using only the supplied state. " + "Return exactly one candidate label.") +CALIBRATION = {"temperature": 1.3480874159655591, "mm_temperature": 1.3954832341582943, + "threshold": 0.9872681877423998} +INVENTORY_PATH = Path(__file__).with_name("pinned_inventory.json") + + +def sha256(path): + digest = hashlib.sha256() + with Path(path).open("rb") as stream: + for block in iter(lambda: stream.read(8 * 1024 * 1024), b""): + digest.update(block) + return digest.hexdigest() + + +def canonical_hash(value): + return hashlib.sha256(json.dumps(value, sort_keys=True, separators=(",", ":"), allow_nan=False).encode()).hexdigest() + + +def producer_runtime(): + versions = {} + for name in ("safetensors", "transformers", "tokenizers"): + try: + versions[name + "_version"] = importlib.metadata.version(name) + except importlib.metadata.PackageNotFoundError: + versions[name + "_version"] = None + return {"python_version": platform.python_version(), "platform": platform.platform(), "machine": platform.machine(), + "torch_version": str(torch.__version__), "torch_threads": torch.get_num_threads(), + "cpu_capability": torch.backends.cpu.get_cpu_capability(), + "float32_matmul_precision": torch.get_float32_matmul_precision(), **versions} + + +def atomic_json(path, value): + temporary = path.with_suffix(path.suffix + ".tmp") + with temporary.open("w", encoding="utf-8") as stream: + json.dump(value, stream, indent=2, ensure_ascii=False, allow_nan=False) + stream.write("\n") + stream.flush() + os.fsync(stream.fileno()) + os.replace(temporary, path) + + +def atomic_tensors(path, tensors): + temporary = path.with_suffix(path.suffix + ".tmp") + save_file(tensors, str(temporary), metadata={"format": "pt"}) + with temporary.open("rb") as stream: + os.fsync(stream.fileno()) + os.replace(temporary, path) + + +def safe_merge(base, a, b, scale, chunk_rows=1024): + """PEFT safe_merge semantics: FP32 B@A delta, in-place BF16 addition. + + Clone the base first and reject nonfinite adapter values and the final + rounded BF16 result. The caller's base tensor never changes. + """ + if base.dtype != torch.bfloat16 or a.dtype != torch.float32 or b.dtype != torch.float32: + raise ValueError("expected BF16 base and FP32 JEMM adapter") + if base.ndim != 2 or a.ndim != 2 or b.ndim != 2 or (b.shape[0], a.shape[1]) != tuple(base.shape) or b.shape[1] != a.shape[0]: + raise ValueError("LoRA dimensions do not match base") + if chunk_rows < 1 or not torch.isfinite(a).all() or not torch.isfinite(b).all(): + raise ValueError("nonfinite LoRA tensors or invalid chunk size") + result = base.clone() + for start in range(0, base.shape[0], chunk_rows): + end = min(start + chunk_rows, base.shape[0]) + delta = (b[start:end] @ a) * scale + result[start:end] += delta + if not torch.isfinite(result[start:end]).all(): + raise ValueError("nonfinite merged BF16 tensor") + return result + + +def validate_artifacts(directory, inventory): + """Verify every pinned file's bytes and tensor inventory without materializing weights.""" + hashes = {} + for name, expected in inventory["files"].items(): + if Path(name).name != name: + raise ValueError("inventory contains an unsafe filename") + path = directory / name + if not path.is_file(): + raise ValueError(f"missing pinned artifact: {path}") + actual = sha256(path) + if actual != expected: + raise ValueError(f"SHA256 mismatch: {path}") + hashes[name] = actual + observed = {} + for name in sorted({v["file"] for v in inventory["tensors"].values()}): + with safe_open(str(directory / name), framework="pt", device="cpu") as reader: + for key in reader.keys(): + if key in observed: + raise ValueError(f"duplicate tensor: {key}") + tensor = reader.get_slice(key) + observed[key] = {"shape": tensor.get_shape(), "dtype": tensor.get_dtype(), "file": name} + if observed != inventory["tensors"]: + missing = sorted(set(inventory["tensors"]) - set(observed)) + extra = sorted(set(observed) - set(inventory["tensors"])) + mismatch = sorted(k for k in observed.keys() & inventory["tensors"].keys() if observed[k] != inventory["tensors"][k]) + raise ValueError(f"tensor inventory mismatch: missing={missing[:8]} extra={extra[:8]} shape/dtype/file={mismatch[:8]}") + return hashes + + +def adapter_pairs(base_inventory, adapter_inventory, config): + if config.get("peft_type") != "LORA" or config.get("base_model_name_or_path") != BASE_MODEL_ID: + raise ValueError("adapter must target pinned Qwen3.8-27B") + for key in ("use_dora", "use_rslora", "use_qalora", "fan_in_fan_out", "lora_bias"): + if config.get(key): + raise ValueError(f"unsupported adapter option: {key}") + for key in ("rank_pattern", "alpha_pattern", "modules_to_save", "target_parameters", "layer_replication"): + if config.get(key): + raise ValueError(f"unsupported adapter option: {key}") + if config.get("bias", "none") != "none": + raise ValueError("adapter bias unsupported") + rank, alpha = config["r"], config["lora_alpha"] + if rank < 1 or alpha <= 0: + raise ValueError("invalid LoRA rank/alpha") + pairs = {} + for name, spec in adapter_inventory.items(): + prefix = "base_model.model." + if not name.startswith(prefix): + raise ValueError(f"unexpected adapter prefix: {name}") + target, letter = None, None + for candidate in ("A", "B"): + suffix = f".lora_{candidate}.weight" + if name.endswith(suffix): + target, letter = name[len(prefix):-len(suffix)] + ".weight", candidate + if target is None or not target.startswith("model.language_model.layers."): + raise ValueError(f"adapter targets unsupported module: {name}") + if target not in base_inventory or spec["dtype"] != "F32": + raise ValueError(f"adapter lacks matching base weight: {name}") + pair = pairs.setdefault(target, {}) + if letter in pair: + raise ValueError(f"duplicate LoRA component: {name}") + pair[letter] = name + for target, pair in pairs.items(): + if set(pair) != {"A", "B"}: + raise ValueError(f"unpaired LoRA tensor: {target}") + rows, cols = base_inventory[target]["shape"] + if adapter_inventory[pair["A"]]["shape"] != [rank, cols] or adapter_inventory[pair["B"]]["shape"] != [rows, rank]: + raise ValueError(f"LoRA shape mismatch: {target}") + return pairs, alpha / rank + + +def export_checkpoint(base, adapter, out, provenance, *, tokenizer=None, inventory=None): + """Public CLI uses the bundled pinned inventory. Tiny CPU tests inject a toy inventory.""" + base, adapter, out = Path(base).resolve(), Path(adapter).resolve(), Path(out).resolve() + if out == base or out == adapter or base in out.parents or adapter in out.parents: + raise ValueError("export directory must be separate from source checkpoints") + if any(provenance.get(key) != value for key, value in PINS.items()): + raise ValueError("download provenance does not match pinned revisions") + inventory = inventory or json.loads(INVENTORY_PATH.read_text()) + source_hashes = {"base": validate_artifacts(base, inventory["base"]), "adapter": validate_artifacts(adapter, inventory["adapter"])} + index = json.loads((base / "model.safetensors.index.json").read_text())["weight_map"] + expected_map = {key: value["file"] for key, value in inventory["base"]["tensors"].items()} + if index != expected_map: + raise ValueError("base index disagrees with pinned safetensors inventory") + config = json.loads((base / "config.json").read_text()) + text = config["text_config"] + if config.get("tie_word_embeddings", False) or text.get("tie_word_embeddings", False): + raise ValueError("JEMM needs the original untied lm_head") + if inventory["base"]["tensors"]["lm_head.weight"]["shape"] != [text["vocab_size"], text["hidden_size"]]: + raise ValueError("untied head shape disagrees with text configuration") + adapter_config = json.loads((adapter / "adapter_config.json").read_text()) + pairs, scale = adapter_pairs(inventory["base"]["tensors"], inventory["adapter"]["tensors"], adapter_config) + calibration = json.loads((adapter / "decision_config.json").read_text()) + if any(calibration.get(key) != value for key, value in CALIBRATION.items()): + raise ValueError("decision calibration differs from pinned JEMM") + if tokenizer is None: + from transformers import AutoTokenizer + tokenizer = AutoTokenizer.from_pretrained(str(base), local_files_only=True) + label_ids = [] + for label in LABELS: + ids = tokenizer.encode(label, add_special_tokens=False) + if len(ids) != 1 or not isinstance(ids[0], int) or not 0 <= ids[0] < text["vocab_size"]: + raise ValueError(f"label is not one valid vocabulary token: {label}") + label_ids.append(ids[0]) + if len(set(label_ids)) != len(LABELS): + raise ValueError("label token IDs must be unique") + if inventory.get("provenance") == PINS and label_ids != list(range(32, 58)) + list(range(15, 21)): + raise ValueError("tokenizer label IDs differ from the pinned JEMM readout") + marker = "\x00SYSTEM1_JEMM_USER\x00" + chat = tokenizer.apply_chat_template([{"role": "system", "content": SYSTEM}, {"role": "user", "content": marker}], + tokenize=False, add_generation_prompt=True, enable_thinking=False, preserve_thinking=False) + if chat.count(marker) != 1: + raise ValueError("chat template did not preserve a unique user marker") + chat_prefix, chat_suffix = chat.split(marker) + producer = {"exporter_sha256": sha256(Path(__file__)), "runtime": producer_runtime()} + contract = {"pins": PINS, "inventory_sha256": canonical_hash(inventory), "source_sha256": source_hashes, + "label_token_ids": label_ids, "chat_prefix": chat_prefix, "chat_suffix": chat_suffix, "producer": producer} + fingerprint = canonical_hash(contract) + manifest_path, progress_path = out / "jemm_export.json", out / "export_progress.json" + if manifest_path.exists(): + manifest = json.loads(manifest_path.read_text()) + if manifest.get("exporter_sha256") != producer["exporter_sha256"] or manifest.get("producer_runtime") != producer["runtime"]: + raise ValueError("existing export belongs to different producer; preserve it and choose a new export directory") + if manifest.get("export_fingerprint") != fingerprint: + raise ValueError("existing export belongs to different source artifacts or contract") + for name, expected in manifest["export_sha256"].items(): + if sha256(out / name) != expected: + raise ValueError(f"completed export checksum mismatch: {name}") + return manifest + if out.exists() and not progress_path.exists(): + raise ValueError("output directory exists without resumable export provenance") + progress = json.loads(progress_path.read_text()) if progress_path.exists() else {"fingerprint": fingerprint, "producer": producer, "completed": {}} + if progress.get("producer") != producer: + raise ValueError("partial export belongs to different producer; preserve it and choose a new export directory") + if progress.get("fingerprint") != fingerprint: + raise ValueError("partial export belongs to different source artifacts or contract") + # Check all adapter values before creating output, including projections in later shards. + adapter_file = adapter / "adapter_model.safetensors" + with safe_open(str(adapter_file), framework="pt", device="cpu") as lora: + for key in lora.keys(): + if not torch.isfinite(lora.get_tensor(key)).all(): + raise ValueError(f"nonfinite adapter tensor: {key}") + out.mkdir(parents=True, exist_ok=True) + atomic_json(progress_path, progress) + weight_map, total_bytes = {}, 0 + for number, shard in enumerate(sorted(set(index.values())), 1): + completed = progress["completed"].get(shard) + if completed is not None: + for name, expected in completed["outputs"].items(): + if sha256(out / name) != expected: + raise ValueError(f"partial export checksum mismatch: {name}") + weight_map.update(completed["weight_map"]) + total_bytes += completed["language_bytes"] + continue + language, visual, selected_head = {}, {}, None + with safe_open(str(base / shard), framework="pt", device="cpu") as reader: + for key in reader.keys(): + if key == "lm_head.weight": + # Reading individual rows avoids materializing the 2.54 GB full head. + view = reader.get_slice(key) + selected_head = torch.cat([view[token:token + 1] for token in label_ids], dim=0).contiguous() + elif key.startswith("model.visual."): + visual[key] = reader.get_tensor(key) + elif key.startswith("model.language_model."): + value = reader.get_tensor(key) + if key in pairs: + pair = pairs[key] + value = safe_merge(value, lora.get_tensor(pair["A"]), lora.get_tensor(pair["B"]), scale) + language[key[len("model.language_model."):]] = value + elif not key.startswith("mtp."): + raise ValueError(f"unexpected base tensor outside validated native scope: {key}") + outputs, mapping, language_bytes = {}, {}, 0 + if language: + filename = f"model-{number:05d}.safetensors" + atomic_tensors(out / filename, language) + outputs[filename] = sha256(out / filename) + mapping = {key: filename for key in language} + language_bytes = sum(value.numel() * value.element_size() for value in language.values()) + if visual: + filename = "vision.safetensors" + if any(filename in x["outputs"] for x in progress["completed"].values()): + raise ValueError("vision spans multiple shards; bundled inventory expected one") + atomic_tensors(out / filename, visual) + outputs[filename] = sha256(out / filename) + if selected_head is not None: + atomic_tensors(out / "jemm_lm_head.safetensors", {"weight": selected_head}) + outputs["jemm_lm_head.safetensors"] = sha256(out / "jemm_lm_head.safetensors") + progress["completed"][shard] = {"source_sha256": source_hashes["base"][shard], "outputs": outputs, + "weight_map": mapping, "language_bytes": language_bytes} + atomic_json(progress_path, progress) + weight_map.update(mapping) + total_bytes += language_bytes + del language, visual, selected_head + gc.collect() + print(f"completed {number}/{len(set(index.values()))}: {shard}", flush=True) + atomic_json(out / "model.safetensors.index.json", {"metadata": {"total_size": total_bytes}, "weight_map": weight_map}) + for name in inventory["base"]["files"]: + if not name.endswith(".safetensors") and name != "model.safetensors.index.json": + temporary = out / (name + ".tmp") + shutil.copyfile(base / name, temporary) + os.replace(temporary, out / name) + for name in ("adapter_config.json", "decision_config.json"): + shutil.copyfile(adapter / name, out / name) + exports = {p.name: sha256(p) for p in sorted(out.iterdir()) if p.is_file() and not p.name.endswith(".tmp") and p.name != "export_progress.json"} + if "jemm_lm_head.safetensors" not in exports or "vision.safetensors" not in exports: + raise ValueError("export lacks selected head or vision tensors") + manifest = {"format": "jemm-native/1", "model_id": "JEMM", **PINS, **CALIBRATION, + "max_tokens": 8192, "max_mm_tokens": 3072, "chat_prefix": chat_prefix, "chat_suffix": chat_suffix, + "enable_thinking": False, "preserve_thinking": False, "labels": list(LABELS), "label_token_ids": label_ids, + "label_head_file": "jemm_lm_head.safetensors", "label_head_tensor": "weight", "label_head_dtype": "BF16", + "label_head_shape": [32, text["hidden_size"]], "untied_lm_head": True, "vision_file": "vision.safetensors", + "lora_scale": scale, "lora_merge": "FP32 delta into BF16 base, finite checked", "source_sha256": source_hashes, + "upstream_source_sha256": inventory.get("upstream_source_sha256", {}), "export_sha256": exports, + "inventory_sha256": canonical_hash(inventory), "exporter_sha256": producer["exporter_sha256"], + "producer_runtime": producer["runtime"], + "download_provenance": provenance, "export_fingerprint": fingerprint, + "omitted_tensors": [k for k in index if k.startswith("mtp.") or k == "lm_head.weight"]} + atomic_json(manifest_path, manifest) + return manifest + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--base", type=Path, required=True) + parser.add_argument("--adapter", type=Path, required=True) + parser.add_argument("--out", type=Path, required=True) + parser.add_argument("--provenance", type=Path, required=True, help="JSON with the four pinned revision/model fields and download provenance") + parser.add_argument("--threads", type=int, default=4) + args = parser.parse_args() + if args.threads < 1: + parser.error("--threads must be positive") + torch.set_num_threads(args.threads) + export_checkpoint(args.base, args.adapter, args.out, json.loads(args.provenance.read_text())) + + +if __name__ == "__main__": + main() diff --git a/recipe/jemm/pinned_inventory.json b/recipe/jemm/pinned_inventory.json new file mode 100644 index 00000000..4153b3e1 --- /dev/null +++ b/recipe/jemm/pinned_inventory.json @@ -0,0 +1,2253 @@ +{ + "adapter": { + "files": { + "adapter_config.json": "7cfc1e46979f37a20012207b4ed72a720d24d74c9205952a5b62c1f81ee11427", + "adapter_model.safetensors": "fac397c8af737bb8d259efbe16f771507f77276fb44ab74942cd2e1bc05c3fef", + "decision_config.json": "545e999a6c76f66ea6f557d4fecad005223500595654265d24d194054da1aa2e" + }, + "tensors": { + "base_model.model.model.language_model.layers.0.linear_attn.in_proj_a.lora_A.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[16,5120]}, + "base_model.model.model.language_model.layers.0.linear_attn.in_proj_a.lora_B.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[48,16]}, + "base_model.model.model.language_model.layers.0.linear_attn.in_proj_b.lora_A.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[16,5120]}, + "base_model.model.model.language_model.layers.0.linear_attn.in_proj_b.lora_B.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[48,16]}, + "base_model.model.model.language_model.layers.0.linear_attn.in_proj_qkv.lora_A.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[16,5120]}, + "base_model.model.model.language_model.layers.0.linear_attn.in_proj_qkv.lora_B.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[10240,16]}, + "base_model.model.model.language_model.layers.0.linear_attn.in_proj_z.lora_A.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[16,5120]}, + "base_model.model.model.language_model.layers.0.linear_attn.in_proj_z.lora_B.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[6144,16]}, + "base_model.model.model.language_model.layers.0.linear_attn.out_proj.lora_A.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[16,6144]}, + "base_model.model.model.language_model.layers.0.linear_attn.out_proj.lora_B.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[5120,16]}, + "base_model.model.model.language_model.layers.0.mlp.down_proj.lora_A.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[16,17408]}, + "base_model.model.model.language_model.layers.0.mlp.down_proj.lora_B.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[5120,16]}, + "base_model.model.model.language_model.layers.0.mlp.gate_proj.lora_A.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[16,5120]}, + "base_model.model.model.language_model.layers.0.mlp.gate_proj.lora_B.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[17408,16]}, + "base_model.model.model.language_model.layers.0.mlp.up_proj.lora_A.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[16,5120]}, + "base_model.model.model.language_model.layers.0.mlp.up_proj.lora_B.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[17408,16]}, + "base_model.model.model.language_model.layers.1.linear_attn.in_proj_a.lora_A.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[16,5120]}, + "base_model.model.model.language_model.layers.1.linear_attn.in_proj_a.lora_B.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[48,16]}, + "base_model.model.model.language_model.layers.1.linear_attn.in_proj_b.lora_A.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[16,5120]}, + "base_model.model.model.language_model.layers.1.linear_attn.in_proj_b.lora_B.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[48,16]}, + "base_model.model.model.language_model.layers.1.linear_attn.in_proj_qkv.lora_A.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[16,5120]}, + "base_model.model.model.language_model.layers.1.linear_attn.in_proj_qkv.lora_B.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[10240,16]}, + "base_model.model.model.language_model.layers.1.linear_attn.in_proj_z.lora_A.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[16,5120]}, + "base_model.model.model.language_model.layers.1.linear_attn.in_proj_z.lora_B.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[6144,16]}, + "base_model.model.model.language_model.layers.1.linear_attn.out_proj.lora_A.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[16,6144]}, + "base_model.model.model.language_model.layers.1.linear_attn.out_proj.lora_B.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[5120,16]}, + "base_model.model.model.language_model.layers.1.mlp.down_proj.lora_A.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[16,17408]}, + "base_model.model.model.language_model.layers.1.mlp.down_proj.lora_B.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[5120,16]}, + "base_model.model.model.language_model.layers.1.mlp.gate_proj.lora_A.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[16,5120]}, + "base_model.model.model.language_model.layers.1.mlp.gate_proj.lora_B.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[17408,16]}, + "base_model.model.model.language_model.layers.1.mlp.up_proj.lora_A.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[16,5120]}, + "base_model.model.model.language_model.layers.1.mlp.up_proj.lora_B.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[17408,16]}, + "base_model.model.model.language_model.layers.10.linear_attn.in_proj_a.lora_A.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[16,5120]}, + "base_model.model.model.language_model.layers.10.linear_attn.in_proj_a.lora_B.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[48,16]}, + "base_model.model.model.language_model.layers.10.linear_attn.in_proj_b.lora_A.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[16,5120]}, + "base_model.model.model.language_model.layers.10.linear_attn.in_proj_b.lora_B.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[48,16]}, + "base_model.model.model.language_model.layers.10.linear_attn.in_proj_qkv.lora_A.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[16,5120]}, + "base_model.model.model.language_model.layers.10.linear_attn.in_proj_qkv.lora_B.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[10240,16]}, + "base_model.model.model.language_model.layers.10.linear_attn.in_proj_z.lora_A.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[16,5120]}, + "base_model.model.model.language_model.layers.10.linear_attn.in_proj_z.lora_B.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[6144,16]}, + "base_model.model.model.language_model.layers.10.linear_attn.out_proj.lora_A.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[16,6144]}, + "base_model.model.model.language_model.layers.10.linear_attn.out_proj.lora_B.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[5120,16]}, + "base_model.model.model.language_model.layers.10.mlp.down_proj.lora_A.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[16,17408]}, + "base_model.model.model.language_model.layers.10.mlp.down_proj.lora_B.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[5120,16]}, + "base_model.model.model.language_model.layers.10.mlp.gate_proj.lora_A.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[16,5120]}, + "base_model.model.model.language_model.layers.10.mlp.gate_proj.lora_B.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[17408,16]}, + "base_model.model.model.language_model.layers.10.mlp.up_proj.lora_A.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[16,5120]}, + "base_model.model.model.language_model.layers.10.mlp.up_proj.lora_B.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[17408,16]}, + "base_model.model.model.language_model.layers.11.mlp.down_proj.lora_A.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[16,17408]}, + "base_model.model.model.language_model.layers.11.mlp.down_proj.lora_B.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[5120,16]}, + "base_model.model.model.language_model.layers.11.mlp.gate_proj.lora_A.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[16,5120]}, + "base_model.model.model.language_model.layers.11.mlp.gate_proj.lora_B.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[17408,16]}, + "base_model.model.model.language_model.layers.11.mlp.up_proj.lora_A.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[16,5120]}, + "base_model.model.model.language_model.layers.11.mlp.up_proj.lora_B.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[17408,16]}, + "base_model.model.model.language_model.layers.11.self_attn.k_proj.lora_A.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[16,5120]}, + "base_model.model.model.language_model.layers.11.self_attn.k_proj.lora_B.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[1024,16]}, + "base_model.model.model.language_model.layers.11.self_attn.o_proj.lora_A.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[16,6144]}, + "base_model.model.model.language_model.layers.11.self_attn.o_proj.lora_B.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[5120,16]}, + "base_model.model.model.language_model.layers.11.self_attn.q_proj.lora_A.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[16,5120]}, + "base_model.model.model.language_model.layers.11.self_attn.q_proj.lora_B.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[12288,16]}, + "base_model.model.model.language_model.layers.11.self_attn.v_proj.lora_A.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[16,5120]}, + "base_model.model.model.language_model.layers.11.self_attn.v_proj.lora_B.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[1024,16]}, + "base_model.model.model.language_model.layers.12.linear_attn.in_proj_a.lora_A.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[16,5120]}, + "base_model.model.model.language_model.layers.12.linear_attn.in_proj_a.lora_B.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[48,16]}, + "base_model.model.model.language_model.layers.12.linear_attn.in_proj_b.lora_A.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[16,5120]}, + "base_model.model.model.language_model.layers.12.linear_attn.in_proj_b.lora_B.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[48,16]}, + "base_model.model.model.language_model.layers.12.linear_attn.in_proj_qkv.lora_A.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[16,5120]}, + "base_model.model.model.language_model.layers.12.linear_attn.in_proj_qkv.lora_B.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[10240,16]}, + "base_model.model.model.language_model.layers.12.linear_attn.in_proj_z.lora_A.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[16,5120]}, + "base_model.model.model.language_model.layers.12.linear_attn.in_proj_z.lora_B.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[6144,16]}, + "base_model.model.model.language_model.layers.12.linear_attn.out_proj.lora_A.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[16,6144]}, + "base_model.model.model.language_model.layers.12.linear_attn.out_proj.lora_B.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[5120,16]}, + "base_model.model.model.language_model.layers.12.mlp.down_proj.lora_A.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[16,17408]}, + "base_model.model.model.language_model.layers.12.mlp.down_proj.lora_B.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[5120,16]}, + "base_model.model.model.language_model.layers.12.mlp.gate_proj.lora_A.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[16,5120]}, + "base_model.model.model.language_model.layers.12.mlp.gate_proj.lora_B.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[17408,16]}, + "base_model.model.model.language_model.layers.12.mlp.up_proj.lora_A.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[16,5120]}, + "base_model.model.model.language_model.layers.12.mlp.up_proj.lora_B.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[17408,16]}, + "base_model.model.model.language_model.layers.13.linear_attn.in_proj_a.lora_A.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[16,5120]}, + "base_model.model.model.language_model.layers.13.linear_attn.in_proj_a.lora_B.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[48,16]}, + "base_model.model.model.language_model.layers.13.linear_attn.in_proj_b.lora_A.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[16,5120]}, + "base_model.model.model.language_model.layers.13.linear_attn.in_proj_b.lora_B.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[48,16]}, + "base_model.model.model.language_model.layers.13.linear_attn.in_proj_qkv.lora_A.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[16,5120]}, + "base_model.model.model.language_model.layers.13.linear_attn.in_proj_qkv.lora_B.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[10240,16]}, + "base_model.model.model.language_model.layers.13.linear_attn.in_proj_z.lora_A.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[16,5120]}, + "base_model.model.model.language_model.layers.13.linear_attn.in_proj_z.lora_B.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[6144,16]}, + "base_model.model.model.language_model.layers.13.linear_attn.out_proj.lora_A.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[16,6144]}, + "base_model.model.model.language_model.layers.13.linear_attn.out_proj.lora_B.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[5120,16]}, + "base_model.model.model.language_model.layers.13.mlp.down_proj.lora_A.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[16,17408]}, + "base_model.model.model.language_model.layers.13.mlp.down_proj.lora_B.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[5120,16]}, + "base_model.model.model.language_model.layers.13.mlp.gate_proj.lora_A.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[16,5120]}, + "base_model.model.model.language_model.layers.13.mlp.gate_proj.lora_B.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[17408,16]}, + "base_model.model.model.language_model.layers.13.mlp.up_proj.lora_A.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[16,5120]}, + "base_model.model.model.language_model.layers.13.mlp.up_proj.lora_B.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[17408,16]}, + "base_model.model.model.language_model.layers.14.linear_attn.in_proj_a.lora_A.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[16,5120]}, + "base_model.model.model.language_model.layers.14.linear_attn.in_proj_a.lora_B.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[48,16]}, + "base_model.model.model.language_model.layers.14.linear_attn.in_proj_b.lora_A.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[16,5120]}, + "base_model.model.model.language_model.layers.14.linear_attn.in_proj_b.lora_B.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[48,16]}, + "base_model.model.model.language_model.layers.14.linear_attn.in_proj_qkv.lora_A.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[16,5120]}, + "base_model.model.model.language_model.layers.14.linear_attn.in_proj_qkv.lora_B.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[10240,16]}, + "base_model.model.model.language_model.layers.14.linear_attn.in_proj_z.lora_A.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[16,5120]}, + "base_model.model.model.language_model.layers.14.linear_attn.in_proj_z.lora_B.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[6144,16]}, + "base_model.model.model.language_model.layers.14.linear_attn.out_proj.lora_A.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[16,6144]}, + "base_model.model.model.language_model.layers.14.linear_attn.out_proj.lora_B.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[5120,16]}, + "base_model.model.model.language_model.layers.14.mlp.down_proj.lora_A.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[16,17408]}, + "base_model.model.model.language_model.layers.14.mlp.down_proj.lora_B.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[5120,16]}, + "base_model.model.model.language_model.layers.14.mlp.gate_proj.lora_A.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[16,5120]}, + "base_model.model.model.language_model.layers.14.mlp.gate_proj.lora_B.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[17408,16]}, + "base_model.model.model.language_model.layers.14.mlp.up_proj.lora_A.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[16,5120]}, + "base_model.model.model.language_model.layers.14.mlp.up_proj.lora_B.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[17408,16]}, + "base_model.model.model.language_model.layers.15.mlp.down_proj.lora_A.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[16,17408]}, + "base_model.model.model.language_model.layers.15.mlp.down_proj.lora_B.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[5120,16]}, + "base_model.model.model.language_model.layers.15.mlp.gate_proj.lora_A.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[16,5120]}, + "base_model.model.model.language_model.layers.15.mlp.gate_proj.lora_B.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[17408,16]}, + "base_model.model.model.language_model.layers.15.mlp.up_proj.lora_A.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[16,5120]}, + "base_model.model.model.language_model.layers.15.mlp.up_proj.lora_B.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[17408,16]}, + "base_model.model.model.language_model.layers.15.self_attn.k_proj.lora_A.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[16,5120]}, + "base_model.model.model.language_model.layers.15.self_attn.k_proj.lora_B.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[1024,16]}, + "base_model.model.model.language_model.layers.15.self_attn.o_proj.lora_A.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[16,6144]}, + "base_model.model.model.language_model.layers.15.self_attn.o_proj.lora_B.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[5120,16]}, + "base_model.model.model.language_model.layers.15.self_attn.q_proj.lora_A.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[16,5120]}, + "base_model.model.model.language_model.layers.15.self_attn.q_proj.lora_B.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[12288,16]}, + "base_model.model.model.language_model.layers.15.self_attn.v_proj.lora_A.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[16,5120]}, + "base_model.model.model.language_model.layers.15.self_attn.v_proj.lora_B.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[1024,16]}, + "base_model.model.model.language_model.layers.16.linear_attn.in_proj_a.lora_A.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[16,5120]}, + "base_model.model.model.language_model.layers.16.linear_attn.in_proj_a.lora_B.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[48,16]}, + "base_model.model.model.language_model.layers.16.linear_attn.in_proj_b.lora_A.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[16,5120]}, + "base_model.model.model.language_model.layers.16.linear_attn.in_proj_b.lora_B.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[48,16]}, + "base_model.model.model.language_model.layers.16.linear_attn.in_proj_qkv.lora_A.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[16,5120]}, + "base_model.model.model.language_model.layers.16.linear_attn.in_proj_qkv.lora_B.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[10240,16]}, + "base_model.model.model.language_model.layers.16.linear_attn.in_proj_z.lora_A.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[16,5120]}, + "base_model.model.model.language_model.layers.16.linear_attn.in_proj_z.lora_B.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[6144,16]}, + "base_model.model.model.language_model.layers.16.linear_attn.out_proj.lora_A.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[16,6144]}, + "base_model.model.model.language_model.layers.16.linear_attn.out_proj.lora_B.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[5120,16]}, + "base_model.model.model.language_model.layers.16.mlp.down_proj.lora_A.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[16,17408]}, + "base_model.model.model.language_model.layers.16.mlp.down_proj.lora_B.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[5120,16]}, + "base_model.model.model.language_model.layers.16.mlp.gate_proj.lora_A.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[16,5120]}, + "base_model.model.model.language_model.layers.16.mlp.gate_proj.lora_B.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[17408,16]}, + "base_model.model.model.language_model.layers.16.mlp.up_proj.lora_A.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[16,5120]}, + "base_model.model.model.language_model.layers.16.mlp.up_proj.lora_B.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[17408,16]}, + "base_model.model.model.language_model.layers.17.linear_attn.in_proj_a.lora_A.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[16,5120]}, + "base_model.model.model.language_model.layers.17.linear_attn.in_proj_a.lora_B.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[48,16]}, + "base_model.model.model.language_model.layers.17.linear_attn.in_proj_b.lora_A.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[16,5120]}, + "base_model.model.model.language_model.layers.17.linear_attn.in_proj_b.lora_B.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[48,16]}, + "base_model.model.model.language_model.layers.17.linear_attn.in_proj_qkv.lora_A.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[16,5120]}, + "base_model.model.model.language_model.layers.17.linear_attn.in_proj_qkv.lora_B.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[10240,16]}, + "base_model.model.model.language_model.layers.17.linear_attn.in_proj_z.lora_A.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[16,5120]}, + "base_model.model.model.language_model.layers.17.linear_attn.in_proj_z.lora_B.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[6144,16]}, + "base_model.model.model.language_model.layers.17.linear_attn.out_proj.lora_A.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[16,6144]}, + "base_model.model.model.language_model.layers.17.linear_attn.out_proj.lora_B.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[5120,16]}, + "base_model.model.model.language_model.layers.17.mlp.down_proj.lora_A.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[16,17408]}, + "base_model.model.model.language_model.layers.17.mlp.down_proj.lora_B.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[5120,16]}, + "base_model.model.model.language_model.layers.17.mlp.gate_proj.lora_A.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[16,5120]}, + "base_model.model.model.language_model.layers.17.mlp.gate_proj.lora_B.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[17408,16]}, + "base_model.model.model.language_model.layers.17.mlp.up_proj.lora_A.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[16,5120]}, + "base_model.model.model.language_model.layers.17.mlp.up_proj.lora_B.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[17408,16]}, + "base_model.model.model.language_model.layers.18.linear_attn.in_proj_a.lora_A.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[16,5120]}, + "base_model.model.model.language_model.layers.18.linear_attn.in_proj_a.lora_B.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[48,16]}, + "base_model.model.model.language_model.layers.18.linear_attn.in_proj_b.lora_A.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[16,5120]}, + "base_model.model.model.language_model.layers.18.linear_attn.in_proj_b.lora_B.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[48,16]}, + "base_model.model.model.language_model.layers.18.linear_attn.in_proj_qkv.lora_A.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[16,5120]}, + "base_model.model.model.language_model.layers.18.linear_attn.in_proj_qkv.lora_B.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[10240,16]}, + "base_model.model.model.language_model.layers.18.linear_attn.in_proj_z.lora_A.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[16,5120]}, + "base_model.model.model.language_model.layers.18.linear_attn.in_proj_z.lora_B.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[6144,16]}, + "base_model.model.model.language_model.layers.18.linear_attn.out_proj.lora_A.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[16,6144]}, + "base_model.model.model.language_model.layers.18.linear_attn.out_proj.lora_B.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[5120,16]}, + "base_model.model.model.language_model.layers.18.mlp.down_proj.lora_A.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[16,17408]}, + "base_model.model.model.language_model.layers.18.mlp.down_proj.lora_B.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[5120,16]}, + "base_model.model.model.language_model.layers.18.mlp.gate_proj.lora_A.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[16,5120]}, + "base_model.model.model.language_model.layers.18.mlp.gate_proj.lora_B.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[17408,16]}, + "base_model.model.model.language_model.layers.18.mlp.up_proj.lora_A.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[16,5120]}, + "base_model.model.model.language_model.layers.18.mlp.up_proj.lora_B.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[17408,16]}, + "base_model.model.model.language_model.layers.19.mlp.down_proj.lora_A.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[16,17408]}, + "base_model.model.model.language_model.layers.19.mlp.down_proj.lora_B.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[5120,16]}, + "base_model.model.model.language_model.layers.19.mlp.gate_proj.lora_A.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[16,5120]}, + "base_model.model.model.language_model.layers.19.mlp.gate_proj.lora_B.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[17408,16]}, + "base_model.model.model.language_model.layers.19.mlp.up_proj.lora_A.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[16,5120]}, + "base_model.model.model.language_model.layers.19.mlp.up_proj.lora_B.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[17408,16]}, + "base_model.model.model.language_model.layers.19.self_attn.k_proj.lora_A.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[16,5120]}, + "base_model.model.model.language_model.layers.19.self_attn.k_proj.lora_B.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[1024,16]}, + "base_model.model.model.language_model.layers.19.self_attn.o_proj.lora_A.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[16,6144]}, + "base_model.model.model.language_model.layers.19.self_attn.o_proj.lora_B.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[5120,16]}, + "base_model.model.model.language_model.layers.19.self_attn.q_proj.lora_A.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[16,5120]}, + "base_model.model.model.language_model.layers.19.self_attn.q_proj.lora_B.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[12288,16]}, + "base_model.model.model.language_model.layers.19.self_attn.v_proj.lora_A.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[16,5120]}, + "base_model.model.model.language_model.layers.19.self_attn.v_proj.lora_B.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[1024,16]}, + "base_model.model.model.language_model.layers.2.linear_attn.in_proj_a.lora_A.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[16,5120]}, + "base_model.model.model.language_model.layers.2.linear_attn.in_proj_a.lora_B.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[48,16]}, + "base_model.model.model.language_model.layers.2.linear_attn.in_proj_b.lora_A.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[16,5120]}, + "base_model.model.model.language_model.layers.2.linear_attn.in_proj_b.lora_B.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[48,16]}, + "base_model.model.model.language_model.layers.2.linear_attn.in_proj_qkv.lora_A.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[16,5120]}, + "base_model.model.model.language_model.layers.2.linear_attn.in_proj_qkv.lora_B.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[10240,16]}, + "base_model.model.model.language_model.layers.2.linear_attn.in_proj_z.lora_A.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[16,5120]}, + "base_model.model.model.language_model.layers.2.linear_attn.in_proj_z.lora_B.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[6144,16]}, + "base_model.model.model.language_model.layers.2.linear_attn.out_proj.lora_A.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[16,6144]}, + "base_model.model.model.language_model.layers.2.linear_attn.out_proj.lora_B.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[5120,16]}, + "base_model.model.model.language_model.layers.2.mlp.down_proj.lora_A.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[16,17408]}, + "base_model.model.model.language_model.layers.2.mlp.down_proj.lora_B.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[5120,16]}, + "base_model.model.model.language_model.layers.2.mlp.gate_proj.lora_A.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[16,5120]}, + "base_model.model.model.language_model.layers.2.mlp.gate_proj.lora_B.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[17408,16]}, + "base_model.model.model.language_model.layers.2.mlp.up_proj.lora_A.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[16,5120]}, + "base_model.model.model.language_model.layers.2.mlp.up_proj.lora_B.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[17408,16]}, + "base_model.model.model.language_model.layers.20.linear_attn.in_proj_a.lora_A.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[16,5120]}, + "base_model.model.model.language_model.layers.20.linear_attn.in_proj_a.lora_B.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[48,16]}, + "base_model.model.model.language_model.layers.20.linear_attn.in_proj_b.lora_A.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[16,5120]}, + "base_model.model.model.language_model.layers.20.linear_attn.in_proj_b.lora_B.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[48,16]}, + "base_model.model.model.language_model.layers.20.linear_attn.in_proj_qkv.lora_A.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[16,5120]}, + "base_model.model.model.language_model.layers.20.linear_attn.in_proj_qkv.lora_B.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[10240,16]}, + "base_model.model.model.language_model.layers.20.linear_attn.in_proj_z.lora_A.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[16,5120]}, + "base_model.model.model.language_model.layers.20.linear_attn.in_proj_z.lora_B.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[6144,16]}, + "base_model.model.model.language_model.layers.20.linear_attn.out_proj.lora_A.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[16,6144]}, + "base_model.model.model.language_model.layers.20.linear_attn.out_proj.lora_B.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[5120,16]}, + "base_model.model.model.language_model.layers.20.mlp.down_proj.lora_A.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[16,17408]}, + "base_model.model.model.language_model.layers.20.mlp.down_proj.lora_B.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[5120,16]}, + "base_model.model.model.language_model.layers.20.mlp.gate_proj.lora_A.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[16,5120]}, + "base_model.model.model.language_model.layers.20.mlp.gate_proj.lora_B.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[17408,16]}, + "base_model.model.model.language_model.layers.20.mlp.up_proj.lora_A.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[16,5120]}, + "base_model.model.model.language_model.layers.20.mlp.up_proj.lora_B.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[17408,16]}, + "base_model.model.model.language_model.layers.21.linear_attn.in_proj_a.lora_A.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[16,5120]}, + "base_model.model.model.language_model.layers.21.linear_attn.in_proj_a.lora_B.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[48,16]}, + "base_model.model.model.language_model.layers.21.linear_attn.in_proj_b.lora_A.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[16,5120]}, + "base_model.model.model.language_model.layers.21.linear_attn.in_proj_b.lora_B.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[48,16]}, + "base_model.model.model.language_model.layers.21.linear_attn.in_proj_qkv.lora_A.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[16,5120]}, + "base_model.model.model.language_model.layers.21.linear_attn.in_proj_qkv.lora_B.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[10240,16]}, + "base_model.model.model.language_model.layers.21.linear_attn.in_proj_z.lora_A.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[16,5120]}, + "base_model.model.model.language_model.layers.21.linear_attn.in_proj_z.lora_B.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[6144,16]}, + "base_model.model.model.language_model.layers.21.linear_attn.out_proj.lora_A.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[16,6144]}, + "base_model.model.model.language_model.layers.21.linear_attn.out_proj.lora_B.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[5120,16]}, + "base_model.model.model.language_model.layers.21.mlp.down_proj.lora_A.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[16,17408]}, + "base_model.model.model.language_model.layers.21.mlp.down_proj.lora_B.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[5120,16]}, + "base_model.model.model.language_model.layers.21.mlp.gate_proj.lora_A.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[16,5120]}, + "base_model.model.model.language_model.layers.21.mlp.gate_proj.lora_B.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[17408,16]}, + "base_model.model.model.language_model.layers.21.mlp.up_proj.lora_A.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[16,5120]}, + "base_model.model.model.language_model.layers.21.mlp.up_proj.lora_B.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[17408,16]}, + "base_model.model.model.language_model.layers.22.linear_attn.in_proj_a.lora_A.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[16,5120]}, + "base_model.model.model.language_model.layers.22.linear_attn.in_proj_a.lora_B.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[48,16]}, + "base_model.model.model.language_model.layers.22.linear_attn.in_proj_b.lora_A.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[16,5120]}, + "base_model.model.model.language_model.layers.22.linear_attn.in_proj_b.lora_B.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[48,16]}, + "base_model.model.model.language_model.layers.22.linear_attn.in_proj_qkv.lora_A.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[16,5120]}, + "base_model.model.model.language_model.layers.22.linear_attn.in_proj_qkv.lora_B.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[10240,16]}, + "base_model.model.model.language_model.layers.22.linear_attn.in_proj_z.lora_A.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[16,5120]}, + "base_model.model.model.language_model.layers.22.linear_attn.in_proj_z.lora_B.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[6144,16]}, + "base_model.model.model.language_model.layers.22.linear_attn.out_proj.lora_A.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[16,6144]}, + "base_model.model.model.language_model.layers.22.linear_attn.out_proj.lora_B.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[5120,16]}, + "base_model.model.model.language_model.layers.22.mlp.down_proj.lora_A.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[16,17408]}, + "base_model.model.model.language_model.layers.22.mlp.down_proj.lora_B.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[5120,16]}, + "base_model.model.model.language_model.layers.22.mlp.gate_proj.lora_A.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[16,5120]}, + "base_model.model.model.language_model.layers.22.mlp.gate_proj.lora_B.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[17408,16]}, + "base_model.model.model.language_model.layers.22.mlp.up_proj.lora_A.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[16,5120]}, + "base_model.model.model.language_model.layers.22.mlp.up_proj.lora_B.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[17408,16]}, + "base_model.model.model.language_model.layers.23.mlp.down_proj.lora_A.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[16,17408]}, + "base_model.model.model.language_model.layers.23.mlp.down_proj.lora_B.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[5120,16]}, + "base_model.model.model.language_model.layers.23.mlp.gate_proj.lora_A.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[16,5120]}, + "base_model.model.model.language_model.layers.23.mlp.gate_proj.lora_B.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[17408,16]}, + "base_model.model.model.language_model.layers.23.mlp.up_proj.lora_A.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[16,5120]}, + "base_model.model.model.language_model.layers.23.mlp.up_proj.lora_B.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[17408,16]}, + "base_model.model.model.language_model.layers.23.self_attn.k_proj.lora_A.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[16,5120]}, + "base_model.model.model.language_model.layers.23.self_attn.k_proj.lora_B.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[1024,16]}, + "base_model.model.model.language_model.layers.23.self_attn.o_proj.lora_A.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[16,6144]}, + "base_model.model.model.language_model.layers.23.self_attn.o_proj.lora_B.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[5120,16]}, + "base_model.model.model.language_model.layers.23.self_attn.q_proj.lora_A.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[16,5120]}, + "base_model.model.model.language_model.layers.23.self_attn.q_proj.lora_B.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[12288,16]}, + "base_model.model.model.language_model.layers.23.self_attn.v_proj.lora_A.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[16,5120]}, + "base_model.model.model.language_model.layers.23.self_attn.v_proj.lora_B.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[1024,16]}, + "base_model.model.model.language_model.layers.24.linear_attn.in_proj_a.lora_A.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[16,5120]}, + "base_model.model.model.language_model.layers.24.linear_attn.in_proj_a.lora_B.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[48,16]}, + "base_model.model.model.language_model.layers.24.linear_attn.in_proj_b.lora_A.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[16,5120]}, + "base_model.model.model.language_model.layers.24.linear_attn.in_proj_b.lora_B.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[48,16]}, + "base_model.model.model.language_model.layers.24.linear_attn.in_proj_qkv.lora_A.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[16,5120]}, + "base_model.model.model.language_model.layers.24.linear_attn.in_proj_qkv.lora_B.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[10240,16]}, + "base_model.model.model.language_model.layers.24.linear_attn.in_proj_z.lora_A.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[16,5120]}, + "base_model.model.model.language_model.layers.24.linear_attn.in_proj_z.lora_B.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[6144,16]}, + "base_model.model.model.language_model.layers.24.linear_attn.out_proj.lora_A.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[16,6144]}, + "base_model.model.model.language_model.layers.24.linear_attn.out_proj.lora_B.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[5120,16]}, + "base_model.model.model.language_model.layers.24.mlp.down_proj.lora_A.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[16,17408]}, + "base_model.model.model.language_model.layers.24.mlp.down_proj.lora_B.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[5120,16]}, + "base_model.model.model.language_model.layers.24.mlp.gate_proj.lora_A.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[16,5120]}, + "base_model.model.model.language_model.layers.24.mlp.gate_proj.lora_B.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[17408,16]}, + "base_model.model.model.language_model.layers.24.mlp.up_proj.lora_A.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[16,5120]}, + "base_model.model.model.language_model.layers.24.mlp.up_proj.lora_B.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[17408,16]}, + "base_model.model.model.language_model.layers.25.linear_attn.in_proj_a.lora_A.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[16,5120]}, + "base_model.model.model.language_model.layers.25.linear_attn.in_proj_a.lora_B.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[48,16]}, + "base_model.model.model.language_model.layers.25.linear_attn.in_proj_b.lora_A.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[16,5120]}, + "base_model.model.model.language_model.layers.25.linear_attn.in_proj_b.lora_B.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[48,16]}, + "base_model.model.model.language_model.layers.25.linear_attn.in_proj_qkv.lora_A.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[16,5120]}, + "base_model.model.model.language_model.layers.25.linear_attn.in_proj_qkv.lora_B.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[10240,16]}, + "base_model.model.model.language_model.layers.25.linear_attn.in_proj_z.lora_A.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[16,5120]}, + "base_model.model.model.language_model.layers.25.linear_attn.in_proj_z.lora_B.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[6144,16]}, + "base_model.model.model.language_model.layers.25.linear_attn.out_proj.lora_A.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[16,6144]}, + "base_model.model.model.language_model.layers.25.linear_attn.out_proj.lora_B.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[5120,16]}, + "base_model.model.model.language_model.layers.25.mlp.down_proj.lora_A.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[16,17408]}, + "base_model.model.model.language_model.layers.25.mlp.down_proj.lora_B.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[5120,16]}, + "base_model.model.model.language_model.layers.25.mlp.gate_proj.lora_A.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[16,5120]}, + "base_model.model.model.language_model.layers.25.mlp.gate_proj.lora_B.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[17408,16]}, + "base_model.model.model.language_model.layers.25.mlp.up_proj.lora_A.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[16,5120]}, + "base_model.model.model.language_model.layers.25.mlp.up_proj.lora_B.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[17408,16]}, + "base_model.model.model.language_model.layers.26.linear_attn.in_proj_a.lora_A.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[16,5120]}, + "base_model.model.model.language_model.layers.26.linear_attn.in_proj_a.lora_B.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[48,16]}, + "base_model.model.model.language_model.layers.26.linear_attn.in_proj_b.lora_A.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[16,5120]}, + "base_model.model.model.language_model.layers.26.linear_attn.in_proj_b.lora_B.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[48,16]}, + "base_model.model.model.language_model.layers.26.linear_attn.in_proj_qkv.lora_A.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[16,5120]}, + "base_model.model.model.language_model.layers.26.linear_attn.in_proj_qkv.lora_B.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[10240,16]}, + "base_model.model.model.language_model.layers.26.linear_attn.in_proj_z.lora_A.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[16,5120]}, + "base_model.model.model.language_model.layers.26.linear_attn.in_proj_z.lora_B.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[6144,16]}, + "base_model.model.model.language_model.layers.26.linear_attn.out_proj.lora_A.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[16,6144]}, + "base_model.model.model.language_model.layers.26.linear_attn.out_proj.lora_B.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[5120,16]}, + "base_model.model.model.language_model.layers.26.mlp.down_proj.lora_A.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[16,17408]}, + "base_model.model.model.language_model.layers.26.mlp.down_proj.lora_B.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[5120,16]}, + "base_model.model.model.language_model.layers.26.mlp.gate_proj.lora_A.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[16,5120]}, + "base_model.model.model.language_model.layers.26.mlp.gate_proj.lora_B.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[17408,16]}, + "base_model.model.model.language_model.layers.26.mlp.up_proj.lora_A.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[16,5120]}, + "base_model.model.model.language_model.layers.26.mlp.up_proj.lora_B.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[17408,16]}, + "base_model.model.model.language_model.layers.27.mlp.down_proj.lora_A.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[16,17408]}, + "base_model.model.model.language_model.layers.27.mlp.down_proj.lora_B.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[5120,16]}, + "base_model.model.model.language_model.layers.27.mlp.gate_proj.lora_A.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[16,5120]}, + "base_model.model.model.language_model.layers.27.mlp.gate_proj.lora_B.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[17408,16]}, + "base_model.model.model.language_model.layers.27.mlp.up_proj.lora_A.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[16,5120]}, + "base_model.model.model.language_model.layers.27.mlp.up_proj.lora_B.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[17408,16]}, + "base_model.model.model.language_model.layers.27.self_attn.k_proj.lora_A.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[16,5120]}, + "base_model.model.model.language_model.layers.27.self_attn.k_proj.lora_B.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[1024,16]}, + "base_model.model.model.language_model.layers.27.self_attn.o_proj.lora_A.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[16,6144]}, + "base_model.model.model.language_model.layers.27.self_attn.o_proj.lora_B.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[5120,16]}, + "base_model.model.model.language_model.layers.27.self_attn.q_proj.lora_A.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[16,5120]}, + "base_model.model.model.language_model.layers.27.self_attn.q_proj.lora_B.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[12288,16]}, + "base_model.model.model.language_model.layers.27.self_attn.v_proj.lora_A.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[16,5120]}, + "base_model.model.model.language_model.layers.27.self_attn.v_proj.lora_B.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[1024,16]}, + "base_model.model.model.language_model.layers.28.linear_attn.in_proj_a.lora_A.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[16,5120]}, + "base_model.model.model.language_model.layers.28.linear_attn.in_proj_a.lora_B.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[48,16]}, + "base_model.model.model.language_model.layers.28.linear_attn.in_proj_b.lora_A.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[16,5120]}, + "base_model.model.model.language_model.layers.28.linear_attn.in_proj_b.lora_B.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[48,16]}, + "base_model.model.model.language_model.layers.28.linear_attn.in_proj_qkv.lora_A.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[16,5120]}, + "base_model.model.model.language_model.layers.28.linear_attn.in_proj_qkv.lora_B.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[10240,16]}, + "base_model.model.model.language_model.layers.28.linear_attn.in_proj_z.lora_A.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[16,5120]}, + "base_model.model.model.language_model.layers.28.linear_attn.in_proj_z.lora_B.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[6144,16]}, + "base_model.model.model.language_model.layers.28.linear_attn.out_proj.lora_A.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[16,6144]}, + "base_model.model.model.language_model.layers.28.linear_attn.out_proj.lora_B.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[5120,16]}, + "base_model.model.model.language_model.layers.28.mlp.down_proj.lora_A.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[16,17408]}, + "base_model.model.model.language_model.layers.28.mlp.down_proj.lora_B.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[5120,16]}, + "base_model.model.model.language_model.layers.28.mlp.gate_proj.lora_A.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[16,5120]}, + "base_model.model.model.language_model.layers.28.mlp.gate_proj.lora_B.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[17408,16]}, + "base_model.model.model.language_model.layers.28.mlp.up_proj.lora_A.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[16,5120]}, + "base_model.model.model.language_model.layers.28.mlp.up_proj.lora_B.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[17408,16]}, + "base_model.model.model.language_model.layers.29.linear_attn.in_proj_a.lora_A.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[16,5120]}, + "base_model.model.model.language_model.layers.29.linear_attn.in_proj_a.lora_B.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[48,16]}, + "base_model.model.model.language_model.layers.29.linear_attn.in_proj_b.lora_A.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[16,5120]}, + "base_model.model.model.language_model.layers.29.linear_attn.in_proj_b.lora_B.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[48,16]}, + "base_model.model.model.language_model.layers.29.linear_attn.in_proj_qkv.lora_A.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[16,5120]}, + "base_model.model.model.language_model.layers.29.linear_attn.in_proj_qkv.lora_B.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[10240,16]}, + "base_model.model.model.language_model.layers.29.linear_attn.in_proj_z.lora_A.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[16,5120]}, + "base_model.model.model.language_model.layers.29.linear_attn.in_proj_z.lora_B.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[6144,16]}, + "base_model.model.model.language_model.layers.29.linear_attn.out_proj.lora_A.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[16,6144]}, + "base_model.model.model.language_model.layers.29.linear_attn.out_proj.lora_B.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[5120,16]}, + "base_model.model.model.language_model.layers.29.mlp.down_proj.lora_A.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[16,17408]}, + "base_model.model.model.language_model.layers.29.mlp.down_proj.lora_B.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[5120,16]}, + "base_model.model.model.language_model.layers.29.mlp.gate_proj.lora_A.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[16,5120]}, + "base_model.model.model.language_model.layers.29.mlp.gate_proj.lora_B.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[17408,16]}, + "base_model.model.model.language_model.layers.29.mlp.up_proj.lora_A.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[16,5120]}, + "base_model.model.model.language_model.layers.29.mlp.up_proj.lora_B.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[17408,16]}, + "base_model.model.model.language_model.layers.3.mlp.down_proj.lora_A.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[16,17408]}, + "base_model.model.model.language_model.layers.3.mlp.down_proj.lora_B.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[5120,16]}, + "base_model.model.model.language_model.layers.3.mlp.gate_proj.lora_A.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[16,5120]}, + "base_model.model.model.language_model.layers.3.mlp.gate_proj.lora_B.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[17408,16]}, + "base_model.model.model.language_model.layers.3.mlp.up_proj.lora_A.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[16,5120]}, + "base_model.model.model.language_model.layers.3.mlp.up_proj.lora_B.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[17408,16]}, + "base_model.model.model.language_model.layers.3.self_attn.k_proj.lora_A.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[16,5120]}, + "base_model.model.model.language_model.layers.3.self_attn.k_proj.lora_B.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[1024,16]}, + "base_model.model.model.language_model.layers.3.self_attn.o_proj.lora_A.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[16,6144]}, + "base_model.model.model.language_model.layers.3.self_attn.o_proj.lora_B.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[5120,16]}, + "base_model.model.model.language_model.layers.3.self_attn.q_proj.lora_A.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[16,5120]}, + "base_model.model.model.language_model.layers.3.self_attn.q_proj.lora_B.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[12288,16]}, + "base_model.model.model.language_model.layers.3.self_attn.v_proj.lora_A.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[16,5120]}, + "base_model.model.model.language_model.layers.3.self_attn.v_proj.lora_B.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[1024,16]}, + "base_model.model.model.language_model.layers.30.linear_attn.in_proj_a.lora_A.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[16,5120]}, + "base_model.model.model.language_model.layers.30.linear_attn.in_proj_a.lora_B.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[48,16]}, + "base_model.model.model.language_model.layers.30.linear_attn.in_proj_b.lora_A.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[16,5120]}, + "base_model.model.model.language_model.layers.30.linear_attn.in_proj_b.lora_B.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[48,16]}, + "base_model.model.model.language_model.layers.30.linear_attn.in_proj_qkv.lora_A.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[16,5120]}, + "base_model.model.model.language_model.layers.30.linear_attn.in_proj_qkv.lora_B.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[10240,16]}, + "base_model.model.model.language_model.layers.30.linear_attn.in_proj_z.lora_A.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[16,5120]}, + "base_model.model.model.language_model.layers.30.linear_attn.in_proj_z.lora_B.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[6144,16]}, + "base_model.model.model.language_model.layers.30.linear_attn.out_proj.lora_A.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[16,6144]}, + "base_model.model.model.language_model.layers.30.linear_attn.out_proj.lora_B.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[5120,16]}, + "base_model.model.model.language_model.layers.30.mlp.down_proj.lora_A.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[16,17408]}, + "base_model.model.model.language_model.layers.30.mlp.down_proj.lora_B.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[5120,16]}, + "base_model.model.model.language_model.layers.30.mlp.gate_proj.lora_A.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[16,5120]}, + "base_model.model.model.language_model.layers.30.mlp.gate_proj.lora_B.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[17408,16]}, + "base_model.model.model.language_model.layers.30.mlp.up_proj.lora_A.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[16,5120]}, + "base_model.model.model.language_model.layers.30.mlp.up_proj.lora_B.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[17408,16]}, + "base_model.model.model.language_model.layers.31.mlp.down_proj.lora_A.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[16,17408]}, + "base_model.model.model.language_model.layers.31.mlp.down_proj.lora_B.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[5120,16]}, + "base_model.model.model.language_model.layers.31.mlp.gate_proj.lora_A.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[16,5120]}, + "base_model.model.model.language_model.layers.31.mlp.gate_proj.lora_B.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[17408,16]}, + "base_model.model.model.language_model.layers.31.mlp.up_proj.lora_A.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[16,5120]}, + "base_model.model.model.language_model.layers.31.mlp.up_proj.lora_B.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[17408,16]}, + "base_model.model.model.language_model.layers.31.self_attn.k_proj.lora_A.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[16,5120]}, + "base_model.model.model.language_model.layers.31.self_attn.k_proj.lora_B.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[1024,16]}, + "base_model.model.model.language_model.layers.31.self_attn.o_proj.lora_A.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[16,6144]}, + "base_model.model.model.language_model.layers.31.self_attn.o_proj.lora_B.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[5120,16]}, + "base_model.model.model.language_model.layers.31.self_attn.q_proj.lora_A.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[16,5120]}, + "base_model.model.model.language_model.layers.31.self_attn.q_proj.lora_B.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[12288,16]}, + "base_model.model.model.language_model.layers.31.self_attn.v_proj.lora_A.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[16,5120]}, + "base_model.model.model.language_model.layers.31.self_attn.v_proj.lora_B.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[1024,16]}, + "base_model.model.model.language_model.layers.32.linear_attn.in_proj_a.lora_A.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[16,5120]}, + "base_model.model.model.language_model.layers.32.linear_attn.in_proj_a.lora_B.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[48,16]}, + "base_model.model.model.language_model.layers.32.linear_attn.in_proj_b.lora_A.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[16,5120]}, + "base_model.model.model.language_model.layers.32.linear_attn.in_proj_b.lora_B.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[48,16]}, + "base_model.model.model.language_model.layers.32.linear_attn.in_proj_qkv.lora_A.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[16,5120]}, + "base_model.model.model.language_model.layers.32.linear_attn.in_proj_qkv.lora_B.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[10240,16]}, + "base_model.model.model.language_model.layers.32.linear_attn.in_proj_z.lora_A.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[16,5120]}, + "base_model.model.model.language_model.layers.32.linear_attn.in_proj_z.lora_B.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[6144,16]}, + "base_model.model.model.language_model.layers.32.linear_attn.out_proj.lora_A.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[16,6144]}, + "base_model.model.model.language_model.layers.32.linear_attn.out_proj.lora_B.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[5120,16]}, + "base_model.model.model.language_model.layers.32.mlp.down_proj.lora_A.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[16,17408]}, + "base_model.model.model.language_model.layers.32.mlp.down_proj.lora_B.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[5120,16]}, + "base_model.model.model.language_model.layers.32.mlp.gate_proj.lora_A.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[16,5120]}, + "base_model.model.model.language_model.layers.32.mlp.gate_proj.lora_B.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[17408,16]}, + "base_model.model.model.language_model.layers.32.mlp.up_proj.lora_A.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[16,5120]}, + "base_model.model.model.language_model.layers.32.mlp.up_proj.lora_B.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[17408,16]}, + "base_model.model.model.language_model.layers.33.linear_attn.in_proj_a.lora_A.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[16,5120]}, + "base_model.model.model.language_model.layers.33.linear_attn.in_proj_a.lora_B.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[48,16]}, + "base_model.model.model.language_model.layers.33.linear_attn.in_proj_b.lora_A.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[16,5120]}, + "base_model.model.model.language_model.layers.33.linear_attn.in_proj_b.lora_B.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[48,16]}, + "base_model.model.model.language_model.layers.33.linear_attn.in_proj_qkv.lora_A.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[16,5120]}, + "base_model.model.model.language_model.layers.33.linear_attn.in_proj_qkv.lora_B.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[10240,16]}, + "base_model.model.model.language_model.layers.33.linear_attn.in_proj_z.lora_A.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[16,5120]}, + "base_model.model.model.language_model.layers.33.linear_attn.in_proj_z.lora_B.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[6144,16]}, + "base_model.model.model.language_model.layers.33.linear_attn.out_proj.lora_A.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[16,6144]}, + "base_model.model.model.language_model.layers.33.linear_attn.out_proj.lora_B.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[5120,16]}, + "base_model.model.model.language_model.layers.33.mlp.down_proj.lora_A.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[16,17408]}, + "base_model.model.model.language_model.layers.33.mlp.down_proj.lora_B.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[5120,16]}, + "base_model.model.model.language_model.layers.33.mlp.gate_proj.lora_A.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[16,5120]}, + "base_model.model.model.language_model.layers.33.mlp.gate_proj.lora_B.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[17408,16]}, + "base_model.model.model.language_model.layers.33.mlp.up_proj.lora_A.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[16,5120]}, + "base_model.model.model.language_model.layers.33.mlp.up_proj.lora_B.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[17408,16]}, + "base_model.model.model.language_model.layers.34.linear_attn.in_proj_a.lora_A.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[16,5120]}, + "base_model.model.model.language_model.layers.34.linear_attn.in_proj_a.lora_B.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[48,16]}, + "base_model.model.model.language_model.layers.34.linear_attn.in_proj_b.lora_A.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[16,5120]}, + "base_model.model.model.language_model.layers.34.linear_attn.in_proj_b.lora_B.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[48,16]}, + "base_model.model.model.language_model.layers.34.linear_attn.in_proj_qkv.lora_A.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[16,5120]}, + "base_model.model.model.language_model.layers.34.linear_attn.in_proj_qkv.lora_B.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[10240,16]}, + "base_model.model.model.language_model.layers.34.linear_attn.in_proj_z.lora_A.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[16,5120]}, + "base_model.model.model.language_model.layers.34.linear_attn.in_proj_z.lora_B.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[6144,16]}, + "base_model.model.model.language_model.layers.34.linear_attn.out_proj.lora_A.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[16,6144]}, + "base_model.model.model.language_model.layers.34.linear_attn.out_proj.lora_B.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[5120,16]}, + "base_model.model.model.language_model.layers.34.mlp.down_proj.lora_A.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[16,17408]}, + "base_model.model.model.language_model.layers.34.mlp.down_proj.lora_B.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[5120,16]}, + "base_model.model.model.language_model.layers.34.mlp.gate_proj.lora_A.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[16,5120]}, + "base_model.model.model.language_model.layers.34.mlp.gate_proj.lora_B.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[17408,16]}, + "base_model.model.model.language_model.layers.34.mlp.up_proj.lora_A.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[16,5120]}, + "base_model.model.model.language_model.layers.34.mlp.up_proj.lora_B.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[17408,16]}, + "base_model.model.model.language_model.layers.35.mlp.down_proj.lora_A.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[16,17408]}, + "base_model.model.model.language_model.layers.35.mlp.down_proj.lora_B.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[5120,16]}, + "base_model.model.model.language_model.layers.35.mlp.gate_proj.lora_A.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[16,5120]}, + "base_model.model.model.language_model.layers.35.mlp.gate_proj.lora_B.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[17408,16]}, + "base_model.model.model.language_model.layers.35.mlp.up_proj.lora_A.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[16,5120]}, + "base_model.model.model.language_model.layers.35.mlp.up_proj.lora_B.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[17408,16]}, + "base_model.model.model.language_model.layers.35.self_attn.k_proj.lora_A.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[16,5120]}, + "base_model.model.model.language_model.layers.35.self_attn.k_proj.lora_B.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[1024,16]}, + "base_model.model.model.language_model.layers.35.self_attn.o_proj.lora_A.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[16,6144]}, + "base_model.model.model.language_model.layers.35.self_attn.o_proj.lora_B.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[5120,16]}, + "base_model.model.model.language_model.layers.35.self_attn.q_proj.lora_A.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[16,5120]}, + "base_model.model.model.language_model.layers.35.self_attn.q_proj.lora_B.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[12288,16]}, + "base_model.model.model.language_model.layers.35.self_attn.v_proj.lora_A.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[16,5120]}, + "base_model.model.model.language_model.layers.35.self_attn.v_proj.lora_B.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[1024,16]}, + "base_model.model.model.language_model.layers.36.linear_attn.in_proj_a.lora_A.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[16,5120]}, + "base_model.model.model.language_model.layers.36.linear_attn.in_proj_a.lora_B.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[48,16]}, + "base_model.model.model.language_model.layers.36.linear_attn.in_proj_b.lora_A.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[16,5120]}, + "base_model.model.model.language_model.layers.36.linear_attn.in_proj_b.lora_B.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[48,16]}, + "base_model.model.model.language_model.layers.36.linear_attn.in_proj_qkv.lora_A.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[16,5120]}, + "base_model.model.model.language_model.layers.36.linear_attn.in_proj_qkv.lora_B.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[10240,16]}, + "base_model.model.model.language_model.layers.36.linear_attn.in_proj_z.lora_A.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[16,5120]}, + "base_model.model.model.language_model.layers.36.linear_attn.in_proj_z.lora_B.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[6144,16]}, + "base_model.model.model.language_model.layers.36.linear_attn.out_proj.lora_A.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[16,6144]}, + "base_model.model.model.language_model.layers.36.linear_attn.out_proj.lora_B.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[5120,16]}, + "base_model.model.model.language_model.layers.36.mlp.down_proj.lora_A.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[16,17408]}, + "base_model.model.model.language_model.layers.36.mlp.down_proj.lora_B.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[5120,16]}, + "base_model.model.model.language_model.layers.36.mlp.gate_proj.lora_A.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[16,5120]}, + "base_model.model.model.language_model.layers.36.mlp.gate_proj.lora_B.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[17408,16]}, + "base_model.model.model.language_model.layers.36.mlp.up_proj.lora_A.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[16,5120]}, + "base_model.model.model.language_model.layers.36.mlp.up_proj.lora_B.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[17408,16]}, + "base_model.model.model.language_model.layers.37.linear_attn.in_proj_a.lora_A.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[16,5120]}, + "base_model.model.model.language_model.layers.37.linear_attn.in_proj_a.lora_B.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[48,16]}, + "base_model.model.model.language_model.layers.37.linear_attn.in_proj_b.lora_A.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[16,5120]}, + "base_model.model.model.language_model.layers.37.linear_attn.in_proj_b.lora_B.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[48,16]}, + "base_model.model.model.language_model.layers.37.linear_attn.in_proj_qkv.lora_A.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[16,5120]}, + "base_model.model.model.language_model.layers.37.linear_attn.in_proj_qkv.lora_B.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[10240,16]}, + "base_model.model.model.language_model.layers.37.linear_attn.in_proj_z.lora_A.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[16,5120]}, + "base_model.model.model.language_model.layers.37.linear_attn.in_proj_z.lora_B.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[6144,16]}, + "base_model.model.model.language_model.layers.37.linear_attn.out_proj.lora_A.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[16,6144]}, + "base_model.model.model.language_model.layers.37.linear_attn.out_proj.lora_B.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[5120,16]}, + "base_model.model.model.language_model.layers.37.mlp.down_proj.lora_A.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[16,17408]}, + "base_model.model.model.language_model.layers.37.mlp.down_proj.lora_B.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[5120,16]}, + "base_model.model.model.language_model.layers.37.mlp.gate_proj.lora_A.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[16,5120]}, + "base_model.model.model.language_model.layers.37.mlp.gate_proj.lora_B.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[17408,16]}, + "base_model.model.model.language_model.layers.37.mlp.up_proj.lora_A.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[16,5120]}, + "base_model.model.model.language_model.layers.37.mlp.up_proj.lora_B.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[17408,16]}, + "base_model.model.model.language_model.layers.38.linear_attn.in_proj_a.lora_A.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[16,5120]}, + "base_model.model.model.language_model.layers.38.linear_attn.in_proj_a.lora_B.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[48,16]}, + "base_model.model.model.language_model.layers.38.linear_attn.in_proj_b.lora_A.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[16,5120]}, + "base_model.model.model.language_model.layers.38.linear_attn.in_proj_b.lora_B.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[48,16]}, + "base_model.model.model.language_model.layers.38.linear_attn.in_proj_qkv.lora_A.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[16,5120]}, + "base_model.model.model.language_model.layers.38.linear_attn.in_proj_qkv.lora_B.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[10240,16]}, + "base_model.model.model.language_model.layers.38.linear_attn.in_proj_z.lora_A.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[16,5120]}, + "base_model.model.model.language_model.layers.38.linear_attn.in_proj_z.lora_B.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[6144,16]}, + "base_model.model.model.language_model.layers.38.linear_attn.out_proj.lora_A.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[16,6144]}, + "base_model.model.model.language_model.layers.38.linear_attn.out_proj.lora_B.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[5120,16]}, + "base_model.model.model.language_model.layers.38.mlp.down_proj.lora_A.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[16,17408]}, + "base_model.model.model.language_model.layers.38.mlp.down_proj.lora_B.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[5120,16]}, + "base_model.model.model.language_model.layers.38.mlp.gate_proj.lora_A.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[16,5120]}, + "base_model.model.model.language_model.layers.38.mlp.gate_proj.lora_B.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[17408,16]}, + "base_model.model.model.language_model.layers.38.mlp.up_proj.lora_A.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[16,5120]}, + "base_model.model.model.language_model.layers.38.mlp.up_proj.lora_B.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[17408,16]}, + "base_model.model.model.language_model.layers.39.mlp.down_proj.lora_A.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[16,17408]}, + "base_model.model.model.language_model.layers.39.mlp.down_proj.lora_B.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[5120,16]}, + "base_model.model.model.language_model.layers.39.mlp.gate_proj.lora_A.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[16,5120]}, + "base_model.model.model.language_model.layers.39.mlp.gate_proj.lora_B.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[17408,16]}, + "base_model.model.model.language_model.layers.39.mlp.up_proj.lora_A.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[16,5120]}, + "base_model.model.model.language_model.layers.39.mlp.up_proj.lora_B.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[17408,16]}, + "base_model.model.model.language_model.layers.39.self_attn.k_proj.lora_A.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[16,5120]}, + "base_model.model.model.language_model.layers.39.self_attn.k_proj.lora_B.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[1024,16]}, + "base_model.model.model.language_model.layers.39.self_attn.o_proj.lora_A.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[16,6144]}, + "base_model.model.model.language_model.layers.39.self_attn.o_proj.lora_B.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[5120,16]}, + "base_model.model.model.language_model.layers.39.self_attn.q_proj.lora_A.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[16,5120]}, + "base_model.model.model.language_model.layers.39.self_attn.q_proj.lora_B.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[12288,16]}, + "base_model.model.model.language_model.layers.39.self_attn.v_proj.lora_A.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[16,5120]}, + "base_model.model.model.language_model.layers.39.self_attn.v_proj.lora_B.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[1024,16]}, + "base_model.model.model.language_model.layers.4.linear_attn.in_proj_a.lora_A.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[16,5120]}, + "base_model.model.model.language_model.layers.4.linear_attn.in_proj_a.lora_B.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[48,16]}, + "base_model.model.model.language_model.layers.4.linear_attn.in_proj_b.lora_A.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[16,5120]}, + "base_model.model.model.language_model.layers.4.linear_attn.in_proj_b.lora_B.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[48,16]}, + "base_model.model.model.language_model.layers.4.linear_attn.in_proj_qkv.lora_A.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[16,5120]}, + "base_model.model.model.language_model.layers.4.linear_attn.in_proj_qkv.lora_B.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[10240,16]}, + "base_model.model.model.language_model.layers.4.linear_attn.in_proj_z.lora_A.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[16,5120]}, + "base_model.model.model.language_model.layers.4.linear_attn.in_proj_z.lora_B.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[6144,16]}, + "base_model.model.model.language_model.layers.4.linear_attn.out_proj.lora_A.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[16,6144]}, + "base_model.model.model.language_model.layers.4.linear_attn.out_proj.lora_B.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[5120,16]}, + "base_model.model.model.language_model.layers.4.mlp.down_proj.lora_A.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[16,17408]}, + "base_model.model.model.language_model.layers.4.mlp.down_proj.lora_B.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[5120,16]}, + "base_model.model.model.language_model.layers.4.mlp.gate_proj.lora_A.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[16,5120]}, + "base_model.model.model.language_model.layers.4.mlp.gate_proj.lora_B.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[17408,16]}, + "base_model.model.model.language_model.layers.4.mlp.up_proj.lora_A.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[16,5120]}, + "base_model.model.model.language_model.layers.4.mlp.up_proj.lora_B.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[17408,16]}, + "base_model.model.model.language_model.layers.40.linear_attn.in_proj_a.lora_A.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[16,5120]}, + "base_model.model.model.language_model.layers.40.linear_attn.in_proj_a.lora_B.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[48,16]}, + "base_model.model.model.language_model.layers.40.linear_attn.in_proj_b.lora_A.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[16,5120]}, + "base_model.model.model.language_model.layers.40.linear_attn.in_proj_b.lora_B.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[48,16]}, + "base_model.model.model.language_model.layers.40.linear_attn.in_proj_qkv.lora_A.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[16,5120]}, + "base_model.model.model.language_model.layers.40.linear_attn.in_proj_qkv.lora_B.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[10240,16]}, + "base_model.model.model.language_model.layers.40.linear_attn.in_proj_z.lora_A.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[16,5120]}, + "base_model.model.model.language_model.layers.40.linear_attn.in_proj_z.lora_B.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[6144,16]}, + "base_model.model.model.language_model.layers.40.linear_attn.out_proj.lora_A.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[16,6144]}, + "base_model.model.model.language_model.layers.40.linear_attn.out_proj.lora_B.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[5120,16]}, + "base_model.model.model.language_model.layers.40.mlp.down_proj.lora_A.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[16,17408]}, + "base_model.model.model.language_model.layers.40.mlp.down_proj.lora_B.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[5120,16]}, + "base_model.model.model.language_model.layers.40.mlp.gate_proj.lora_A.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[16,5120]}, + "base_model.model.model.language_model.layers.40.mlp.gate_proj.lora_B.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[17408,16]}, + "base_model.model.model.language_model.layers.40.mlp.up_proj.lora_A.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[16,5120]}, + "base_model.model.model.language_model.layers.40.mlp.up_proj.lora_B.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[17408,16]}, + "base_model.model.model.language_model.layers.41.linear_attn.in_proj_a.lora_A.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[16,5120]}, + "base_model.model.model.language_model.layers.41.linear_attn.in_proj_a.lora_B.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[48,16]}, + "base_model.model.model.language_model.layers.41.linear_attn.in_proj_b.lora_A.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[16,5120]}, + "base_model.model.model.language_model.layers.41.linear_attn.in_proj_b.lora_B.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[48,16]}, + "base_model.model.model.language_model.layers.41.linear_attn.in_proj_qkv.lora_A.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[16,5120]}, + "base_model.model.model.language_model.layers.41.linear_attn.in_proj_qkv.lora_B.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[10240,16]}, + "base_model.model.model.language_model.layers.41.linear_attn.in_proj_z.lora_A.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[16,5120]}, + "base_model.model.model.language_model.layers.41.linear_attn.in_proj_z.lora_B.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[6144,16]}, + "base_model.model.model.language_model.layers.41.linear_attn.out_proj.lora_A.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[16,6144]}, + "base_model.model.model.language_model.layers.41.linear_attn.out_proj.lora_B.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[5120,16]}, + "base_model.model.model.language_model.layers.41.mlp.down_proj.lora_A.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[16,17408]}, + "base_model.model.model.language_model.layers.41.mlp.down_proj.lora_B.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[5120,16]}, + "base_model.model.model.language_model.layers.41.mlp.gate_proj.lora_A.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[16,5120]}, + "base_model.model.model.language_model.layers.41.mlp.gate_proj.lora_B.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[17408,16]}, + "base_model.model.model.language_model.layers.41.mlp.up_proj.lora_A.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[16,5120]}, + "base_model.model.model.language_model.layers.41.mlp.up_proj.lora_B.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[17408,16]}, + "base_model.model.model.language_model.layers.42.linear_attn.in_proj_a.lora_A.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[16,5120]}, + "base_model.model.model.language_model.layers.42.linear_attn.in_proj_a.lora_B.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[48,16]}, + "base_model.model.model.language_model.layers.42.linear_attn.in_proj_b.lora_A.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[16,5120]}, + "base_model.model.model.language_model.layers.42.linear_attn.in_proj_b.lora_B.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[48,16]}, + "base_model.model.model.language_model.layers.42.linear_attn.in_proj_qkv.lora_A.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[16,5120]}, + "base_model.model.model.language_model.layers.42.linear_attn.in_proj_qkv.lora_B.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[10240,16]}, + "base_model.model.model.language_model.layers.42.linear_attn.in_proj_z.lora_A.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[16,5120]}, + "base_model.model.model.language_model.layers.42.linear_attn.in_proj_z.lora_B.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[6144,16]}, + "base_model.model.model.language_model.layers.42.linear_attn.out_proj.lora_A.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[16,6144]}, + "base_model.model.model.language_model.layers.42.linear_attn.out_proj.lora_B.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[5120,16]}, + "base_model.model.model.language_model.layers.42.mlp.down_proj.lora_A.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[16,17408]}, + "base_model.model.model.language_model.layers.42.mlp.down_proj.lora_B.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[5120,16]}, + "base_model.model.model.language_model.layers.42.mlp.gate_proj.lora_A.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[16,5120]}, + "base_model.model.model.language_model.layers.42.mlp.gate_proj.lora_B.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[17408,16]}, + "base_model.model.model.language_model.layers.42.mlp.up_proj.lora_A.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[16,5120]}, + "base_model.model.model.language_model.layers.42.mlp.up_proj.lora_B.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[17408,16]}, + "base_model.model.model.language_model.layers.43.mlp.down_proj.lora_A.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[16,17408]}, + "base_model.model.model.language_model.layers.43.mlp.down_proj.lora_B.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[5120,16]}, + "base_model.model.model.language_model.layers.43.mlp.gate_proj.lora_A.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[16,5120]}, + "base_model.model.model.language_model.layers.43.mlp.gate_proj.lora_B.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[17408,16]}, + "base_model.model.model.language_model.layers.43.mlp.up_proj.lora_A.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[16,5120]}, + "base_model.model.model.language_model.layers.43.mlp.up_proj.lora_B.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[17408,16]}, + "base_model.model.model.language_model.layers.43.self_attn.k_proj.lora_A.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[16,5120]}, + "base_model.model.model.language_model.layers.43.self_attn.k_proj.lora_B.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[1024,16]}, + "base_model.model.model.language_model.layers.43.self_attn.o_proj.lora_A.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[16,6144]}, + "base_model.model.model.language_model.layers.43.self_attn.o_proj.lora_B.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[5120,16]}, + "base_model.model.model.language_model.layers.43.self_attn.q_proj.lora_A.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[16,5120]}, + "base_model.model.model.language_model.layers.43.self_attn.q_proj.lora_B.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[12288,16]}, + "base_model.model.model.language_model.layers.43.self_attn.v_proj.lora_A.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[16,5120]}, + "base_model.model.model.language_model.layers.43.self_attn.v_proj.lora_B.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[1024,16]}, + "base_model.model.model.language_model.layers.44.linear_attn.in_proj_a.lora_A.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[16,5120]}, + "base_model.model.model.language_model.layers.44.linear_attn.in_proj_a.lora_B.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[48,16]}, + "base_model.model.model.language_model.layers.44.linear_attn.in_proj_b.lora_A.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[16,5120]}, + "base_model.model.model.language_model.layers.44.linear_attn.in_proj_b.lora_B.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[48,16]}, + "base_model.model.model.language_model.layers.44.linear_attn.in_proj_qkv.lora_A.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[16,5120]}, + "base_model.model.model.language_model.layers.44.linear_attn.in_proj_qkv.lora_B.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[10240,16]}, + "base_model.model.model.language_model.layers.44.linear_attn.in_proj_z.lora_A.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[16,5120]}, + "base_model.model.model.language_model.layers.44.linear_attn.in_proj_z.lora_B.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[6144,16]}, + "base_model.model.model.language_model.layers.44.linear_attn.out_proj.lora_A.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[16,6144]}, + "base_model.model.model.language_model.layers.44.linear_attn.out_proj.lora_B.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[5120,16]}, + "base_model.model.model.language_model.layers.44.mlp.down_proj.lora_A.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[16,17408]}, + "base_model.model.model.language_model.layers.44.mlp.down_proj.lora_B.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[5120,16]}, + "base_model.model.model.language_model.layers.44.mlp.gate_proj.lora_A.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[16,5120]}, + "base_model.model.model.language_model.layers.44.mlp.gate_proj.lora_B.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[17408,16]}, + "base_model.model.model.language_model.layers.44.mlp.up_proj.lora_A.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[16,5120]}, + "base_model.model.model.language_model.layers.44.mlp.up_proj.lora_B.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[17408,16]}, + "base_model.model.model.language_model.layers.45.linear_attn.in_proj_a.lora_A.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[16,5120]}, + "base_model.model.model.language_model.layers.45.linear_attn.in_proj_a.lora_B.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[48,16]}, + "base_model.model.model.language_model.layers.45.linear_attn.in_proj_b.lora_A.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[16,5120]}, + "base_model.model.model.language_model.layers.45.linear_attn.in_proj_b.lora_B.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[48,16]}, + "base_model.model.model.language_model.layers.45.linear_attn.in_proj_qkv.lora_A.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[16,5120]}, + "base_model.model.model.language_model.layers.45.linear_attn.in_proj_qkv.lora_B.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[10240,16]}, + "base_model.model.model.language_model.layers.45.linear_attn.in_proj_z.lora_A.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[16,5120]}, + "base_model.model.model.language_model.layers.45.linear_attn.in_proj_z.lora_B.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[6144,16]}, + "base_model.model.model.language_model.layers.45.linear_attn.out_proj.lora_A.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[16,6144]}, + "base_model.model.model.language_model.layers.45.linear_attn.out_proj.lora_B.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[5120,16]}, + "base_model.model.model.language_model.layers.45.mlp.down_proj.lora_A.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[16,17408]}, + "base_model.model.model.language_model.layers.45.mlp.down_proj.lora_B.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[5120,16]}, + "base_model.model.model.language_model.layers.45.mlp.gate_proj.lora_A.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[16,5120]}, + "base_model.model.model.language_model.layers.45.mlp.gate_proj.lora_B.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[17408,16]}, + "base_model.model.model.language_model.layers.45.mlp.up_proj.lora_A.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[16,5120]}, + "base_model.model.model.language_model.layers.45.mlp.up_proj.lora_B.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[17408,16]}, + "base_model.model.model.language_model.layers.46.linear_attn.in_proj_a.lora_A.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[16,5120]}, + "base_model.model.model.language_model.layers.46.linear_attn.in_proj_a.lora_B.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[48,16]}, + "base_model.model.model.language_model.layers.46.linear_attn.in_proj_b.lora_A.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[16,5120]}, + "base_model.model.model.language_model.layers.46.linear_attn.in_proj_b.lora_B.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[48,16]}, + "base_model.model.model.language_model.layers.46.linear_attn.in_proj_qkv.lora_A.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[16,5120]}, + "base_model.model.model.language_model.layers.46.linear_attn.in_proj_qkv.lora_B.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[10240,16]}, + "base_model.model.model.language_model.layers.46.linear_attn.in_proj_z.lora_A.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[16,5120]}, + "base_model.model.model.language_model.layers.46.linear_attn.in_proj_z.lora_B.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[6144,16]}, + "base_model.model.model.language_model.layers.46.linear_attn.out_proj.lora_A.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[16,6144]}, + "base_model.model.model.language_model.layers.46.linear_attn.out_proj.lora_B.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[5120,16]}, + "base_model.model.model.language_model.layers.46.mlp.down_proj.lora_A.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[16,17408]}, + "base_model.model.model.language_model.layers.46.mlp.down_proj.lora_B.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[5120,16]}, + "base_model.model.model.language_model.layers.46.mlp.gate_proj.lora_A.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[16,5120]}, + "base_model.model.model.language_model.layers.46.mlp.gate_proj.lora_B.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[17408,16]}, + "base_model.model.model.language_model.layers.46.mlp.up_proj.lora_A.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[16,5120]}, + "base_model.model.model.language_model.layers.46.mlp.up_proj.lora_B.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[17408,16]}, + "base_model.model.model.language_model.layers.47.mlp.down_proj.lora_A.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[16,17408]}, + "base_model.model.model.language_model.layers.47.mlp.down_proj.lora_B.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[5120,16]}, + "base_model.model.model.language_model.layers.47.mlp.gate_proj.lora_A.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[16,5120]}, + "base_model.model.model.language_model.layers.47.mlp.gate_proj.lora_B.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[17408,16]}, + "base_model.model.model.language_model.layers.47.mlp.up_proj.lora_A.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[16,5120]}, + "base_model.model.model.language_model.layers.47.mlp.up_proj.lora_B.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[17408,16]}, + "base_model.model.model.language_model.layers.47.self_attn.k_proj.lora_A.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[16,5120]}, + "base_model.model.model.language_model.layers.47.self_attn.k_proj.lora_B.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[1024,16]}, + "base_model.model.model.language_model.layers.47.self_attn.o_proj.lora_A.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[16,6144]}, + "base_model.model.model.language_model.layers.47.self_attn.o_proj.lora_B.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[5120,16]}, + "base_model.model.model.language_model.layers.47.self_attn.q_proj.lora_A.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[16,5120]}, + "base_model.model.model.language_model.layers.47.self_attn.q_proj.lora_B.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[12288,16]}, + "base_model.model.model.language_model.layers.47.self_attn.v_proj.lora_A.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[16,5120]}, + "base_model.model.model.language_model.layers.47.self_attn.v_proj.lora_B.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[1024,16]}, + "base_model.model.model.language_model.layers.48.linear_attn.in_proj_a.lora_A.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[16,5120]}, + "base_model.model.model.language_model.layers.48.linear_attn.in_proj_a.lora_B.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[48,16]}, + "base_model.model.model.language_model.layers.48.linear_attn.in_proj_b.lora_A.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[16,5120]}, + "base_model.model.model.language_model.layers.48.linear_attn.in_proj_b.lora_B.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[48,16]}, + "base_model.model.model.language_model.layers.48.linear_attn.in_proj_qkv.lora_A.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[16,5120]}, + "base_model.model.model.language_model.layers.48.linear_attn.in_proj_qkv.lora_B.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[10240,16]}, + "base_model.model.model.language_model.layers.48.linear_attn.in_proj_z.lora_A.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[16,5120]}, + "base_model.model.model.language_model.layers.48.linear_attn.in_proj_z.lora_B.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[6144,16]}, + "base_model.model.model.language_model.layers.48.linear_attn.out_proj.lora_A.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[16,6144]}, + "base_model.model.model.language_model.layers.48.linear_attn.out_proj.lora_B.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[5120,16]}, + "base_model.model.model.language_model.layers.48.mlp.down_proj.lora_A.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[16,17408]}, + "base_model.model.model.language_model.layers.48.mlp.down_proj.lora_B.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[5120,16]}, + "base_model.model.model.language_model.layers.48.mlp.gate_proj.lora_A.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[16,5120]}, + "base_model.model.model.language_model.layers.48.mlp.gate_proj.lora_B.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[17408,16]}, + "base_model.model.model.language_model.layers.48.mlp.up_proj.lora_A.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[16,5120]}, + "base_model.model.model.language_model.layers.48.mlp.up_proj.lora_B.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[17408,16]}, + "base_model.model.model.language_model.layers.49.linear_attn.in_proj_a.lora_A.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[16,5120]}, + "base_model.model.model.language_model.layers.49.linear_attn.in_proj_a.lora_B.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[48,16]}, + "base_model.model.model.language_model.layers.49.linear_attn.in_proj_b.lora_A.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[16,5120]}, + "base_model.model.model.language_model.layers.49.linear_attn.in_proj_b.lora_B.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[48,16]}, + "base_model.model.model.language_model.layers.49.linear_attn.in_proj_qkv.lora_A.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[16,5120]}, + "base_model.model.model.language_model.layers.49.linear_attn.in_proj_qkv.lora_B.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[10240,16]}, + "base_model.model.model.language_model.layers.49.linear_attn.in_proj_z.lora_A.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[16,5120]}, + "base_model.model.model.language_model.layers.49.linear_attn.in_proj_z.lora_B.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[6144,16]}, + "base_model.model.model.language_model.layers.49.linear_attn.out_proj.lora_A.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[16,6144]}, + "base_model.model.model.language_model.layers.49.linear_attn.out_proj.lora_B.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[5120,16]}, + "base_model.model.model.language_model.layers.49.mlp.down_proj.lora_A.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[16,17408]}, + "base_model.model.model.language_model.layers.49.mlp.down_proj.lora_B.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[5120,16]}, + "base_model.model.model.language_model.layers.49.mlp.gate_proj.lora_A.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[16,5120]}, + "base_model.model.model.language_model.layers.49.mlp.gate_proj.lora_B.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[17408,16]}, + "base_model.model.model.language_model.layers.49.mlp.up_proj.lora_A.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[16,5120]}, + "base_model.model.model.language_model.layers.49.mlp.up_proj.lora_B.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[17408,16]}, + "base_model.model.model.language_model.layers.5.linear_attn.in_proj_a.lora_A.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[16,5120]}, + "base_model.model.model.language_model.layers.5.linear_attn.in_proj_a.lora_B.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[48,16]}, + "base_model.model.model.language_model.layers.5.linear_attn.in_proj_b.lora_A.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[16,5120]}, + "base_model.model.model.language_model.layers.5.linear_attn.in_proj_b.lora_B.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[48,16]}, + "base_model.model.model.language_model.layers.5.linear_attn.in_proj_qkv.lora_A.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[16,5120]}, + "base_model.model.model.language_model.layers.5.linear_attn.in_proj_qkv.lora_B.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[10240,16]}, + "base_model.model.model.language_model.layers.5.linear_attn.in_proj_z.lora_A.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[16,5120]}, + "base_model.model.model.language_model.layers.5.linear_attn.in_proj_z.lora_B.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[6144,16]}, + "base_model.model.model.language_model.layers.5.linear_attn.out_proj.lora_A.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[16,6144]}, + "base_model.model.model.language_model.layers.5.linear_attn.out_proj.lora_B.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[5120,16]}, + "base_model.model.model.language_model.layers.5.mlp.down_proj.lora_A.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[16,17408]}, + "base_model.model.model.language_model.layers.5.mlp.down_proj.lora_B.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[5120,16]}, + "base_model.model.model.language_model.layers.5.mlp.gate_proj.lora_A.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[16,5120]}, + "base_model.model.model.language_model.layers.5.mlp.gate_proj.lora_B.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[17408,16]}, + "base_model.model.model.language_model.layers.5.mlp.up_proj.lora_A.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[16,5120]}, + "base_model.model.model.language_model.layers.5.mlp.up_proj.lora_B.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[17408,16]}, + "base_model.model.model.language_model.layers.50.linear_attn.in_proj_a.lora_A.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[16,5120]}, + "base_model.model.model.language_model.layers.50.linear_attn.in_proj_a.lora_B.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[48,16]}, + "base_model.model.model.language_model.layers.50.linear_attn.in_proj_b.lora_A.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[16,5120]}, + "base_model.model.model.language_model.layers.50.linear_attn.in_proj_b.lora_B.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[48,16]}, + "base_model.model.model.language_model.layers.50.linear_attn.in_proj_qkv.lora_A.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[16,5120]}, + "base_model.model.model.language_model.layers.50.linear_attn.in_proj_qkv.lora_B.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[10240,16]}, + "base_model.model.model.language_model.layers.50.linear_attn.in_proj_z.lora_A.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[16,5120]}, + "base_model.model.model.language_model.layers.50.linear_attn.in_proj_z.lora_B.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[6144,16]}, + "base_model.model.model.language_model.layers.50.linear_attn.out_proj.lora_A.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[16,6144]}, + "base_model.model.model.language_model.layers.50.linear_attn.out_proj.lora_B.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[5120,16]}, + "base_model.model.model.language_model.layers.50.mlp.down_proj.lora_A.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[16,17408]}, + "base_model.model.model.language_model.layers.50.mlp.down_proj.lora_B.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[5120,16]}, + "base_model.model.model.language_model.layers.50.mlp.gate_proj.lora_A.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[16,5120]}, + "base_model.model.model.language_model.layers.50.mlp.gate_proj.lora_B.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[17408,16]}, + "base_model.model.model.language_model.layers.50.mlp.up_proj.lora_A.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[16,5120]}, + "base_model.model.model.language_model.layers.50.mlp.up_proj.lora_B.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[17408,16]}, + "base_model.model.model.language_model.layers.51.mlp.down_proj.lora_A.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[16,17408]}, + "base_model.model.model.language_model.layers.51.mlp.down_proj.lora_B.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[5120,16]}, + "base_model.model.model.language_model.layers.51.mlp.gate_proj.lora_A.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[16,5120]}, + "base_model.model.model.language_model.layers.51.mlp.gate_proj.lora_B.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[17408,16]}, + "base_model.model.model.language_model.layers.51.mlp.up_proj.lora_A.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[16,5120]}, + "base_model.model.model.language_model.layers.51.mlp.up_proj.lora_B.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[17408,16]}, + "base_model.model.model.language_model.layers.51.self_attn.k_proj.lora_A.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[16,5120]}, + "base_model.model.model.language_model.layers.51.self_attn.k_proj.lora_B.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[1024,16]}, + "base_model.model.model.language_model.layers.51.self_attn.o_proj.lora_A.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[16,6144]}, + "base_model.model.model.language_model.layers.51.self_attn.o_proj.lora_B.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[5120,16]}, + "base_model.model.model.language_model.layers.51.self_attn.q_proj.lora_A.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[16,5120]}, + "base_model.model.model.language_model.layers.51.self_attn.q_proj.lora_B.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[12288,16]}, + "base_model.model.model.language_model.layers.51.self_attn.v_proj.lora_A.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[16,5120]}, + "base_model.model.model.language_model.layers.51.self_attn.v_proj.lora_B.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[1024,16]}, + "base_model.model.model.language_model.layers.52.linear_attn.in_proj_a.lora_A.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[16,5120]}, + "base_model.model.model.language_model.layers.52.linear_attn.in_proj_a.lora_B.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[48,16]}, + "base_model.model.model.language_model.layers.52.linear_attn.in_proj_b.lora_A.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[16,5120]}, + "base_model.model.model.language_model.layers.52.linear_attn.in_proj_b.lora_B.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[48,16]}, + "base_model.model.model.language_model.layers.52.linear_attn.in_proj_qkv.lora_A.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[16,5120]}, + "base_model.model.model.language_model.layers.52.linear_attn.in_proj_qkv.lora_B.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[10240,16]}, + "base_model.model.model.language_model.layers.52.linear_attn.in_proj_z.lora_A.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[16,5120]}, + "base_model.model.model.language_model.layers.52.linear_attn.in_proj_z.lora_B.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[6144,16]}, + "base_model.model.model.language_model.layers.52.linear_attn.out_proj.lora_A.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[16,6144]}, + "base_model.model.model.language_model.layers.52.linear_attn.out_proj.lora_B.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[5120,16]}, + "base_model.model.model.language_model.layers.52.mlp.down_proj.lora_A.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[16,17408]}, + "base_model.model.model.language_model.layers.52.mlp.down_proj.lora_B.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[5120,16]}, + "base_model.model.model.language_model.layers.52.mlp.gate_proj.lora_A.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[16,5120]}, + "base_model.model.model.language_model.layers.52.mlp.gate_proj.lora_B.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[17408,16]}, + "base_model.model.model.language_model.layers.52.mlp.up_proj.lora_A.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[16,5120]}, + "base_model.model.model.language_model.layers.52.mlp.up_proj.lora_B.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[17408,16]}, + "base_model.model.model.language_model.layers.53.linear_attn.in_proj_a.lora_A.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[16,5120]}, + "base_model.model.model.language_model.layers.53.linear_attn.in_proj_a.lora_B.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[48,16]}, + "base_model.model.model.language_model.layers.53.linear_attn.in_proj_b.lora_A.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[16,5120]}, + "base_model.model.model.language_model.layers.53.linear_attn.in_proj_b.lora_B.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[48,16]}, + "base_model.model.model.language_model.layers.53.linear_attn.in_proj_qkv.lora_A.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[16,5120]}, + "base_model.model.model.language_model.layers.53.linear_attn.in_proj_qkv.lora_B.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[10240,16]}, + "base_model.model.model.language_model.layers.53.linear_attn.in_proj_z.lora_A.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[16,5120]}, + "base_model.model.model.language_model.layers.53.linear_attn.in_proj_z.lora_B.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[6144,16]}, + "base_model.model.model.language_model.layers.53.linear_attn.out_proj.lora_A.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[16,6144]}, + "base_model.model.model.language_model.layers.53.linear_attn.out_proj.lora_B.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[5120,16]}, + "base_model.model.model.language_model.layers.53.mlp.down_proj.lora_A.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[16,17408]}, + "base_model.model.model.language_model.layers.53.mlp.down_proj.lora_B.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[5120,16]}, + "base_model.model.model.language_model.layers.53.mlp.gate_proj.lora_A.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[16,5120]}, + "base_model.model.model.language_model.layers.53.mlp.gate_proj.lora_B.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[17408,16]}, + "base_model.model.model.language_model.layers.53.mlp.up_proj.lora_A.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[16,5120]}, + "base_model.model.model.language_model.layers.53.mlp.up_proj.lora_B.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[17408,16]}, + "base_model.model.model.language_model.layers.54.linear_attn.in_proj_a.lora_A.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[16,5120]}, + "base_model.model.model.language_model.layers.54.linear_attn.in_proj_a.lora_B.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[48,16]}, + "base_model.model.model.language_model.layers.54.linear_attn.in_proj_b.lora_A.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[16,5120]}, + "base_model.model.model.language_model.layers.54.linear_attn.in_proj_b.lora_B.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[48,16]}, + "base_model.model.model.language_model.layers.54.linear_attn.in_proj_qkv.lora_A.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[16,5120]}, + "base_model.model.model.language_model.layers.54.linear_attn.in_proj_qkv.lora_B.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[10240,16]}, + "base_model.model.model.language_model.layers.54.linear_attn.in_proj_z.lora_A.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[16,5120]}, + "base_model.model.model.language_model.layers.54.linear_attn.in_proj_z.lora_B.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[6144,16]}, + "base_model.model.model.language_model.layers.54.linear_attn.out_proj.lora_A.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[16,6144]}, + "base_model.model.model.language_model.layers.54.linear_attn.out_proj.lora_B.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[5120,16]}, + "base_model.model.model.language_model.layers.54.mlp.down_proj.lora_A.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[16,17408]}, + "base_model.model.model.language_model.layers.54.mlp.down_proj.lora_B.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[5120,16]}, + "base_model.model.model.language_model.layers.54.mlp.gate_proj.lora_A.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[16,5120]}, + "base_model.model.model.language_model.layers.54.mlp.gate_proj.lora_B.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[17408,16]}, + "base_model.model.model.language_model.layers.54.mlp.up_proj.lora_A.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[16,5120]}, + "base_model.model.model.language_model.layers.54.mlp.up_proj.lora_B.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[17408,16]}, + "base_model.model.model.language_model.layers.55.mlp.down_proj.lora_A.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[16,17408]}, + "base_model.model.model.language_model.layers.55.mlp.down_proj.lora_B.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[5120,16]}, + "base_model.model.model.language_model.layers.55.mlp.gate_proj.lora_A.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[16,5120]}, + "base_model.model.model.language_model.layers.55.mlp.gate_proj.lora_B.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[17408,16]}, + "base_model.model.model.language_model.layers.55.mlp.up_proj.lora_A.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[16,5120]}, + "base_model.model.model.language_model.layers.55.mlp.up_proj.lora_B.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[17408,16]}, + "base_model.model.model.language_model.layers.55.self_attn.k_proj.lora_A.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[16,5120]}, + "base_model.model.model.language_model.layers.55.self_attn.k_proj.lora_B.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[1024,16]}, + "base_model.model.model.language_model.layers.55.self_attn.o_proj.lora_A.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[16,6144]}, + "base_model.model.model.language_model.layers.55.self_attn.o_proj.lora_B.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[5120,16]}, + "base_model.model.model.language_model.layers.55.self_attn.q_proj.lora_A.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[16,5120]}, + "base_model.model.model.language_model.layers.55.self_attn.q_proj.lora_B.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[12288,16]}, + "base_model.model.model.language_model.layers.55.self_attn.v_proj.lora_A.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[16,5120]}, + "base_model.model.model.language_model.layers.55.self_attn.v_proj.lora_B.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[1024,16]}, + "base_model.model.model.language_model.layers.56.linear_attn.in_proj_a.lora_A.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[16,5120]}, + "base_model.model.model.language_model.layers.56.linear_attn.in_proj_a.lora_B.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[48,16]}, + "base_model.model.model.language_model.layers.56.linear_attn.in_proj_b.lora_A.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[16,5120]}, + "base_model.model.model.language_model.layers.56.linear_attn.in_proj_b.lora_B.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[48,16]}, + "base_model.model.model.language_model.layers.56.linear_attn.in_proj_qkv.lora_A.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[16,5120]}, + "base_model.model.model.language_model.layers.56.linear_attn.in_proj_qkv.lora_B.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[10240,16]}, + "base_model.model.model.language_model.layers.56.linear_attn.in_proj_z.lora_A.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[16,5120]}, + "base_model.model.model.language_model.layers.56.linear_attn.in_proj_z.lora_B.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[6144,16]}, + "base_model.model.model.language_model.layers.56.linear_attn.out_proj.lora_A.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[16,6144]}, + "base_model.model.model.language_model.layers.56.linear_attn.out_proj.lora_B.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[5120,16]}, + "base_model.model.model.language_model.layers.56.mlp.down_proj.lora_A.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[16,17408]}, + "base_model.model.model.language_model.layers.56.mlp.down_proj.lora_B.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[5120,16]}, + "base_model.model.model.language_model.layers.56.mlp.gate_proj.lora_A.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[16,5120]}, + "base_model.model.model.language_model.layers.56.mlp.gate_proj.lora_B.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[17408,16]}, + "base_model.model.model.language_model.layers.56.mlp.up_proj.lora_A.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[16,5120]}, + "base_model.model.model.language_model.layers.56.mlp.up_proj.lora_B.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[17408,16]}, + "base_model.model.model.language_model.layers.57.linear_attn.in_proj_a.lora_A.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[16,5120]}, + "base_model.model.model.language_model.layers.57.linear_attn.in_proj_a.lora_B.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[48,16]}, + "base_model.model.model.language_model.layers.57.linear_attn.in_proj_b.lora_A.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[16,5120]}, + "base_model.model.model.language_model.layers.57.linear_attn.in_proj_b.lora_B.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[48,16]}, + "base_model.model.model.language_model.layers.57.linear_attn.in_proj_qkv.lora_A.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[16,5120]}, + "base_model.model.model.language_model.layers.57.linear_attn.in_proj_qkv.lora_B.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[10240,16]}, + "base_model.model.model.language_model.layers.57.linear_attn.in_proj_z.lora_A.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[16,5120]}, + "base_model.model.model.language_model.layers.57.linear_attn.in_proj_z.lora_B.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[6144,16]}, + "base_model.model.model.language_model.layers.57.linear_attn.out_proj.lora_A.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[16,6144]}, + "base_model.model.model.language_model.layers.57.linear_attn.out_proj.lora_B.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[5120,16]}, + "base_model.model.model.language_model.layers.57.mlp.down_proj.lora_A.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[16,17408]}, + "base_model.model.model.language_model.layers.57.mlp.down_proj.lora_B.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[5120,16]}, + "base_model.model.model.language_model.layers.57.mlp.gate_proj.lora_A.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[16,5120]}, + "base_model.model.model.language_model.layers.57.mlp.gate_proj.lora_B.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[17408,16]}, + "base_model.model.model.language_model.layers.57.mlp.up_proj.lora_A.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[16,5120]}, + "base_model.model.model.language_model.layers.57.mlp.up_proj.lora_B.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[17408,16]}, + "base_model.model.model.language_model.layers.58.linear_attn.in_proj_a.lora_A.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[16,5120]}, + "base_model.model.model.language_model.layers.58.linear_attn.in_proj_a.lora_B.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[48,16]}, + "base_model.model.model.language_model.layers.58.linear_attn.in_proj_b.lora_A.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[16,5120]}, + "base_model.model.model.language_model.layers.58.linear_attn.in_proj_b.lora_B.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[48,16]}, + "base_model.model.model.language_model.layers.58.linear_attn.in_proj_qkv.lora_A.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[16,5120]}, + "base_model.model.model.language_model.layers.58.linear_attn.in_proj_qkv.lora_B.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[10240,16]}, + "base_model.model.model.language_model.layers.58.linear_attn.in_proj_z.lora_A.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[16,5120]}, + "base_model.model.model.language_model.layers.58.linear_attn.in_proj_z.lora_B.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[6144,16]}, + "base_model.model.model.language_model.layers.58.linear_attn.out_proj.lora_A.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[16,6144]}, + "base_model.model.model.language_model.layers.58.linear_attn.out_proj.lora_B.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[5120,16]}, + "base_model.model.model.language_model.layers.58.mlp.down_proj.lora_A.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[16,17408]}, + "base_model.model.model.language_model.layers.58.mlp.down_proj.lora_B.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[5120,16]}, + "base_model.model.model.language_model.layers.58.mlp.gate_proj.lora_A.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[16,5120]}, + "base_model.model.model.language_model.layers.58.mlp.gate_proj.lora_B.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[17408,16]}, + "base_model.model.model.language_model.layers.58.mlp.up_proj.lora_A.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[16,5120]}, + "base_model.model.model.language_model.layers.58.mlp.up_proj.lora_B.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[17408,16]}, + "base_model.model.model.language_model.layers.59.mlp.down_proj.lora_A.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[16,17408]}, + "base_model.model.model.language_model.layers.59.mlp.down_proj.lora_B.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[5120,16]}, + "base_model.model.model.language_model.layers.59.mlp.gate_proj.lora_A.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[16,5120]}, + "base_model.model.model.language_model.layers.59.mlp.gate_proj.lora_B.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[17408,16]}, + "base_model.model.model.language_model.layers.59.mlp.up_proj.lora_A.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[16,5120]}, + "base_model.model.model.language_model.layers.59.mlp.up_proj.lora_B.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[17408,16]}, + "base_model.model.model.language_model.layers.59.self_attn.k_proj.lora_A.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[16,5120]}, + "base_model.model.model.language_model.layers.59.self_attn.k_proj.lora_B.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[1024,16]}, + "base_model.model.model.language_model.layers.59.self_attn.o_proj.lora_A.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[16,6144]}, + "base_model.model.model.language_model.layers.59.self_attn.o_proj.lora_B.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[5120,16]}, + "base_model.model.model.language_model.layers.59.self_attn.q_proj.lora_A.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[16,5120]}, + "base_model.model.model.language_model.layers.59.self_attn.q_proj.lora_B.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[12288,16]}, + "base_model.model.model.language_model.layers.59.self_attn.v_proj.lora_A.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[16,5120]}, + "base_model.model.model.language_model.layers.59.self_attn.v_proj.lora_B.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[1024,16]}, + "base_model.model.model.language_model.layers.6.linear_attn.in_proj_a.lora_A.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[16,5120]}, + "base_model.model.model.language_model.layers.6.linear_attn.in_proj_a.lora_B.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[48,16]}, + "base_model.model.model.language_model.layers.6.linear_attn.in_proj_b.lora_A.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[16,5120]}, + "base_model.model.model.language_model.layers.6.linear_attn.in_proj_b.lora_B.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[48,16]}, + "base_model.model.model.language_model.layers.6.linear_attn.in_proj_qkv.lora_A.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[16,5120]}, + "base_model.model.model.language_model.layers.6.linear_attn.in_proj_qkv.lora_B.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[10240,16]}, + "base_model.model.model.language_model.layers.6.linear_attn.in_proj_z.lora_A.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[16,5120]}, + "base_model.model.model.language_model.layers.6.linear_attn.in_proj_z.lora_B.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[6144,16]}, + "base_model.model.model.language_model.layers.6.linear_attn.out_proj.lora_A.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[16,6144]}, + "base_model.model.model.language_model.layers.6.linear_attn.out_proj.lora_B.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[5120,16]}, + "base_model.model.model.language_model.layers.6.mlp.down_proj.lora_A.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[16,17408]}, + "base_model.model.model.language_model.layers.6.mlp.down_proj.lora_B.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[5120,16]}, + "base_model.model.model.language_model.layers.6.mlp.gate_proj.lora_A.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[16,5120]}, + "base_model.model.model.language_model.layers.6.mlp.gate_proj.lora_B.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[17408,16]}, + "base_model.model.model.language_model.layers.6.mlp.up_proj.lora_A.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[16,5120]}, + "base_model.model.model.language_model.layers.6.mlp.up_proj.lora_B.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[17408,16]}, + "base_model.model.model.language_model.layers.60.linear_attn.in_proj_a.lora_A.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[16,5120]}, + "base_model.model.model.language_model.layers.60.linear_attn.in_proj_a.lora_B.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[48,16]}, + "base_model.model.model.language_model.layers.60.linear_attn.in_proj_b.lora_A.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[16,5120]}, + "base_model.model.model.language_model.layers.60.linear_attn.in_proj_b.lora_B.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[48,16]}, + "base_model.model.model.language_model.layers.60.linear_attn.in_proj_qkv.lora_A.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[16,5120]}, + "base_model.model.model.language_model.layers.60.linear_attn.in_proj_qkv.lora_B.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[10240,16]}, + "base_model.model.model.language_model.layers.60.linear_attn.in_proj_z.lora_A.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[16,5120]}, + "base_model.model.model.language_model.layers.60.linear_attn.in_proj_z.lora_B.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[6144,16]}, + "base_model.model.model.language_model.layers.60.linear_attn.out_proj.lora_A.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[16,6144]}, + "base_model.model.model.language_model.layers.60.linear_attn.out_proj.lora_B.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[5120,16]}, + "base_model.model.model.language_model.layers.60.mlp.down_proj.lora_A.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[16,17408]}, + "base_model.model.model.language_model.layers.60.mlp.down_proj.lora_B.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[5120,16]}, + "base_model.model.model.language_model.layers.60.mlp.gate_proj.lora_A.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[16,5120]}, + "base_model.model.model.language_model.layers.60.mlp.gate_proj.lora_B.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[17408,16]}, + "base_model.model.model.language_model.layers.60.mlp.up_proj.lora_A.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[16,5120]}, + "base_model.model.model.language_model.layers.60.mlp.up_proj.lora_B.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[17408,16]}, + "base_model.model.model.language_model.layers.61.linear_attn.in_proj_a.lora_A.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[16,5120]}, + "base_model.model.model.language_model.layers.61.linear_attn.in_proj_a.lora_B.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[48,16]}, + "base_model.model.model.language_model.layers.61.linear_attn.in_proj_b.lora_A.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[16,5120]}, + "base_model.model.model.language_model.layers.61.linear_attn.in_proj_b.lora_B.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[48,16]}, + "base_model.model.model.language_model.layers.61.linear_attn.in_proj_qkv.lora_A.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[16,5120]}, + "base_model.model.model.language_model.layers.61.linear_attn.in_proj_qkv.lora_B.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[10240,16]}, + "base_model.model.model.language_model.layers.61.linear_attn.in_proj_z.lora_A.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[16,5120]}, + "base_model.model.model.language_model.layers.61.linear_attn.in_proj_z.lora_B.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[6144,16]}, + "base_model.model.model.language_model.layers.61.linear_attn.out_proj.lora_A.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[16,6144]}, + "base_model.model.model.language_model.layers.61.linear_attn.out_proj.lora_B.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[5120,16]}, + "base_model.model.model.language_model.layers.61.mlp.down_proj.lora_A.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[16,17408]}, + "base_model.model.model.language_model.layers.61.mlp.down_proj.lora_B.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[5120,16]}, + "base_model.model.model.language_model.layers.61.mlp.gate_proj.lora_A.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[16,5120]}, + "base_model.model.model.language_model.layers.61.mlp.gate_proj.lora_B.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[17408,16]}, + "base_model.model.model.language_model.layers.61.mlp.up_proj.lora_A.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[16,5120]}, + "base_model.model.model.language_model.layers.61.mlp.up_proj.lora_B.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[17408,16]}, + "base_model.model.model.language_model.layers.62.linear_attn.in_proj_a.lora_A.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[16,5120]}, + "base_model.model.model.language_model.layers.62.linear_attn.in_proj_a.lora_B.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[48,16]}, + "base_model.model.model.language_model.layers.62.linear_attn.in_proj_b.lora_A.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[16,5120]}, + "base_model.model.model.language_model.layers.62.linear_attn.in_proj_b.lora_B.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[48,16]}, + "base_model.model.model.language_model.layers.62.linear_attn.in_proj_qkv.lora_A.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[16,5120]}, + "base_model.model.model.language_model.layers.62.linear_attn.in_proj_qkv.lora_B.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[10240,16]}, + "base_model.model.model.language_model.layers.62.linear_attn.in_proj_z.lora_A.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[16,5120]}, + "base_model.model.model.language_model.layers.62.linear_attn.in_proj_z.lora_B.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[6144,16]}, + "base_model.model.model.language_model.layers.62.linear_attn.out_proj.lora_A.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[16,6144]}, + "base_model.model.model.language_model.layers.62.linear_attn.out_proj.lora_B.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[5120,16]}, + "base_model.model.model.language_model.layers.62.mlp.down_proj.lora_A.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[16,17408]}, + "base_model.model.model.language_model.layers.62.mlp.down_proj.lora_B.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[5120,16]}, + "base_model.model.model.language_model.layers.62.mlp.gate_proj.lora_A.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[16,5120]}, + "base_model.model.model.language_model.layers.62.mlp.gate_proj.lora_B.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[17408,16]}, + "base_model.model.model.language_model.layers.62.mlp.up_proj.lora_A.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[16,5120]}, + "base_model.model.model.language_model.layers.62.mlp.up_proj.lora_B.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[17408,16]}, + "base_model.model.model.language_model.layers.63.mlp.down_proj.lora_A.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[16,17408]}, + "base_model.model.model.language_model.layers.63.mlp.down_proj.lora_B.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[5120,16]}, + "base_model.model.model.language_model.layers.63.mlp.gate_proj.lora_A.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[16,5120]}, + "base_model.model.model.language_model.layers.63.mlp.gate_proj.lora_B.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[17408,16]}, + "base_model.model.model.language_model.layers.63.mlp.up_proj.lora_A.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[16,5120]}, + "base_model.model.model.language_model.layers.63.mlp.up_proj.lora_B.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[17408,16]}, + "base_model.model.model.language_model.layers.63.self_attn.k_proj.lora_A.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[16,5120]}, + "base_model.model.model.language_model.layers.63.self_attn.k_proj.lora_B.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[1024,16]}, + "base_model.model.model.language_model.layers.63.self_attn.o_proj.lora_A.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[16,6144]}, + "base_model.model.model.language_model.layers.63.self_attn.o_proj.lora_B.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[5120,16]}, + "base_model.model.model.language_model.layers.63.self_attn.q_proj.lora_A.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[16,5120]}, + "base_model.model.model.language_model.layers.63.self_attn.q_proj.lora_B.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[12288,16]}, + "base_model.model.model.language_model.layers.63.self_attn.v_proj.lora_A.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[16,5120]}, + "base_model.model.model.language_model.layers.63.self_attn.v_proj.lora_B.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[1024,16]}, + "base_model.model.model.language_model.layers.7.mlp.down_proj.lora_A.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[16,17408]}, + "base_model.model.model.language_model.layers.7.mlp.down_proj.lora_B.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[5120,16]}, + "base_model.model.model.language_model.layers.7.mlp.gate_proj.lora_A.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[16,5120]}, + "base_model.model.model.language_model.layers.7.mlp.gate_proj.lora_B.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[17408,16]}, + "base_model.model.model.language_model.layers.7.mlp.up_proj.lora_A.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[16,5120]}, + "base_model.model.model.language_model.layers.7.mlp.up_proj.lora_B.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[17408,16]}, + "base_model.model.model.language_model.layers.7.self_attn.k_proj.lora_A.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[16,5120]}, + "base_model.model.model.language_model.layers.7.self_attn.k_proj.lora_B.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[1024,16]}, + "base_model.model.model.language_model.layers.7.self_attn.o_proj.lora_A.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[16,6144]}, + "base_model.model.model.language_model.layers.7.self_attn.o_proj.lora_B.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[5120,16]}, + "base_model.model.model.language_model.layers.7.self_attn.q_proj.lora_A.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[16,5120]}, + "base_model.model.model.language_model.layers.7.self_attn.q_proj.lora_B.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[12288,16]}, + "base_model.model.model.language_model.layers.7.self_attn.v_proj.lora_A.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[16,5120]}, + "base_model.model.model.language_model.layers.7.self_attn.v_proj.lora_B.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[1024,16]}, + "base_model.model.model.language_model.layers.8.linear_attn.in_proj_a.lora_A.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[16,5120]}, + "base_model.model.model.language_model.layers.8.linear_attn.in_proj_a.lora_B.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[48,16]}, + "base_model.model.model.language_model.layers.8.linear_attn.in_proj_b.lora_A.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[16,5120]}, + "base_model.model.model.language_model.layers.8.linear_attn.in_proj_b.lora_B.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[48,16]}, + "base_model.model.model.language_model.layers.8.linear_attn.in_proj_qkv.lora_A.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[16,5120]}, + "base_model.model.model.language_model.layers.8.linear_attn.in_proj_qkv.lora_B.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[10240,16]}, + "base_model.model.model.language_model.layers.8.linear_attn.in_proj_z.lora_A.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[16,5120]}, + "base_model.model.model.language_model.layers.8.linear_attn.in_proj_z.lora_B.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[6144,16]}, + "base_model.model.model.language_model.layers.8.linear_attn.out_proj.lora_A.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[16,6144]}, + "base_model.model.model.language_model.layers.8.linear_attn.out_proj.lora_B.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[5120,16]}, + "base_model.model.model.language_model.layers.8.mlp.down_proj.lora_A.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[16,17408]}, + "base_model.model.model.language_model.layers.8.mlp.down_proj.lora_B.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[5120,16]}, + "base_model.model.model.language_model.layers.8.mlp.gate_proj.lora_A.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[16,5120]}, + "base_model.model.model.language_model.layers.8.mlp.gate_proj.lora_B.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[17408,16]}, + "base_model.model.model.language_model.layers.8.mlp.up_proj.lora_A.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[16,5120]}, + "base_model.model.model.language_model.layers.8.mlp.up_proj.lora_B.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[17408,16]}, + "base_model.model.model.language_model.layers.9.linear_attn.in_proj_a.lora_A.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[16,5120]}, + "base_model.model.model.language_model.layers.9.linear_attn.in_proj_a.lora_B.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[48,16]}, + "base_model.model.model.language_model.layers.9.linear_attn.in_proj_b.lora_A.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[16,5120]}, + "base_model.model.model.language_model.layers.9.linear_attn.in_proj_b.lora_B.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[48,16]}, + "base_model.model.model.language_model.layers.9.linear_attn.in_proj_qkv.lora_A.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[16,5120]}, + "base_model.model.model.language_model.layers.9.linear_attn.in_proj_qkv.lora_B.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[10240,16]}, + "base_model.model.model.language_model.layers.9.linear_attn.in_proj_z.lora_A.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[16,5120]}, + "base_model.model.model.language_model.layers.9.linear_attn.in_proj_z.lora_B.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[6144,16]}, + "base_model.model.model.language_model.layers.9.linear_attn.out_proj.lora_A.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[16,6144]}, + "base_model.model.model.language_model.layers.9.linear_attn.out_proj.lora_B.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[5120,16]}, + "base_model.model.model.language_model.layers.9.mlp.down_proj.lora_A.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[16,17408]}, + "base_model.model.model.language_model.layers.9.mlp.down_proj.lora_B.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[5120,16]}, + "base_model.model.model.language_model.layers.9.mlp.gate_proj.lora_A.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[16,5120]}, + "base_model.model.model.language_model.layers.9.mlp.gate_proj.lora_B.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[17408,16]}, + "base_model.model.model.language_model.layers.9.mlp.up_proj.lora_A.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[16,5120]}, + "base_model.model.model.language_model.layers.9.mlp.up_proj.lora_B.weight": {"dtype":"F32","file":"adapter_model.safetensors","shape":[17408,16]} + } + }, + "base": { + "files": { + "chat_template.jinja": "c3cf9e34abf4f9e36c2d72165aa9c132d3e2a725b6c2586aaa3a8af9d7a81041", + "config.json": "191e0af232104ed8b65258cf3fb2b842e288008baca7633c11b82a1ac7203aab", + "generation_config.json": "e70c136c1b78ddc1fb0905bac8e733a4dc448d4f852a5dd75143fffc70be550e", + "merges.txt": "a9d356d7bdf1ef4949e3e748e95b8e10ad9d4e2e838eddc38a0a7b6b94d1db8d", + "model-00001-of-00018.safetensors": "ba0ce20aae489ad196733da5064bcdf159a1fe84f53336648196e1ebb7751b1c", + "model-00002-of-00018.safetensors": "06a148c01bfbe3faa14a5f184a7ff29a706f7ae1c8b2705d2058e26d17a001fb", + "model-00003-of-00018.safetensors": "2e1bf62cbcd406eaa64b60d10353e1f0ef4039d0976e56f05cabe953454f9968", + "model-00004-of-00018.safetensors": "511e34063187882659753c4d93f3859f93c019fd438d8813071921c81d9a3f1a", + "model-00005-of-00018.safetensors": "635cb53446dc74f219740fc59e18b774f877b803b9722e289ca62575a6efa701", + "model-00006-of-00018.safetensors": "0bc5214fac607f0e6cc92eec3789d4b8559410ef9fce66621ba8158e8410dae0", + "model-00007-of-00018.safetensors": "80b0c49033e9a0d5762562aa12f4acdb7f54da586f3d0110f28c48d91cf07892", + "model-00008-of-00018.safetensors": "7192c5b66185d3592927daabee1cc19e6f6e0ce75988ee20e824b624765fda79", + "model-00009-of-00018.safetensors": "af3c48cc37af44f3db6ae0579baf019180d48d9c527caa0a1f03ff85813a56d8", + "model-00010-of-00018.safetensors": "163490a76f3bea3a40855b7efc04ce6d27afaf1a34f0bbde495b9491f76457c9", + "model-00011-of-00018.safetensors": "5f3ae1b948aeee39da77aec558e8236cd65fe4d7cb7686a76bb007acc563c6d8", + "model-00012-of-00018.safetensors": "a3de1c7114677a8f5ac5c4892c90e8238ea5c1e2038c80e757dfc87c3902ca55", + "model-00013-of-00018.safetensors": "06ab79a41f74c9c5cb734816feb0c7fc364104b227165ee7391231e1155aa02a", + "model-00014-of-00018.safetensors": "4138ed94603065ba884bbcadedb04d7718bb40117e85e6f5c6fc5b9c05b7a85b", + "model-00015-of-00018.safetensors": "69224e27b9de4e7dbf6fc936c6eaae08447bda3b80a6c31a871ab451173afd22", + "model-00016-of-00018.safetensors": "73cb9a1089fb6155cb648609478d6633be8a5c7d9ca5a05bc8925ce8a553cefe", + "model-00017-of-00018.safetensors": "beb51f01056142ac4984bd800507b0dd0fd18de57f8e9ef6ea41d1a3598983a8", + "model-00018-of-00018.safetensors": "1d3479509e21494658f9b64d317f5ea8e55c4025d28c702d6c4d0b356ce8ea06", + "model.safetensors.index.json": "77042094076611b69791a610065f28b7013b8c621795fa86ddccc8bac7d1b9df", + "preprocessor_config.json": "27225450ac9c6529872ee1924fcb0962ff5634834f817040f444118116f4e516", + "tokenizer.json": "0997f410c57a1f4e53b09e4be8f4a172d90edd9564368fb0847030937229b9f3", + "tokenizer_config.json": "b11349aafa7cdc6a320767cf7ceb29ed82f7eda5d65e8e0819e76f0ce947bf27", + "video_preprocessor_config.json": "7768af27c1fafa9cc9011c1dc20067e03f8915e03b63504550e11d5066986d13", + "vocab.json": "ce99b4cb2983d118806ce0a8b777a35b093e2000a503ebde25853284c9dfa003" + }, + "tensors": { + "lm_head.weight": {"dtype":"BF16","file":"model-00018-of-00018.safetensors","shape":[248320,5120]}, + "model.language_model.embed_tokens.weight": {"dtype":"BF16","file":"model-00003-of-00018.safetensors","shape":[248320,5120]}, + "model.language_model.layers.0.input_layernorm.weight": {"dtype":"BF16","file":"model-00001-of-00018.safetensors","shape":[5120]}, + "model.language_model.layers.0.linear_attn.A_log": {"dtype":"BF16","file":"model-00001-of-00018.safetensors","shape":[48]}, + "model.language_model.layers.0.linear_attn.conv1d.weight": {"dtype":"BF16","file":"model-00001-of-00018.safetensors","shape":[10240,1,4]}, + "model.language_model.layers.0.linear_attn.dt_bias": {"dtype":"BF16","file":"model-00001-of-00018.safetensors","shape":[48]}, + "model.language_model.layers.0.linear_attn.in_proj_a.weight": {"dtype":"BF16","file":"model-00001-of-00018.safetensors","shape":[48,5120]}, + "model.language_model.layers.0.linear_attn.in_proj_b.weight": {"dtype":"BF16","file":"model-00001-of-00018.safetensors","shape":[48,5120]}, + "model.language_model.layers.0.linear_attn.in_proj_qkv.weight": {"dtype":"BF16","file":"model-00001-of-00018.safetensors","shape":[10240,5120]}, + "model.language_model.layers.0.linear_attn.in_proj_z.weight": {"dtype":"BF16","file":"model-00001-of-00018.safetensors","shape":[6144,5120]}, + "model.language_model.layers.0.linear_attn.norm.weight": {"dtype":"BF16","file":"model-00001-of-00018.safetensors","shape":[128]}, + "model.language_model.layers.0.linear_attn.out_proj.weight": {"dtype":"BF16","file":"model-00001-of-00018.safetensors","shape":[5120,6144]}, + "model.language_model.layers.0.mlp.down_proj.weight": {"dtype":"BF16","file":"model-00001-of-00018.safetensors","shape":[5120,17408]}, + "model.language_model.layers.0.mlp.gate_proj.weight": {"dtype":"BF16","file":"model-00001-of-00018.safetensors","shape":[17408,5120]}, + "model.language_model.layers.0.mlp.up_proj.weight": {"dtype":"BF16","file":"model-00001-of-00018.safetensors","shape":[17408,5120]}, + "model.language_model.layers.0.post_attention_layernorm.weight": {"dtype":"BF16","file":"model-00001-of-00018.safetensors","shape":[5120]}, + "model.language_model.layers.1.input_layernorm.weight": {"dtype":"BF16","file":"model-00001-of-00018.safetensors","shape":[5120]}, + "model.language_model.layers.1.linear_attn.A_log": {"dtype":"BF16","file":"model-00001-of-00018.safetensors","shape":[48]}, + "model.language_model.layers.1.linear_attn.conv1d.weight": {"dtype":"BF16","file":"model-00001-of-00018.safetensors","shape":[10240,1,4]}, + "model.language_model.layers.1.linear_attn.dt_bias": {"dtype":"BF16","file":"model-00001-of-00018.safetensors","shape":[48]}, + "model.language_model.layers.1.linear_attn.in_proj_a.weight": {"dtype":"BF16","file":"model-00001-of-00018.safetensors","shape":[48,5120]}, + "model.language_model.layers.1.linear_attn.in_proj_b.weight": {"dtype":"BF16","file":"model-00001-of-00018.safetensors","shape":[48,5120]}, + "model.language_model.layers.1.linear_attn.in_proj_qkv.weight": {"dtype":"BF16","file":"model-00001-of-00018.safetensors","shape":[10240,5120]}, + "model.language_model.layers.1.linear_attn.in_proj_z.weight": {"dtype":"BF16","file":"model-00001-of-00018.safetensors","shape":[6144,5120]}, + "model.language_model.layers.1.linear_attn.norm.weight": {"dtype":"BF16","file":"model-00001-of-00018.safetensors","shape":[128]}, + "model.language_model.layers.1.linear_attn.out_proj.weight": {"dtype":"BF16","file":"model-00001-of-00018.safetensors","shape":[5120,6144]}, + "model.language_model.layers.1.mlp.down_proj.weight": {"dtype":"BF16","file":"model-00001-of-00018.safetensors","shape":[5120,17408]}, + "model.language_model.layers.1.mlp.gate_proj.weight": {"dtype":"BF16","file":"model-00001-of-00018.safetensors","shape":[17408,5120]}, + "model.language_model.layers.1.mlp.up_proj.weight": {"dtype":"BF16","file":"model-00001-of-00018.safetensors","shape":[17408,5120]}, + "model.language_model.layers.1.post_attention_layernorm.weight": {"dtype":"BF16","file":"model-00001-of-00018.safetensors","shape":[5120]}, + "model.language_model.layers.10.input_layernorm.weight": {"dtype":"BF16","file":"model-00004-of-00018.safetensors","shape":[5120]}, + "model.language_model.layers.10.linear_attn.A_log": {"dtype":"BF16","file":"model-00004-of-00018.safetensors","shape":[48]}, + "model.language_model.layers.10.linear_attn.conv1d.weight": {"dtype":"BF16","file":"model-00004-of-00018.safetensors","shape":[10240,1,4]}, + "model.language_model.layers.10.linear_attn.dt_bias": {"dtype":"BF16","file":"model-00004-of-00018.safetensors","shape":[48]}, + "model.language_model.layers.10.linear_attn.in_proj_a.weight": {"dtype":"BF16","file":"model-00004-of-00018.safetensors","shape":[48,5120]}, + "model.language_model.layers.10.linear_attn.in_proj_b.weight": {"dtype":"BF16","file":"model-00004-of-00018.safetensors","shape":[48,5120]}, + "model.language_model.layers.10.linear_attn.in_proj_qkv.weight": {"dtype":"BF16","file":"model-00004-of-00018.safetensors","shape":[10240,5120]}, + "model.language_model.layers.10.linear_attn.in_proj_z.weight": {"dtype":"BF16","file":"model-00004-of-00018.safetensors","shape":[6144,5120]}, + "model.language_model.layers.10.linear_attn.norm.weight": {"dtype":"BF16","file":"model-00004-of-00018.safetensors","shape":[128]}, + "model.language_model.layers.10.linear_attn.out_proj.weight": {"dtype":"BF16","file":"model-00004-of-00018.safetensors","shape":[5120,6144]}, + "model.language_model.layers.10.mlp.down_proj.weight": {"dtype":"BF16","file":"model-00004-of-00018.safetensors","shape":[5120,17408]}, + "model.language_model.layers.10.mlp.gate_proj.weight": {"dtype":"BF16","file":"model-00004-of-00018.safetensors","shape":[17408,5120]}, + "model.language_model.layers.10.mlp.up_proj.weight": {"dtype":"BF16","file":"model-00004-of-00018.safetensors","shape":[17408,5120]}, + "model.language_model.layers.10.post_attention_layernorm.weight": {"dtype":"BF16","file":"model-00004-of-00018.safetensors","shape":[5120]}, + "model.language_model.layers.11.input_layernorm.weight": {"dtype":"BF16","file":"model-00004-of-00018.safetensors","shape":[5120]}, + "model.language_model.layers.11.mlp.down_proj.weight": {"dtype":"BF16","file":"model-00004-of-00018.safetensors","shape":[5120,17408]}, + "model.language_model.layers.11.mlp.gate_proj.weight": {"dtype":"BF16","file":"model-00004-of-00018.safetensors","shape":[17408,5120]}, + "model.language_model.layers.11.mlp.up_proj.weight": {"dtype":"BF16","file":"model-00004-of-00018.safetensors","shape":[17408,5120]}, + "model.language_model.layers.11.post_attention_layernorm.weight": {"dtype":"BF16","file":"model-00004-of-00018.safetensors","shape":[5120]}, + "model.language_model.layers.11.self_attn.k_norm.weight": {"dtype":"BF16","file":"model-00004-of-00018.safetensors","shape":[256]}, + "model.language_model.layers.11.self_attn.k_proj.weight": {"dtype":"BF16","file":"model-00004-of-00018.safetensors","shape":[1024,5120]}, + "model.language_model.layers.11.self_attn.o_proj.weight": {"dtype":"BF16","file":"model-00004-of-00018.safetensors","shape":[5120,6144]}, + "model.language_model.layers.11.self_attn.q_norm.weight": {"dtype":"BF16","file":"model-00004-of-00018.safetensors","shape":[256]}, + "model.language_model.layers.11.self_attn.q_proj.weight": {"dtype":"BF16","file":"model-00004-of-00018.safetensors","shape":[12288,5120]}, + "model.language_model.layers.11.self_attn.v_proj.weight": {"dtype":"BF16","file":"model-00004-of-00018.safetensors","shape":[1024,5120]}, + "model.language_model.layers.12.input_layernorm.weight": {"dtype":"BF16","file":"model-00004-of-00018.safetensors","shape":[5120]}, + "model.language_model.layers.12.linear_attn.A_log": {"dtype":"BF16","file":"model-00004-of-00018.safetensors","shape":[48]}, + "model.language_model.layers.12.linear_attn.conv1d.weight": {"dtype":"BF16","file":"model-00004-of-00018.safetensors","shape":[10240,1,4]}, + "model.language_model.layers.12.linear_attn.dt_bias": {"dtype":"BF16","file":"model-00004-of-00018.safetensors","shape":[48]}, + "model.language_model.layers.12.linear_attn.in_proj_a.weight": {"dtype":"BF16","file":"model-00004-of-00018.safetensors","shape":[48,5120]}, + "model.language_model.layers.12.linear_attn.in_proj_b.weight": {"dtype":"BF16","file":"model-00004-of-00018.safetensors","shape":[48,5120]}, + "model.language_model.layers.12.linear_attn.in_proj_qkv.weight": {"dtype":"BF16","file":"model-00004-of-00018.safetensors","shape":[10240,5120]}, + "model.language_model.layers.12.linear_attn.in_proj_z.weight": {"dtype":"BF16","file":"model-00004-of-00018.safetensors","shape":[6144,5120]}, + "model.language_model.layers.12.linear_attn.norm.weight": {"dtype":"BF16","file":"model-00004-of-00018.safetensors","shape":[128]}, + "model.language_model.layers.12.linear_attn.out_proj.weight": {"dtype":"BF16","file":"model-00004-of-00018.safetensors","shape":[5120,6144]}, + "model.language_model.layers.12.mlp.down_proj.weight": {"dtype":"BF16","file":"model-00004-of-00018.safetensors","shape":[5120,17408]}, + "model.language_model.layers.12.mlp.gate_proj.weight": {"dtype":"BF16","file":"model-00004-of-00018.safetensors","shape":[17408,5120]}, + "model.language_model.layers.12.mlp.up_proj.weight": {"dtype":"BF16","file":"model-00004-of-00018.safetensors","shape":[17408,5120]}, + "model.language_model.layers.12.post_attention_layernorm.weight": {"dtype":"BF16","file":"model-00004-of-00018.safetensors","shape":[5120]}, + "model.language_model.layers.13.input_layernorm.weight": {"dtype":"BF16","file":"model-00004-of-00018.safetensors","shape":[5120]}, + "model.language_model.layers.13.linear_attn.A_log": {"dtype":"BF16","file":"model-00004-of-00018.safetensors","shape":[48]}, + "model.language_model.layers.13.linear_attn.conv1d.weight": {"dtype":"BF16","file":"model-00004-of-00018.safetensors","shape":[10240,1,4]}, + "model.language_model.layers.13.linear_attn.dt_bias": {"dtype":"BF16","file":"model-00004-of-00018.safetensors","shape":[48]}, + "model.language_model.layers.13.linear_attn.in_proj_a.weight": {"dtype":"BF16","file":"model-00004-of-00018.safetensors","shape":[48,5120]}, + "model.language_model.layers.13.linear_attn.in_proj_b.weight": {"dtype":"BF16","file":"model-00004-of-00018.safetensors","shape":[48,5120]}, + "model.language_model.layers.13.linear_attn.in_proj_qkv.weight": {"dtype":"BF16","file":"model-00004-of-00018.safetensors","shape":[10240,5120]}, + "model.language_model.layers.13.linear_attn.in_proj_z.weight": {"dtype":"BF16","file":"model-00004-of-00018.safetensors","shape":[6144,5120]}, + "model.language_model.layers.13.linear_attn.norm.weight": {"dtype":"BF16","file":"model-00004-of-00018.safetensors","shape":[128]}, + "model.language_model.layers.13.linear_attn.out_proj.weight": {"dtype":"BF16","file":"model-00004-of-00018.safetensors","shape":[5120,6144]}, + "model.language_model.layers.13.mlp.down_proj.weight": {"dtype":"BF16","file":"model-00004-of-00018.safetensors","shape":[5120,17408]}, + "model.language_model.layers.13.mlp.gate_proj.weight": {"dtype":"BF16","file":"model-00004-of-00018.safetensors","shape":[17408,5120]}, + "model.language_model.layers.13.mlp.up_proj.weight": {"dtype":"BF16","file":"model-00004-of-00018.safetensors","shape":[17408,5120]}, + "model.language_model.layers.13.post_attention_layernorm.weight": {"dtype":"BF16","file":"model-00004-of-00018.safetensors","shape":[5120]}, + "model.language_model.layers.14.input_layernorm.weight": {"dtype":"BF16","file":"model-00004-of-00018.safetensors","shape":[5120]}, + "model.language_model.layers.14.linear_attn.A_log": {"dtype":"BF16","file":"model-00004-of-00018.safetensors","shape":[48]}, + "model.language_model.layers.14.linear_attn.conv1d.weight": {"dtype":"BF16","file":"model-00004-of-00018.safetensors","shape":[10240,1,4]}, + "model.language_model.layers.14.linear_attn.dt_bias": {"dtype":"BF16","file":"model-00004-of-00018.safetensors","shape":[48]}, + "model.language_model.layers.14.linear_attn.in_proj_a.weight": {"dtype":"BF16","file":"model-00004-of-00018.safetensors","shape":[48,5120]}, + "model.language_model.layers.14.linear_attn.in_proj_b.weight": {"dtype":"BF16","file":"model-00004-of-00018.safetensors","shape":[48,5120]}, + "model.language_model.layers.14.linear_attn.in_proj_qkv.weight": {"dtype":"BF16","file":"model-00004-of-00018.safetensors","shape":[10240,5120]}, + "model.language_model.layers.14.linear_attn.in_proj_z.weight": {"dtype":"BF16","file":"model-00004-of-00018.safetensors","shape":[6144,5120]}, + "model.language_model.layers.14.linear_attn.norm.weight": {"dtype":"BF16","file":"model-00004-of-00018.safetensors","shape":[128]}, + "model.language_model.layers.14.linear_attn.out_proj.weight": {"dtype":"BF16","file":"model-00004-of-00018.safetensors","shape":[5120,6144]}, + "model.language_model.layers.14.mlp.down_proj.weight": {"dtype":"BF16","file":"model-00004-of-00018.safetensors","shape":[5120,17408]}, + "model.language_model.layers.14.mlp.gate_proj.weight": {"dtype":"BF16","file":"model-00004-of-00018.safetensors","shape":[17408,5120]}, + "model.language_model.layers.14.mlp.up_proj.weight": {"dtype":"BF16","file":"model-00004-of-00018.safetensors","shape":[17408,5120]}, + "model.language_model.layers.14.post_attention_layernorm.weight": {"dtype":"BF16","file":"model-00004-of-00018.safetensors","shape":[5120]}, + "model.language_model.layers.15.input_layernorm.weight": {"dtype":"BF16","file":"model-00005-of-00018.safetensors","shape":[5120]}, + "model.language_model.layers.15.mlp.down_proj.weight": {"dtype":"BF16","file":"model-00005-of-00018.safetensors","shape":[5120,17408]}, + "model.language_model.layers.15.mlp.gate_proj.weight": {"dtype":"BF16","file":"model-00004-of-00018.safetensors","shape":[17408,5120]}, + "model.language_model.layers.15.mlp.up_proj.weight": {"dtype":"BF16","file":"model-00005-of-00018.safetensors","shape":[17408,5120]}, + "model.language_model.layers.15.post_attention_layernorm.weight": {"dtype":"BF16","file":"model-00004-of-00018.safetensors","shape":[5120]}, + "model.language_model.layers.15.self_attn.k_norm.weight": {"dtype":"BF16","file":"model-00005-of-00018.safetensors","shape":[256]}, + "model.language_model.layers.15.self_attn.k_proj.weight": {"dtype":"BF16","file":"model-00005-of-00018.safetensors","shape":[1024,5120]}, + "model.language_model.layers.15.self_attn.o_proj.weight": {"dtype":"BF16","file":"model-00005-of-00018.safetensors","shape":[5120,6144]}, + "model.language_model.layers.15.self_attn.q_norm.weight": {"dtype":"BF16","file":"model-00005-of-00018.safetensors","shape":[256]}, + "model.language_model.layers.15.self_attn.q_proj.weight": {"dtype":"BF16","file":"model-00005-of-00018.safetensors","shape":[12288,5120]}, + "model.language_model.layers.15.self_attn.v_proj.weight": {"dtype":"BF16","file":"model-00005-of-00018.safetensors","shape":[1024,5120]}, + "model.language_model.layers.16.input_layernorm.weight": {"dtype":"BF16","file":"model-00006-of-00018.safetensors","shape":[5120]}, + "model.language_model.layers.16.linear_attn.A_log": {"dtype":"BF16","file":"model-00006-of-00018.safetensors","shape":[48]}, + "model.language_model.layers.16.linear_attn.conv1d.weight": {"dtype":"BF16","file":"model-00006-of-00018.safetensors","shape":[10240,1,4]}, + "model.language_model.layers.16.linear_attn.dt_bias": {"dtype":"BF16","file":"model-00006-of-00018.safetensors","shape":[48]}, + "model.language_model.layers.16.linear_attn.in_proj_a.weight": {"dtype":"BF16","file":"model-00006-of-00018.safetensors","shape":[48,5120]}, + "model.language_model.layers.16.linear_attn.in_proj_b.weight": {"dtype":"BF16","file":"model-00006-of-00018.safetensors","shape":[48,5120]}, + "model.language_model.layers.16.linear_attn.in_proj_qkv.weight": {"dtype":"BF16","file":"model-00006-of-00018.safetensors","shape":[10240,5120]}, + "model.language_model.layers.16.linear_attn.in_proj_z.weight": {"dtype":"BF16","file":"model-00006-of-00018.safetensors","shape":[6144,5120]}, + "model.language_model.layers.16.linear_attn.norm.weight": {"dtype":"BF16","file":"model-00006-of-00018.safetensors","shape":[128]}, + "model.language_model.layers.16.linear_attn.out_proj.weight": {"dtype":"BF16","file":"model-00006-of-00018.safetensors","shape":[5120,6144]}, + "model.language_model.layers.16.mlp.down_proj.weight": {"dtype":"BF16","file":"model-00006-of-00018.safetensors","shape":[5120,17408]}, + "model.language_model.layers.16.mlp.gate_proj.weight": {"dtype":"BF16","file":"model-00006-of-00018.safetensors","shape":[17408,5120]}, + "model.language_model.layers.16.mlp.up_proj.weight": {"dtype":"BF16","file":"model-00006-of-00018.safetensors","shape":[17408,5120]}, + "model.language_model.layers.16.post_attention_layernorm.weight": {"dtype":"BF16","file":"model-00006-of-00018.safetensors","shape":[5120]}, + "model.language_model.layers.17.input_layernorm.weight": {"dtype":"BF16","file":"model-00006-of-00018.safetensors","shape":[5120]}, + "model.language_model.layers.17.linear_attn.A_log": {"dtype":"BF16","file":"model-00006-of-00018.safetensors","shape":[48]}, + "model.language_model.layers.17.linear_attn.conv1d.weight": {"dtype":"BF16","file":"model-00006-of-00018.safetensors","shape":[10240,1,4]}, + "model.language_model.layers.17.linear_attn.dt_bias": {"dtype":"BF16","file":"model-00006-of-00018.safetensors","shape":[48]}, + "model.language_model.layers.17.linear_attn.in_proj_a.weight": {"dtype":"BF16","file":"model-00006-of-00018.safetensors","shape":[48,5120]}, + "model.language_model.layers.17.linear_attn.in_proj_b.weight": {"dtype":"BF16","file":"model-00006-of-00018.safetensors","shape":[48,5120]}, + "model.language_model.layers.17.linear_attn.in_proj_qkv.weight": {"dtype":"BF16","file":"model-00006-of-00018.safetensors","shape":[10240,5120]}, + "model.language_model.layers.17.linear_attn.in_proj_z.weight": {"dtype":"BF16","file":"model-00006-of-00018.safetensors","shape":[6144,5120]}, + "model.language_model.layers.17.linear_attn.norm.weight": {"dtype":"BF16","file":"model-00006-of-00018.safetensors","shape":[128]}, + "model.language_model.layers.17.linear_attn.out_proj.weight": {"dtype":"BF16","file":"model-00006-of-00018.safetensors","shape":[5120,6144]}, + "model.language_model.layers.17.mlp.down_proj.weight": {"dtype":"BF16","file":"model-00006-of-00018.safetensors","shape":[5120,17408]}, + "model.language_model.layers.17.mlp.gate_proj.weight": {"dtype":"BF16","file":"model-00006-of-00018.safetensors","shape":[17408,5120]}, + "model.language_model.layers.17.mlp.up_proj.weight": {"dtype":"BF16","file":"model-00006-of-00018.safetensors","shape":[17408,5120]}, + "model.language_model.layers.17.post_attention_layernorm.weight": {"dtype":"BF16","file":"model-00006-of-00018.safetensors","shape":[5120]}, + "model.language_model.layers.18.input_layernorm.weight": {"dtype":"BF16","file":"model-00006-of-00018.safetensors","shape":[5120]}, + "model.language_model.layers.18.linear_attn.A_log": {"dtype":"BF16","file":"model-00006-of-00018.safetensors","shape":[48]}, + "model.language_model.layers.18.linear_attn.conv1d.weight": {"dtype":"BF16","file":"model-00006-of-00018.safetensors","shape":[10240,1,4]}, + "model.language_model.layers.18.linear_attn.dt_bias": {"dtype":"BF16","file":"model-00006-of-00018.safetensors","shape":[48]}, + "model.language_model.layers.18.linear_attn.in_proj_a.weight": {"dtype":"BF16","file":"model-00006-of-00018.safetensors","shape":[48,5120]}, + "model.language_model.layers.18.linear_attn.in_proj_b.weight": {"dtype":"BF16","file":"model-00006-of-00018.safetensors","shape":[48,5120]}, + "model.language_model.layers.18.linear_attn.in_proj_qkv.weight": {"dtype":"BF16","file":"model-00006-of-00018.safetensors","shape":[10240,5120]}, + "model.language_model.layers.18.linear_attn.in_proj_z.weight": {"dtype":"BF16","file":"model-00006-of-00018.safetensors","shape":[6144,5120]}, + "model.language_model.layers.18.linear_attn.norm.weight": {"dtype":"BF16","file":"model-00006-of-00018.safetensors","shape":[128]}, + "model.language_model.layers.18.linear_attn.out_proj.weight": {"dtype":"BF16","file":"model-00006-of-00018.safetensors","shape":[5120,6144]}, + "model.language_model.layers.18.mlp.down_proj.weight": {"dtype":"BF16","file":"model-00006-of-00018.safetensors","shape":[5120,17408]}, + "model.language_model.layers.18.mlp.gate_proj.weight": {"dtype":"BF16","file":"model-00006-of-00018.safetensors","shape":[17408,5120]}, + "model.language_model.layers.18.mlp.up_proj.weight": {"dtype":"BF16","file":"model-00006-of-00018.safetensors","shape":[17408,5120]}, + "model.language_model.layers.18.post_attention_layernorm.weight": {"dtype":"BF16","file":"model-00006-of-00018.safetensors","shape":[5120]}, + "model.language_model.layers.19.input_layernorm.weight": {"dtype":"BF16","file":"model-00006-of-00018.safetensors","shape":[5120]}, + "model.language_model.layers.19.mlp.down_proj.weight": {"dtype":"BF16","file":"model-00006-of-00018.safetensors","shape":[5120,17408]}, + "model.language_model.layers.19.mlp.gate_proj.weight": {"dtype":"BF16","file":"model-00006-of-00018.safetensors","shape":[17408,5120]}, + "model.language_model.layers.19.mlp.up_proj.weight": {"dtype":"BF16","file":"model-00006-of-00018.safetensors","shape":[17408,5120]}, + "model.language_model.layers.19.post_attention_layernorm.weight": {"dtype":"BF16","file":"model-00006-of-00018.safetensors","shape":[5120]}, + "model.language_model.layers.19.self_attn.k_norm.weight": {"dtype":"BF16","file":"model-00006-of-00018.safetensors","shape":[256]}, + "model.language_model.layers.19.self_attn.k_proj.weight": {"dtype":"BF16","file":"model-00006-of-00018.safetensors","shape":[1024,5120]}, + "model.language_model.layers.19.self_attn.o_proj.weight": {"dtype":"BF16","file":"model-00006-of-00018.safetensors","shape":[5120,6144]}, + "model.language_model.layers.19.self_attn.q_norm.weight": {"dtype":"BF16","file":"model-00006-of-00018.safetensors","shape":[256]}, + "model.language_model.layers.19.self_attn.q_proj.weight": {"dtype":"BF16","file":"model-00006-of-00018.safetensors","shape":[12288,5120]}, + "model.language_model.layers.19.self_attn.v_proj.weight": {"dtype":"BF16","file":"model-00006-of-00018.safetensors","shape":[1024,5120]}, + "model.language_model.layers.2.input_layernorm.weight": {"dtype":"BF16","file":"model-00001-of-00018.safetensors","shape":[5120]}, + "model.language_model.layers.2.linear_attn.A_log": {"dtype":"BF16","file":"model-00001-of-00018.safetensors","shape":[48]}, + "model.language_model.layers.2.linear_attn.conv1d.weight": {"dtype":"BF16","file":"model-00001-of-00018.safetensors","shape":[10240,1,4]}, + "model.language_model.layers.2.linear_attn.dt_bias": {"dtype":"BF16","file":"model-00001-of-00018.safetensors","shape":[48]}, + "model.language_model.layers.2.linear_attn.in_proj_a.weight": {"dtype":"BF16","file":"model-00001-of-00018.safetensors","shape":[48,5120]}, + "model.language_model.layers.2.linear_attn.in_proj_b.weight": {"dtype":"BF16","file":"model-00001-of-00018.safetensors","shape":[48,5120]}, + "model.language_model.layers.2.linear_attn.in_proj_qkv.weight": {"dtype":"BF16","file":"model-00001-of-00018.safetensors","shape":[10240,5120]}, + "model.language_model.layers.2.linear_attn.in_proj_z.weight": {"dtype":"BF16","file":"model-00001-of-00018.safetensors","shape":[6144,5120]}, + "model.language_model.layers.2.linear_attn.norm.weight": {"dtype":"BF16","file":"model-00001-of-00018.safetensors","shape":[128]}, + "model.language_model.layers.2.linear_attn.out_proj.weight": {"dtype":"BF16","file":"model-00001-of-00018.safetensors","shape":[5120,6144]}, + "model.language_model.layers.2.mlp.down_proj.weight": {"dtype":"BF16","file":"model-00001-of-00018.safetensors","shape":[5120,17408]}, + "model.language_model.layers.2.mlp.gate_proj.weight": {"dtype":"BF16","file":"model-00001-of-00018.safetensors","shape":[17408,5120]}, + "model.language_model.layers.2.mlp.up_proj.weight": {"dtype":"BF16","file":"model-00001-of-00018.safetensors","shape":[17408,5120]}, + "model.language_model.layers.2.post_attention_layernorm.weight": {"dtype":"BF16","file":"model-00001-of-00018.safetensors","shape":[5120]}, + "model.language_model.layers.20.input_layernorm.weight": {"dtype":"BF16","file":"model-00006-of-00018.safetensors","shape":[5120]}, + "model.language_model.layers.20.linear_attn.A_log": {"dtype":"BF16","file":"model-00006-of-00018.safetensors","shape":[48]}, + "model.language_model.layers.20.linear_attn.conv1d.weight": {"dtype":"BF16","file":"model-00006-of-00018.safetensors","shape":[10240,1,4]}, + "model.language_model.layers.20.linear_attn.dt_bias": {"dtype":"BF16","file":"model-00006-of-00018.safetensors","shape":[48]}, + "model.language_model.layers.20.linear_attn.in_proj_a.weight": {"dtype":"BF16","file":"model-00006-of-00018.safetensors","shape":[48,5120]}, + "model.language_model.layers.20.linear_attn.in_proj_b.weight": {"dtype":"BF16","file":"model-00006-of-00018.safetensors","shape":[48,5120]}, + "model.language_model.layers.20.linear_attn.in_proj_qkv.weight": {"dtype":"BF16","file":"model-00006-of-00018.safetensors","shape":[10240,5120]}, + "model.language_model.layers.20.linear_attn.in_proj_z.weight": {"dtype":"BF16","file":"model-00006-of-00018.safetensors","shape":[6144,5120]}, + "model.language_model.layers.20.linear_attn.norm.weight": {"dtype":"BF16","file":"model-00006-of-00018.safetensors","shape":[128]}, + "model.language_model.layers.20.linear_attn.out_proj.weight": {"dtype":"BF16","file":"model-00006-of-00018.safetensors","shape":[5120,6144]}, + "model.language_model.layers.20.mlp.down_proj.weight": {"dtype":"BF16","file":"model-00006-of-00018.safetensors","shape":[5120,17408]}, + "model.language_model.layers.20.mlp.gate_proj.weight": {"dtype":"BF16","file":"model-00006-of-00018.safetensors","shape":[17408,5120]}, + "model.language_model.layers.20.mlp.up_proj.weight": {"dtype":"BF16","file":"model-00006-of-00018.safetensors","shape":[17408,5120]}, + "model.language_model.layers.20.post_attention_layernorm.weight": {"dtype":"BF16","file":"model-00006-of-00018.safetensors","shape":[5120]}, + "model.language_model.layers.21.input_layernorm.weight": {"dtype":"BF16","file":"model-00006-of-00018.safetensors","shape":[5120]}, + "model.language_model.layers.21.linear_attn.A_log": {"dtype":"BF16","file":"model-00006-of-00018.safetensors","shape":[48]}, + "model.language_model.layers.21.linear_attn.conv1d.weight": {"dtype":"BF16","file":"model-00006-of-00018.safetensors","shape":[10240,1,4]}, + "model.language_model.layers.21.linear_attn.dt_bias": {"dtype":"BF16","file":"model-00006-of-00018.safetensors","shape":[48]}, + "model.language_model.layers.21.linear_attn.in_proj_a.weight": {"dtype":"BF16","file":"model-00006-of-00018.safetensors","shape":[48,5120]}, + "model.language_model.layers.21.linear_attn.in_proj_b.weight": {"dtype":"BF16","file":"model-00006-of-00018.safetensors","shape":[48,5120]}, + "model.language_model.layers.21.linear_attn.in_proj_qkv.weight": {"dtype":"BF16","file":"model-00006-of-00018.safetensors","shape":[10240,5120]}, + "model.language_model.layers.21.linear_attn.in_proj_z.weight": {"dtype":"BF16","file":"model-00006-of-00018.safetensors","shape":[6144,5120]}, + "model.language_model.layers.21.linear_attn.norm.weight": {"dtype":"BF16","file":"model-00006-of-00018.safetensors","shape":[128]}, + "model.language_model.layers.21.linear_attn.out_proj.weight": {"dtype":"BF16","file":"model-00007-of-00018.safetensors","shape":[5120,6144]}, + "model.language_model.layers.21.mlp.down_proj.weight": {"dtype":"BF16","file":"model-00007-of-00018.safetensors","shape":[5120,17408]}, + "model.language_model.layers.21.mlp.gate_proj.weight": {"dtype":"BF16","file":"model-00007-of-00018.safetensors","shape":[17408,5120]}, + "model.language_model.layers.21.mlp.up_proj.weight": {"dtype":"BF16","file":"model-00007-of-00018.safetensors","shape":[17408,5120]}, + "model.language_model.layers.21.post_attention_layernorm.weight": {"dtype":"BF16","file":"model-00007-of-00018.safetensors","shape":[5120]}, + "model.language_model.layers.22.input_layernorm.weight": {"dtype":"BF16","file":"model-00007-of-00018.safetensors","shape":[5120]}, + "model.language_model.layers.22.linear_attn.A_log": {"dtype":"BF16","file":"model-00007-of-00018.safetensors","shape":[48]}, + "model.language_model.layers.22.linear_attn.conv1d.weight": {"dtype":"BF16","file":"model-00007-of-00018.safetensors","shape":[10240,1,4]}, + "model.language_model.layers.22.linear_attn.dt_bias": {"dtype":"BF16","file":"model-00007-of-00018.safetensors","shape":[48]}, + "model.language_model.layers.22.linear_attn.in_proj_a.weight": {"dtype":"BF16","file":"model-00007-of-00018.safetensors","shape":[48,5120]}, + "model.language_model.layers.22.linear_attn.in_proj_b.weight": {"dtype":"BF16","file":"model-00007-of-00018.safetensors","shape":[48,5120]}, + "model.language_model.layers.22.linear_attn.in_proj_qkv.weight": {"dtype":"BF16","file":"model-00007-of-00018.safetensors","shape":[10240,5120]}, + "model.language_model.layers.22.linear_attn.in_proj_z.weight": {"dtype":"BF16","file":"model-00007-of-00018.safetensors","shape":[6144,5120]}, + "model.language_model.layers.22.linear_attn.norm.weight": {"dtype":"BF16","file":"model-00007-of-00018.safetensors","shape":[128]}, + "model.language_model.layers.22.linear_attn.out_proj.weight": {"dtype":"BF16","file":"model-00007-of-00018.safetensors","shape":[5120,6144]}, + "model.language_model.layers.22.mlp.down_proj.weight": {"dtype":"BF16","file":"model-00007-of-00018.safetensors","shape":[5120,17408]}, + "model.language_model.layers.22.mlp.gate_proj.weight": {"dtype":"BF16","file":"model-00007-of-00018.safetensors","shape":[17408,5120]}, + "model.language_model.layers.22.mlp.up_proj.weight": {"dtype":"BF16","file":"model-00007-of-00018.safetensors","shape":[17408,5120]}, + "model.language_model.layers.22.post_attention_layernorm.weight": {"dtype":"BF16","file":"model-00007-of-00018.safetensors","shape":[5120]}, + "model.language_model.layers.23.input_layernorm.weight": {"dtype":"BF16","file":"model-00007-of-00018.safetensors","shape":[5120]}, + "model.language_model.layers.23.mlp.down_proj.weight": {"dtype":"BF16","file":"model-00007-of-00018.safetensors","shape":[5120,17408]}, + "model.language_model.layers.23.mlp.gate_proj.weight": {"dtype":"BF16","file":"model-00007-of-00018.safetensors","shape":[17408,5120]}, + "model.language_model.layers.23.mlp.up_proj.weight": {"dtype":"BF16","file":"model-00007-of-00018.safetensors","shape":[17408,5120]}, + "model.language_model.layers.23.post_attention_layernorm.weight": {"dtype":"BF16","file":"model-00007-of-00018.safetensors","shape":[5120]}, + "model.language_model.layers.23.self_attn.k_norm.weight": {"dtype":"BF16","file":"model-00007-of-00018.safetensors","shape":[256]}, + "model.language_model.layers.23.self_attn.k_proj.weight": {"dtype":"BF16","file":"model-00007-of-00018.safetensors","shape":[1024,5120]}, + "model.language_model.layers.23.self_attn.o_proj.weight": {"dtype":"BF16","file":"model-00007-of-00018.safetensors","shape":[5120,6144]}, + "model.language_model.layers.23.self_attn.q_norm.weight": {"dtype":"BF16","file":"model-00007-of-00018.safetensors","shape":[256]}, + "model.language_model.layers.23.self_attn.q_proj.weight": {"dtype":"BF16","file":"model-00007-of-00018.safetensors","shape":[12288,5120]}, + "model.language_model.layers.23.self_attn.v_proj.weight": {"dtype":"BF16","file":"model-00007-of-00018.safetensors","shape":[1024,5120]}, + "model.language_model.layers.24.input_layernorm.weight": {"dtype":"BF16","file":"model-00008-of-00018.safetensors","shape":[5120]}, + "model.language_model.layers.24.linear_attn.A_log": {"dtype":"BF16","file":"model-00008-of-00018.safetensors","shape":[48]}, + "model.language_model.layers.24.linear_attn.conv1d.weight": {"dtype":"BF16","file":"model-00008-of-00018.safetensors","shape":[10240,1,4]}, + "model.language_model.layers.24.linear_attn.dt_bias": {"dtype":"BF16","file":"model-00008-of-00018.safetensors","shape":[48]}, + "model.language_model.layers.24.linear_attn.in_proj_a.weight": {"dtype":"BF16","file":"model-00008-of-00018.safetensors","shape":[48,5120]}, + "model.language_model.layers.24.linear_attn.in_proj_b.weight": {"dtype":"BF16","file":"model-00008-of-00018.safetensors","shape":[48,5120]}, + "model.language_model.layers.24.linear_attn.in_proj_qkv.weight": {"dtype":"BF16","file":"model-00008-of-00018.safetensors","shape":[10240,5120]}, + "model.language_model.layers.24.linear_attn.in_proj_z.weight": {"dtype":"BF16","file":"model-00008-of-00018.safetensors","shape":[6144,5120]}, + "model.language_model.layers.24.linear_attn.norm.weight": {"dtype":"BF16","file":"model-00008-of-00018.safetensors","shape":[128]}, + "model.language_model.layers.24.linear_attn.out_proj.weight": {"dtype":"BF16","file":"model-00008-of-00018.safetensors","shape":[5120,6144]}, + "model.language_model.layers.24.mlp.down_proj.weight": {"dtype":"BF16","file":"model-00008-of-00018.safetensors","shape":[5120,17408]}, + "model.language_model.layers.24.mlp.gate_proj.weight": {"dtype":"BF16","file":"model-00008-of-00018.safetensors","shape":[17408,5120]}, + "model.language_model.layers.24.mlp.up_proj.weight": {"dtype":"BF16","file":"model-00008-of-00018.safetensors","shape":[17408,5120]}, + "model.language_model.layers.24.post_attention_layernorm.weight": {"dtype":"BF16","file":"model-00008-of-00018.safetensors","shape":[5120]}, + "model.language_model.layers.25.input_layernorm.weight": {"dtype":"BF16","file":"model-00008-of-00018.safetensors","shape":[5120]}, + "model.language_model.layers.25.linear_attn.A_log": {"dtype":"BF16","file":"model-00008-of-00018.safetensors","shape":[48]}, + "model.language_model.layers.25.linear_attn.conv1d.weight": {"dtype":"BF16","file":"model-00008-of-00018.safetensors","shape":[10240,1,4]}, + "model.language_model.layers.25.linear_attn.dt_bias": {"dtype":"BF16","file":"model-00008-of-00018.safetensors","shape":[48]}, + "model.language_model.layers.25.linear_attn.in_proj_a.weight": {"dtype":"BF16","file":"model-00008-of-00018.safetensors","shape":[48,5120]}, + "model.language_model.layers.25.linear_attn.in_proj_b.weight": {"dtype":"BF16","file":"model-00008-of-00018.safetensors","shape":[48,5120]}, + "model.language_model.layers.25.linear_attn.in_proj_qkv.weight": {"dtype":"BF16","file":"model-00008-of-00018.safetensors","shape":[10240,5120]}, + "model.language_model.layers.25.linear_attn.in_proj_z.weight": {"dtype":"BF16","file":"model-00008-of-00018.safetensors","shape":[6144,5120]}, + "model.language_model.layers.25.linear_attn.norm.weight": {"dtype":"BF16","file":"model-00008-of-00018.safetensors","shape":[128]}, + "model.language_model.layers.25.linear_attn.out_proj.weight": {"dtype":"BF16","file":"model-00008-of-00018.safetensors","shape":[5120,6144]}, + "model.language_model.layers.25.mlp.down_proj.weight": {"dtype":"BF16","file":"model-00008-of-00018.safetensors","shape":[5120,17408]}, + "model.language_model.layers.25.mlp.gate_proj.weight": {"dtype":"BF16","file":"model-00008-of-00018.safetensors","shape":[17408,5120]}, + "model.language_model.layers.25.mlp.up_proj.weight": {"dtype":"BF16","file":"model-00008-of-00018.safetensors","shape":[17408,5120]}, + "model.language_model.layers.25.post_attention_layernorm.weight": {"dtype":"BF16","file":"model-00008-of-00018.safetensors","shape":[5120]}, + "model.language_model.layers.26.input_layernorm.weight": {"dtype":"BF16","file":"model-00008-of-00018.safetensors","shape":[5120]}, + "model.language_model.layers.26.linear_attn.A_log": {"dtype":"BF16","file":"model-00008-of-00018.safetensors","shape":[48]}, + "model.language_model.layers.26.linear_attn.conv1d.weight": {"dtype":"BF16","file":"model-00008-of-00018.safetensors","shape":[10240,1,4]}, + "model.language_model.layers.26.linear_attn.dt_bias": {"dtype":"BF16","file":"model-00008-of-00018.safetensors","shape":[48]}, + "model.language_model.layers.26.linear_attn.in_proj_a.weight": {"dtype":"BF16","file":"model-00008-of-00018.safetensors","shape":[48,5120]}, + "model.language_model.layers.26.linear_attn.in_proj_b.weight": {"dtype":"BF16","file":"model-00008-of-00018.safetensors","shape":[48,5120]}, + "model.language_model.layers.26.linear_attn.in_proj_qkv.weight": {"dtype":"BF16","file":"model-00008-of-00018.safetensors","shape":[10240,5120]}, + "model.language_model.layers.26.linear_attn.in_proj_z.weight": {"dtype":"BF16","file":"model-00008-of-00018.safetensors","shape":[6144,5120]}, + "model.language_model.layers.26.linear_attn.norm.weight": {"dtype":"BF16","file":"model-00008-of-00018.safetensors","shape":[128]}, + "model.language_model.layers.26.linear_attn.out_proj.weight": {"dtype":"BF16","file":"model-00008-of-00018.safetensors","shape":[5120,6144]}, + "model.language_model.layers.26.mlp.down_proj.weight": {"dtype":"BF16","file":"model-00008-of-00018.safetensors","shape":[5120,17408]}, + "model.language_model.layers.26.mlp.gate_proj.weight": {"dtype":"BF16","file":"model-00008-of-00018.safetensors","shape":[17408,5120]}, + "model.language_model.layers.26.mlp.up_proj.weight": {"dtype":"BF16","file":"model-00008-of-00018.safetensors","shape":[17408,5120]}, + "model.language_model.layers.26.post_attention_layernorm.weight": {"dtype":"BF16","file":"model-00008-of-00018.safetensors","shape":[5120]}, + "model.language_model.layers.27.input_layernorm.weight": {"dtype":"BF16","file":"model-00008-of-00018.safetensors","shape":[5120]}, + "model.language_model.layers.27.mlp.down_proj.weight": {"dtype":"BF16","file":"model-00008-of-00018.safetensors","shape":[5120,17408]}, + "model.language_model.layers.27.mlp.gate_proj.weight": {"dtype":"BF16","file":"model-00008-of-00018.safetensors","shape":[17408,5120]}, + "model.language_model.layers.27.mlp.up_proj.weight": {"dtype":"BF16","file":"model-00008-of-00018.safetensors","shape":[17408,5120]}, + "model.language_model.layers.27.post_attention_layernorm.weight": {"dtype":"BF16","file":"model-00008-of-00018.safetensors","shape":[5120]}, + "model.language_model.layers.27.self_attn.k_norm.weight": {"dtype":"BF16","file":"model-00008-of-00018.safetensors","shape":[256]}, + "model.language_model.layers.27.self_attn.k_proj.weight": {"dtype":"BF16","file":"model-00008-of-00018.safetensors","shape":[1024,5120]}, + "model.language_model.layers.27.self_attn.o_proj.weight": {"dtype":"BF16","file":"model-00008-of-00018.safetensors","shape":[5120,6144]}, + "model.language_model.layers.27.self_attn.q_norm.weight": {"dtype":"BF16","file":"model-00008-of-00018.safetensors","shape":[256]}, + "model.language_model.layers.27.self_attn.q_proj.weight": {"dtype":"BF16","file":"model-00008-of-00018.safetensors","shape":[12288,5120]}, + "model.language_model.layers.27.self_attn.v_proj.weight": {"dtype":"BF16","file":"model-00008-of-00018.safetensors","shape":[1024,5120]}, + "model.language_model.layers.28.input_layernorm.weight": {"dtype":"BF16","file":"model-00008-of-00018.safetensors","shape":[5120]}, + "model.language_model.layers.28.linear_attn.A_log": {"dtype":"BF16","file":"model-00008-of-00018.safetensors","shape":[48]}, + "model.language_model.layers.28.linear_attn.conv1d.weight": {"dtype":"BF16","file":"model-00008-of-00018.safetensors","shape":[10240,1,4]}, + "model.language_model.layers.28.linear_attn.dt_bias": {"dtype":"BF16","file":"model-00008-of-00018.safetensors","shape":[48]}, + "model.language_model.layers.28.linear_attn.in_proj_a.weight": {"dtype":"BF16","file":"model-00008-of-00018.safetensors","shape":[48,5120]}, + "model.language_model.layers.28.linear_attn.in_proj_b.weight": {"dtype":"BF16","file":"model-00008-of-00018.safetensors","shape":[48,5120]}, + "model.language_model.layers.28.linear_attn.in_proj_qkv.weight": {"dtype":"BF16","file":"model-00008-of-00018.safetensors","shape":[10240,5120]}, + "model.language_model.layers.28.linear_attn.in_proj_z.weight": {"dtype":"BF16","file":"model-00008-of-00018.safetensors","shape":[6144,5120]}, + "model.language_model.layers.28.linear_attn.norm.weight": {"dtype":"BF16","file":"model-00008-of-00018.safetensors","shape":[128]}, + "model.language_model.layers.28.linear_attn.out_proj.weight": {"dtype":"BF16","file":"model-00008-of-00018.safetensors","shape":[5120,6144]}, + "model.language_model.layers.28.mlp.down_proj.weight": {"dtype":"BF16","file":"model-00008-of-00018.safetensors","shape":[5120,17408]}, + "model.language_model.layers.28.mlp.gate_proj.weight": {"dtype":"BF16","file":"model-00008-of-00018.safetensors","shape":[17408,5120]}, + "model.language_model.layers.28.mlp.up_proj.weight": {"dtype":"BF16","file":"model-00008-of-00018.safetensors","shape":[17408,5120]}, + "model.language_model.layers.28.post_attention_layernorm.weight": {"dtype":"BF16","file":"model-00008-of-00018.safetensors","shape":[5120]}, + "model.language_model.layers.29.input_layernorm.weight": {"dtype":"BF16","file":"model-00008-of-00018.safetensors","shape":[5120]}, + "model.language_model.layers.29.linear_attn.A_log": {"dtype":"BF16","file":"model-00008-of-00018.safetensors","shape":[48]}, + "model.language_model.layers.29.linear_attn.conv1d.weight": {"dtype":"BF16","file":"model-00008-of-00018.safetensors","shape":[10240,1,4]}, + "model.language_model.layers.29.linear_attn.dt_bias": {"dtype":"BF16","file":"model-00008-of-00018.safetensors","shape":[48]}, + "model.language_model.layers.29.linear_attn.in_proj_a.weight": {"dtype":"BF16","file":"model-00008-of-00018.safetensors","shape":[48,5120]}, + "model.language_model.layers.29.linear_attn.in_proj_b.weight": {"dtype":"BF16","file":"model-00008-of-00018.safetensors","shape":[48,5120]}, + "model.language_model.layers.29.linear_attn.in_proj_qkv.weight": {"dtype":"BF16","file":"model-00008-of-00018.safetensors","shape":[10240,5120]}, + "model.language_model.layers.29.linear_attn.in_proj_z.weight": {"dtype":"BF16","file":"model-00008-of-00018.safetensors","shape":[6144,5120]}, + "model.language_model.layers.29.linear_attn.norm.weight": {"dtype":"BF16","file":"model-00008-of-00018.safetensors","shape":[128]}, + "model.language_model.layers.29.linear_attn.out_proj.weight": {"dtype":"BF16","file":"model-00009-of-00018.safetensors","shape":[5120,6144]}, + "model.language_model.layers.29.mlp.down_proj.weight": {"dtype":"BF16","file":"model-00009-of-00018.safetensors","shape":[5120,17408]}, + "model.language_model.layers.29.mlp.gate_proj.weight": {"dtype":"BF16","file":"model-00009-of-00018.safetensors","shape":[17408,5120]}, + "model.language_model.layers.29.mlp.up_proj.weight": {"dtype":"BF16","file":"model-00009-of-00018.safetensors","shape":[17408,5120]}, + "model.language_model.layers.29.post_attention_layernorm.weight": {"dtype":"BF16","file":"model-00009-of-00018.safetensors","shape":[5120]}, + "model.language_model.layers.3.input_layernorm.weight": {"dtype":"BF16","file":"model-00001-of-00018.safetensors","shape":[5120]}, + "model.language_model.layers.3.mlp.down_proj.weight": {"dtype":"BF16","file":"model-00001-of-00018.safetensors","shape":[5120,17408]}, + "model.language_model.layers.3.mlp.gate_proj.weight": {"dtype":"BF16","file":"model-00001-of-00018.safetensors","shape":[17408,5120]}, + "model.language_model.layers.3.mlp.up_proj.weight": {"dtype":"BF16","file":"model-00001-of-00018.safetensors","shape":[17408,5120]}, + "model.language_model.layers.3.post_attention_layernorm.weight": {"dtype":"BF16","file":"model-00001-of-00018.safetensors","shape":[5120]}, + "model.language_model.layers.3.self_attn.k_norm.weight": {"dtype":"BF16","file":"model-00001-of-00018.safetensors","shape":[256]}, + "model.language_model.layers.3.self_attn.k_proj.weight": {"dtype":"BF16","file":"model-00001-of-00018.safetensors","shape":[1024,5120]}, + "model.language_model.layers.3.self_attn.o_proj.weight": {"dtype":"BF16","file":"model-00001-of-00018.safetensors","shape":[5120,6144]}, + "model.language_model.layers.3.self_attn.q_norm.weight": {"dtype":"BF16","file":"model-00001-of-00018.safetensors","shape":[256]}, + "model.language_model.layers.3.self_attn.q_proj.weight": {"dtype":"BF16","file":"model-00001-of-00018.safetensors","shape":[12288,5120]}, + "model.language_model.layers.3.self_attn.v_proj.weight": {"dtype":"BF16","file":"model-00001-of-00018.safetensors","shape":[1024,5120]}, + "model.language_model.layers.30.input_layernorm.weight": {"dtype":"BF16","file":"model-00009-of-00018.safetensors","shape":[5120]}, + "model.language_model.layers.30.linear_attn.A_log": {"dtype":"BF16","file":"model-00009-of-00018.safetensors","shape":[48]}, + "model.language_model.layers.30.linear_attn.conv1d.weight": {"dtype":"BF16","file":"model-00009-of-00018.safetensors","shape":[10240,1,4]}, + "model.language_model.layers.30.linear_attn.dt_bias": {"dtype":"BF16","file":"model-00009-of-00018.safetensors","shape":[48]}, + "model.language_model.layers.30.linear_attn.in_proj_a.weight": {"dtype":"BF16","file":"model-00009-of-00018.safetensors","shape":[48,5120]}, + "model.language_model.layers.30.linear_attn.in_proj_b.weight": {"dtype":"BF16","file":"model-00009-of-00018.safetensors","shape":[48,5120]}, + "model.language_model.layers.30.linear_attn.in_proj_qkv.weight": {"dtype":"BF16","file":"model-00009-of-00018.safetensors","shape":[10240,5120]}, + "model.language_model.layers.30.linear_attn.in_proj_z.weight": {"dtype":"BF16","file":"model-00009-of-00018.safetensors","shape":[6144,5120]}, + "model.language_model.layers.30.linear_attn.norm.weight": {"dtype":"BF16","file":"model-00009-of-00018.safetensors","shape":[128]}, + "model.language_model.layers.30.linear_attn.out_proj.weight": {"dtype":"BF16","file":"model-00009-of-00018.safetensors","shape":[5120,6144]}, + "model.language_model.layers.30.mlp.down_proj.weight": {"dtype":"BF16","file":"model-00009-of-00018.safetensors","shape":[5120,17408]}, + "model.language_model.layers.30.mlp.gate_proj.weight": {"dtype":"BF16","file":"model-00009-of-00018.safetensors","shape":[17408,5120]}, + "model.language_model.layers.30.mlp.up_proj.weight": {"dtype":"BF16","file":"model-00009-of-00018.safetensors","shape":[17408,5120]}, + "model.language_model.layers.30.post_attention_layernorm.weight": {"dtype":"BF16","file":"model-00009-of-00018.safetensors","shape":[5120]}, + "model.language_model.layers.31.input_layernorm.weight": {"dtype":"BF16","file":"model-00009-of-00018.safetensors","shape":[5120]}, + "model.language_model.layers.31.mlp.down_proj.weight": {"dtype":"BF16","file":"model-00009-of-00018.safetensors","shape":[5120,17408]}, + "model.language_model.layers.31.mlp.gate_proj.weight": {"dtype":"BF16","file":"model-00009-of-00018.safetensors","shape":[17408,5120]}, + "model.language_model.layers.31.mlp.up_proj.weight": {"dtype":"BF16","file":"model-00009-of-00018.safetensors","shape":[17408,5120]}, + "model.language_model.layers.31.post_attention_layernorm.weight": {"dtype":"BF16","file":"model-00009-of-00018.safetensors","shape":[5120]}, + "model.language_model.layers.31.self_attn.k_norm.weight": {"dtype":"BF16","file":"model-00009-of-00018.safetensors","shape":[256]}, + "model.language_model.layers.31.self_attn.k_proj.weight": {"dtype":"BF16","file":"model-00009-of-00018.safetensors","shape":[1024,5120]}, + "model.language_model.layers.31.self_attn.o_proj.weight": {"dtype":"BF16","file":"model-00009-of-00018.safetensors","shape":[5120,6144]}, + "model.language_model.layers.31.self_attn.q_norm.weight": {"dtype":"BF16","file":"model-00009-of-00018.safetensors","shape":[256]}, + "model.language_model.layers.31.self_attn.q_proj.weight": {"dtype":"BF16","file":"model-00009-of-00018.safetensors","shape":[12288,5120]}, + "model.language_model.layers.31.self_attn.v_proj.weight": {"dtype":"BF16","file":"model-00009-of-00018.safetensors","shape":[1024,5120]}, + "model.language_model.layers.32.input_layernorm.weight": {"dtype":"BF16","file":"model-00010-of-00018.safetensors","shape":[5120]}, + "model.language_model.layers.32.linear_attn.A_log": {"dtype":"BF16","file":"model-00010-of-00018.safetensors","shape":[48]}, + "model.language_model.layers.32.linear_attn.conv1d.weight": {"dtype":"BF16","file":"model-00010-of-00018.safetensors","shape":[10240,1,4]}, + "model.language_model.layers.32.linear_attn.dt_bias": {"dtype":"BF16","file":"model-00010-of-00018.safetensors","shape":[48]}, + "model.language_model.layers.32.linear_attn.in_proj_a.weight": {"dtype":"BF16","file":"model-00010-of-00018.safetensors","shape":[48,5120]}, + "model.language_model.layers.32.linear_attn.in_proj_b.weight": {"dtype":"BF16","file":"model-00010-of-00018.safetensors","shape":[48,5120]}, + "model.language_model.layers.32.linear_attn.in_proj_qkv.weight": {"dtype":"BF16","file":"model-00010-of-00018.safetensors","shape":[10240,5120]}, + "model.language_model.layers.32.linear_attn.in_proj_z.weight": {"dtype":"BF16","file":"model-00010-of-00018.safetensors","shape":[6144,5120]}, + "model.language_model.layers.32.linear_attn.norm.weight": {"dtype":"BF16","file":"model-00010-of-00018.safetensors","shape":[128]}, + "model.language_model.layers.32.linear_attn.out_proj.weight": {"dtype":"BF16","file":"model-00010-of-00018.safetensors","shape":[5120,6144]}, + "model.language_model.layers.32.mlp.down_proj.weight": {"dtype":"BF16","file":"model-00010-of-00018.safetensors","shape":[5120,17408]}, + "model.language_model.layers.32.mlp.gate_proj.weight": {"dtype":"BF16","file":"model-00010-of-00018.safetensors","shape":[17408,5120]}, + "model.language_model.layers.32.mlp.up_proj.weight": {"dtype":"BF16","file":"model-00010-of-00018.safetensors","shape":[17408,5120]}, + "model.language_model.layers.32.post_attention_layernorm.weight": {"dtype":"BF16","file":"model-00010-of-00018.safetensors","shape":[5120]}, + "model.language_model.layers.33.input_layernorm.weight": {"dtype":"BF16","file":"model-00010-of-00018.safetensors","shape":[5120]}, + "model.language_model.layers.33.linear_attn.A_log": {"dtype":"BF16","file":"model-00010-of-00018.safetensors","shape":[48]}, + "model.language_model.layers.33.linear_attn.conv1d.weight": {"dtype":"BF16","file":"model-00010-of-00018.safetensors","shape":[10240,1,4]}, + "model.language_model.layers.33.linear_attn.dt_bias": {"dtype":"BF16","file":"model-00010-of-00018.safetensors","shape":[48]}, + "model.language_model.layers.33.linear_attn.in_proj_a.weight": {"dtype":"BF16","file":"model-00010-of-00018.safetensors","shape":[48,5120]}, + "model.language_model.layers.33.linear_attn.in_proj_b.weight": {"dtype":"BF16","file":"model-00010-of-00018.safetensors","shape":[48,5120]}, + "model.language_model.layers.33.linear_attn.in_proj_qkv.weight": {"dtype":"BF16","file":"model-00010-of-00018.safetensors","shape":[10240,5120]}, + "model.language_model.layers.33.linear_attn.in_proj_z.weight": {"dtype":"BF16","file":"model-00010-of-00018.safetensors","shape":[6144,5120]}, + "model.language_model.layers.33.linear_attn.norm.weight": {"dtype":"BF16","file":"model-00010-of-00018.safetensors","shape":[128]}, + "model.language_model.layers.33.linear_attn.out_proj.weight": {"dtype":"BF16","file":"model-00010-of-00018.safetensors","shape":[5120,6144]}, + "model.language_model.layers.33.mlp.down_proj.weight": {"dtype":"BF16","file":"model-00010-of-00018.safetensors","shape":[5120,17408]}, + "model.language_model.layers.33.mlp.gate_proj.weight": {"dtype":"BF16","file":"model-00010-of-00018.safetensors","shape":[17408,5120]}, + "model.language_model.layers.33.mlp.up_proj.weight": {"dtype":"BF16","file":"model-00010-of-00018.safetensors","shape":[17408,5120]}, + "model.language_model.layers.33.post_attention_layernorm.weight": {"dtype":"BF16","file":"model-00010-of-00018.safetensors","shape":[5120]}, + "model.language_model.layers.34.input_layernorm.weight": {"dtype":"BF16","file":"model-00010-of-00018.safetensors","shape":[5120]}, + "model.language_model.layers.34.linear_attn.A_log": {"dtype":"BF16","file":"model-00010-of-00018.safetensors","shape":[48]}, + "model.language_model.layers.34.linear_attn.conv1d.weight": {"dtype":"BF16","file":"model-00010-of-00018.safetensors","shape":[10240,1,4]}, + "model.language_model.layers.34.linear_attn.dt_bias": {"dtype":"BF16","file":"model-00010-of-00018.safetensors","shape":[48]}, + "model.language_model.layers.34.linear_attn.in_proj_a.weight": {"dtype":"BF16","file":"model-00010-of-00018.safetensors","shape":[48,5120]}, + "model.language_model.layers.34.linear_attn.in_proj_b.weight": {"dtype":"BF16","file":"model-00010-of-00018.safetensors","shape":[48,5120]}, + "model.language_model.layers.34.linear_attn.in_proj_qkv.weight": {"dtype":"BF16","file":"model-00010-of-00018.safetensors","shape":[10240,5120]}, + "model.language_model.layers.34.linear_attn.in_proj_z.weight": {"dtype":"BF16","file":"model-00010-of-00018.safetensors","shape":[6144,5120]}, + "model.language_model.layers.34.linear_attn.norm.weight": {"dtype":"BF16","file":"model-00010-of-00018.safetensors","shape":[128]}, + "model.language_model.layers.34.linear_attn.out_proj.weight": {"dtype":"BF16","file":"model-00010-of-00018.safetensors","shape":[5120,6144]}, + "model.language_model.layers.34.mlp.down_proj.weight": {"dtype":"BF16","file":"model-00010-of-00018.safetensors","shape":[5120,17408]}, + "model.language_model.layers.34.mlp.gate_proj.weight": {"dtype":"BF16","file":"model-00010-of-00018.safetensors","shape":[17408,5120]}, + "model.language_model.layers.34.mlp.up_proj.weight": {"dtype":"BF16","file":"model-00010-of-00018.safetensors","shape":[17408,5120]}, + "model.language_model.layers.34.post_attention_layernorm.weight": {"dtype":"BF16","file":"model-00010-of-00018.safetensors","shape":[5120]}, + "model.language_model.layers.35.input_layernorm.weight": {"dtype":"BF16","file":"model-00010-of-00018.safetensors","shape":[5120]}, + "model.language_model.layers.35.mlp.down_proj.weight": {"dtype":"BF16","file":"model-00010-of-00018.safetensors","shape":[5120,17408]}, + "model.language_model.layers.35.mlp.gate_proj.weight": {"dtype":"BF16","file":"model-00010-of-00018.safetensors","shape":[17408,5120]}, + "model.language_model.layers.35.mlp.up_proj.weight": {"dtype":"BF16","file":"model-00010-of-00018.safetensors","shape":[17408,5120]}, + "model.language_model.layers.35.post_attention_layernorm.weight": {"dtype":"BF16","file":"model-00010-of-00018.safetensors","shape":[5120]}, + "model.language_model.layers.35.self_attn.k_norm.weight": {"dtype":"BF16","file":"model-00010-of-00018.safetensors","shape":[256]}, + "model.language_model.layers.35.self_attn.k_proj.weight": {"dtype":"BF16","file":"model-00010-of-00018.safetensors","shape":[1024,5120]}, + "model.language_model.layers.35.self_attn.o_proj.weight": {"dtype":"BF16","file":"model-00010-of-00018.safetensors","shape":[5120,6144]}, + "model.language_model.layers.35.self_attn.q_norm.weight": {"dtype":"BF16","file":"model-00010-of-00018.safetensors","shape":[256]}, + "model.language_model.layers.35.self_attn.q_proj.weight": {"dtype":"BF16","file":"model-00010-of-00018.safetensors","shape":[12288,5120]}, + "model.language_model.layers.35.self_attn.v_proj.weight": {"dtype":"BF16","file":"model-00010-of-00018.safetensors","shape":[1024,5120]}, + "model.language_model.layers.36.input_layernorm.weight": {"dtype":"BF16","file":"model-00010-of-00018.safetensors","shape":[5120]}, + "model.language_model.layers.36.linear_attn.A_log": {"dtype":"BF16","file":"model-00010-of-00018.safetensors","shape":[48]}, + "model.language_model.layers.36.linear_attn.conv1d.weight": {"dtype":"BF16","file":"model-00010-of-00018.safetensors","shape":[10240,1,4]}, + "model.language_model.layers.36.linear_attn.dt_bias": {"dtype":"BF16","file":"model-00010-of-00018.safetensors","shape":[48]}, + "model.language_model.layers.36.linear_attn.in_proj_a.weight": {"dtype":"BF16","file":"model-00010-of-00018.safetensors","shape":[48,5120]}, + "model.language_model.layers.36.linear_attn.in_proj_b.weight": {"dtype":"BF16","file":"model-00010-of-00018.safetensors","shape":[48,5120]}, + "model.language_model.layers.36.linear_attn.in_proj_qkv.weight": {"dtype":"BF16","file":"model-00010-of-00018.safetensors","shape":[10240,5120]}, + "model.language_model.layers.36.linear_attn.in_proj_z.weight": {"dtype":"BF16","file":"model-00010-of-00018.safetensors","shape":[6144,5120]}, + "model.language_model.layers.36.linear_attn.norm.weight": {"dtype":"BF16","file":"model-00010-of-00018.safetensors","shape":[128]}, + "model.language_model.layers.36.linear_attn.out_proj.weight": {"dtype":"BF16","file":"model-00010-of-00018.safetensors","shape":[5120,6144]}, + "model.language_model.layers.36.mlp.down_proj.weight": {"dtype":"BF16","file":"model-00010-of-00018.safetensors","shape":[5120,17408]}, + "model.language_model.layers.36.mlp.gate_proj.weight": {"dtype":"BF16","file":"model-00010-of-00018.safetensors","shape":[17408,5120]}, + "model.language_model.layers.36.mlp.up_proj.weight": {"dtype":"BF16","file":"model-00010-of-00018.safetensors","shape":[17408,5120]}, + "model.language_model.layers.36.post_attention_layernorm.weight": {"dtype":"BF16","file":"model-00010-of-00018.safetensors","shape":[5120]}, + "model.language_model.layers.37.input_layernorm.weight": {"dtype":"BF16","file":"model-00010-of-00018.safetensors","shape":[5120]}, + "model.language_model.layers.37.linear_attn.A_log": {"dtype":"BF16","file":"model-00010-of-00018.safetensors","shape":[48]}, + "model.language_model.layers.37.linear_attn.conv1d.weight": {"dtype":"BF16","file":"model-00010-of-00018.safetensors","shape":[10240,1,4]}, + "model.language_model.layers.37.linear_attn.dt_bias": {"dtype":"BF16","file":"model-00010-of-00018.safetensors","shape":[48]}, + "model.language_model.layers.37.linear_attn.in_proj_a.weight": {"dtype":"BF16","file":"model-00010-of-00018.safetensors","shape":[48,5120]}, + "model.language_model.layers.37.linear_attn.in_proj_b.weight": {"dtype":"BF16","file":"model-00010-of-00018.safetensors","shape":[48,5120]}, + "model.language_model.layers.37.linear_attn.in_proj_qkv.weight": {"dtype":"BF16","file":"model-00010-of-00018.safetensors","shape":[10240,5120]}, + "model.language_model.layers.37.linear_attn.in_proj_z.weight": {"dtype":"BF16","file":"model-00010-of-00018.safetensors","shape":[6144,5120]}, + "model.language_model.layers.37.linear_attn.norm.weight": {"dtype":"BF16","file":"model-00010-of-00018.safetensors","shape":[128]}, + "model.language_model.layers.37.linear_attn.out_proj.weight": {"dtype":"BF16","file":"model-00011-of-00018.safetensors","shape":[5120,6144]}, + "model.language_model.layers.37.mlp.down_proj.weight": {"dtype":"BF16","file":"model-00011-of-00018.safetensors","shape":[5120,17408]}, + "model.language_model.layers.37.mlp.gate_proj.weight": {"dtype":"BF16","file":"model-00011-of-00018.safetensors","shape":[17408,5120]}, + "model.language_model.layers.37.mlp.up_proj.weight": {"dtype":"BF16","file":"model-00011-of-00018.safetensors","shape":[17408,5120]}, + "model.language_model.layers.37.post_attention_layernorm.weight": {"dtype":"BF16","file":"model-00011-of-00018.safetensors","shape":[5120]}, + "model.language_model.layers.38.input_layernorm.weight": {"dtype":"BF16","file":"model-00011-of-00018.safetensors","shape":[5120]}, + "model.language_model.layers.38.linear_attn.A_log": {"dtype":"BF16","file":"model-00011-of-00018.safetensors","shape":[48]}, + "model.language_model.layers.38.linear_attn.conv1d.weight": {"dtype":"BF16","file":"model-00011-of-00018.safetensors","shape":[10240,1,4]}, + "model.language_model.layers.38.linear_attn.dt_bias": {"dtype":"BF16","file":"model-00011-of-00018.safetensors","shape":[48]}, + "model.language_model.layers.38.linear_attn.in_proj_a.weight": {"dtype":"BF16","file":"model-00011-of-00018.safetensors","shape":[48,5120]}, + "model.language_model.layers.38.linear_attn.in_proj_b.weight": {"dtype":"BF16","file":"model-00011-of-00018.safetensors","shape":[48,5120]}, + "model.language_model.layers.38.linear_attn.in_proj_qkv.weight": {"dtype":"BF16","file":"model-00011-of-00018.safetensors","shape":[10240,5120]}, + "model.language_model.layers.38.linear_attn.in_proj_z.weight": {"dtype":"BF16","file":"model-00011-of-00018.safetensors","shape":[6144,5120]}, + "model.language_model.layers.38.linear_attn.norm.weight": {"dtype":"BF16","file":"model-00011-of-00018.safetensors","shape":[128]}, + "model.language_model.layers.38.linear_attn.out_proj.weight": {"dtype":"BF16","file":"model-00011-of-00018.safetensors","shape":[5120,6144]}, + "model.language_model.layers.38.mlp.down_proj.weight": {"dtype":"BF16","file":"model-00011-of-00018.safetensors","shape":[5120,17408]}, + "model.language_model.layers.38.mlp.gate_proj.weight": {"dtype":"BF16","file":"model-00011-of-00018.safetensors","shape":[17408,5120]}, + "model.language_model.layers.38.mlp.up_proj.weight": {"dtype":"BF16","file":"model-00011-of-00018.safetensors","shape":[17408,5120]}, + "model.language_model.layers.38.post_attention_layernorm.weight": {"dtype":"BF16","file":"model-00011-of-00018.safetensors","shape":[5120]}, + "model.language_model.layers.39.input_layernorm.weight": {"dtype":"BF16","file":"model-00011-of-00018.safetensors","shape":[5120]}, + "model.language_model.layers.39.mlp.down_proj.weight": {"dtype":"BF16","file":"model-00011-of-00018.safetensors","shape":[5120,17408]}, + "model.language_model.layers.39.mlp.gate_proj.weight": {"dtype":"BF16","file":"model-00011-of-00018.safetensors","shape":[17408,5120]}, + "model.language_model.layers.39.mlp.up_proj.weight": {"dtype":"BF16","file":"model-00011-of-00018.safetensors","shape":[17408,5120]}, + "model.language_model.layers.39.post_attention_layernorm.weight": {"dtype":"BF16","file":"model-00011-of-00018.safetensors","shape":[5120]}, + "model.language_model.layers.39.self_attn.k_norm.weight": {"dtype":"BF16","file":"model-00011-of-00018.safetensors","shape":[256]}, + "model.language_model.layers.39.self_attn.k_proj.weight": {"dtype":"BF16","file":"model-00011-of-00018.safetensors","shape":[1024,5120]}, + "model.language_model.layers.39.self_attn.o_proj.weight": {"dtype":"BF16","file":"model-00011-of-00018.safetensors","shape":[5120,6144]}, + "model.language_model.layers.39.self_attn.q_norm.weight": {"dtype":"BF16","file":"model-00011-of-00018.safetensors","shape":[256]}, + "model.language_model.layers.39.self_attn.q_proj.weight": {"dtype":"BF16","file":"model-00011-of-00018.safetensors","shape":[12288,5120]}, + "model.language_model.layers.39.self_attn.v_proj.weight": {"dtype":"BF16","file":"model-00011-of-00018.safetensors","shape":[1024,5120]}, + "model.language_model.layers.4.input_layernorm.weight": {"dtype":"BF16","file":"model-00001-of-00018.safetensors","shape":[5120]}, + "model.language_model.layers.4.linear_attn.A_log": {"dtype":"BF16","file":"model-00001-of-00018.safetensors","shape":[48]}, + "model.language_model.layers.4.linear_attn.conv1d.weight": {"dtype":"BF16","file":"model-00001-of-00018.safetensors","shape":[10240,1,4]}, + "model.language_model.layers.4.linear_attn.dt_bias": {"dtype":"BF16","file":"model-00001-of-00018.safetensors","shape":[48]}, + "model.language_model.layers.4.linear_attn.in_proj_a.weight": {"dtype":"BF16","file":"model-00001-of-00018.safetensors","shape":[48,5120]}, + "model.language_model.layers.4.linear_attn.in_proj_b.weight": {"dtype":"BF16","file":"model-00001-of-00018.safetensors","shape":[48,5120]}, + "model.language_model.layers.4.linear_attn.in_proj_qkv.weight": {"dtype":"BF16","file":"model-00002-of-00018.safetensors","shape":[10240,5120]}, + "model.language_model.layers.4.linear_attn.in_proj_z.weight": {"dtype":"BF16","file":"model-00002-of-00018.safetensors","shape":[6144,5120]}, + "model.language_model.layers.4.linear_attn.norm.weight": {"dtype":"BF16","file":"model-00002-of-00018.safetensors","shape":[128]}, + "model.language_model.layers.4.linear_attn.out_proj.weight": {"dtype":"BF16","file":"model-00002-of-00018.safetensors","shape":[5120,6144]}, + "model.language_model.layers.4.mlp.down_proj.weight": {"dtype":"BF16","file":"model-00002-of-00018.safetensors","shape":[5120,17408]}, + "model.language_model.layers.4.mlp.gate_proj.weight": {"dtype":"BF16","file":"model-00002-of-00018.safetensors","shape":[17408,5120]}, + "model.language_model.layers.4.mlp.up_proj.weight": {"dtype":"BF16","file":"model-00002-of-00018.safetensors","shape":[17408,5120]}, + "model.language_model.layers.4.post_attention_layernorm.weight": {"dtype":"BF16","file":"model-00002-of-00018.safetensors","shape":[5120]}, + "model.language_model.layers.40.input_layernorm.weight": {"dtype":"BF16","file":"model-00012-of-00018.safetensors","shape":[5120]}, + "model.language_model.layers.40.linear_attn.A_log": {"dtype":"BF16","file":"model-00012-of-00018.safetensors","shape":[48]}, + "model.language_model.layers.40.linear_attn.conv1d.weight": {"dtype":"BF16","file":"model-00012-of-00018.safetensors","shape":[10240,1,4]}, + "model.language_model.layers.40.linear_attn.dt_bias": {"dtype":"BF16","file":"model-00012-of-00018.safetensors","shape":[48]}, + "model.language_model.layers.40.linear_attn.in_proj_a.weight": {"dtype":"BF16","file":"model-00012-of-00018.safetensors","shape":[48,5120]}, + "model.language_model.layers.40.linear_attn.in_proj_b.weight": {"dtype":"BF16","file":"model-00012-of-00018.safetensors","shape":[48,5120]}, + "model.language_model.layers.40.linear_attn.in_proj_qkv.weight": {"dtype":"BF16","file":"model-00012-of-00018.safetensors","shape":[10240,5120]}, + "model.language_model.layers.40.linear_attn.in_proj_z.weight": {"dtype":"BF16","file":"model-00012-of-00018.safetensors","shape":[6144,5120]}, + "model.language_model.layers.40.linear_attn.norm.weight": {"dtype":"BF16","file":"model-00012-of-00018.safetensors","shape":[128]}, + "model.language_model.layers.40.linear_attn.out_proj.weight": {"dtype":"BF16","file":"model-00012-of-00018.safetensors","shape":[5120,6144]}, + "model.language_model.layers.40.mlp.down_proj.weight": {"dtype":"BF16","file":"model-00012-of-00018.safetensors","shape":[5120,17408]}, + "model.language_model.layers.40.mlp.gate_proj.weight": {"dtype":"BF16","file":"model-00012-of-00018.safetensors","shape":[17408,5120]}, + "model.language_model.layers.40.mlp.up_proj.weight": {"dtype":"BF16","file":"model-00012-of-00018.safetensors","shape":[17408,5120]}, + "model.language_model.layers.40.post_attention_layernorm.weight": {"dtype":"BF16","file":"model-00012-of-00018.safetensors","shape":[5120]}, + "model.language_model.layers.41.input_layernorm.weight": {"dtype":"BF16","file":"model-00012-of-00018.safetensors","shape":[5120]}, + "model.language_model.layers.41.linear_attn.A_log": {"dtype":"BF16","file":"model-00012-of-00018.safetensors","shape":[48]}, + "model.language_model.layers.41.linear_attn.conv1d.weight": {"dtype":"BF16","file":"model-00012-of-00018.safetensors","shape":[10240,1,4]}, + "model.language_model.layers.41.linear_attn.dt_bias": {"dtype":"BF16","file":"model-00012-of-00018.safetensors","shape":[48]}, + "model.language_model.layers.41.linear_attn.in_proj_a.weight": {"dtype":"BF16","file":"model-00012-of-00018.safetensors","shape":[48,5120]}, + "model.language_model.layers.41.linear_attn.in_proj_b.weight": {"dtype":"BF16","file":"model-00012-of-00018.safetensors","shape":[48,5120]}, + "model.language_model.layers.41.linear_attn.in_proj_qkv.weight": {"dtype":"BF16","file":"model-00012-of-00018.safetensors","shape":[10240,5120]}, + "model.language_model.layers.41.linear_attn.in_proj_z.weight": {"dtype":"BF16","file":"model-00012-of-00018.safetensors","shape":[6144,5120]}, + "model.language_model.layers.41.linear_attn.norm.weight": {"dtype":"BF16","file":"model-00012-of-00018.safetensors","shape":[128]}, + "model.language_model.layers.41.linear_attn.out_proj.weight": {"dtype":"BF16","file":"model-00012-of-00018.safetensors","shape":[5120,6144]}, + "model.language_model.layers.41.mlp.down_proj.weight": {"dtype":"BF16","file":"model-00012-of-00018.safetensors","shape":[5120,17408]}, + "model.language_model.layers.41.mlp.gate_proj.weight": {"dtype":"BF16","file":"model-00012-of-00018.safetensors","shape":[17408,5120]}, + "model.language_model.layers.41.mlp.up_proj.weight": {"dtype":"BF16","file":"model-00012-of-00018.safetensors","shape":[17408,5120]}, + "model.language_model.layers.41.post_attention_layernorm.weight": {"dtype":"BF16","file":"model-00012-of-00018.safetensors","shape":[5120]}, + "model.language_model.layers.42.input_layernorm.weight": {"dtype":"BF16","file":"model-00012-of-00018.safetensors","shape":[5120]}, + "model.language_model.layers.42.linear_attn.A_log": {"dtype":"BF16","file":"model-00012-of-00018.safetensors","shape":[48]}, + "model.language_model.layers.42.linear_attn.conv1d.weight": {"dtype":"BF16","file":"model-00012-of-00018.safetensors","shape":[10240,1,4]}, + "model.language_model.layers.42.linear_attn.dt_bias": {"dtype":"BF16","file":"model-00012-of-00018.safetensors","shape":[48]}, + "model.language_model.layers.42.linear_attn.in_proj_a.weight": {"dtype":"BF16","file":"model-00012-of-00018.safetensors","shape":[48,5120]}, + "model.language_model.layers.42.linear_attn.in_proj_b.weight": {"dtype":"BF16","file":"model-00012-of-00018.safetensors","shape":[48,5120]}, + "model.language_model.layers.42.linear_attn.in_proj_qkv.weight": {"dtype":"BF16","file":"model-00012-of-00018.safetensors","shape":[10240,5120]}, + "model.language_model.layers.42.linear_attn.in_proj_z.weight": {"dtype":"BF16","file":"model-00012-of-00018.safetensors","shape":[6144,5120]}, + "model.language_model.layers.42.linear_attn.norm.weight": {"dtype":"BF16","file":"model-00012-of-00018.safetensors","shape":[128]}, + "model.language_model.layers.42.linear_attn.out_proj.weight": {"dtype":"BF16","file":"model-00012-of-00018.safetensors","shape":[5120,6144]}, + "model.language_model.layers.42.mlp.down_proj.weight": {"dtype":"BF16","file":"model-00012-of-00018.safetensors","shape":[5120,17408]}, + "model.language_model.layers.42.mlp.gate_proj.weight": {"dtype":"BF16","file":"model-00012-of-00018.safetensors","shape":[17408,5120]}, + "model.language_model.layers.42.mlp.up_proj.weight": {"dtype":"BF16","file":"model-00012-of-00018.safetensors","shape":[17408,5120]}, + "model.language_model.layers.42.post_attention_layernorm.weight": {"dtype":"BF16","file":"model-00012-of-00018.safetensors","shape":[5120]}, + "model.language_model.layers.43.input_layernorm.weight": {"dtype":"BF16","file":"model-00012-of-00018.safetensors","shape":[5120]}, + "model.language_model.layers.43.mlp.down_proj.weight": {"dtype":"BF16","file":"model-00012-of-00018.safetensors","shape":[5120,17408]}, + "model.language_model.layers.43.mlp.gate_proj.weight": {"dtype":"BF16","file":"model-00012-of-00018.safetensors","shape":[17408,5120]}, + "model.language_model.layers.43.mlp.up_proj.weight": {"dtype":"BF16","file":"model-00012-of-00018.safetensors","shape":[17408,5120]}, + "model.language_model.layers.43.post_attention_layernorm.weight": {"dtype":"BF16","file":"model-00012-of-00018.safetensors","shape":[5120]}, + "model.language_model.layers.43.self_attn.k_norm.weight": {"dtype":"BF16","file":"model-00012-of-00018.safetensors","shape":[256]}, + "model.language_model.layers.43.self_attn.k_proj.weight": {"dtype":"BF16","file":"model-00012-of-00018.safetensors","shape":[1024,5120]}, + "model.language_model.layers.43.self_attn.o_proj.weight": {"dtype":"BF16","file":"model-00012-of-00018.safetensors","shape":[5120,6144]}, + "model.language_model.layers.43.self_attn.q_norm.weight": {"dtype":"BF16","file":"model-00012-of-00018.safetensors","shape":[256]}, + "model.language_model.layers.43.self_attn.q_proj.weight": {"dtype":"BF16","file":"model-00012-of-00018.safetensors","shape":[12288,5120]}, + "model.language_model.layers.43.self_attn.v_proj.weight": {"dtype":"BF16","file":"model-00012-of-00018.safetensors","shape":[1024,5120]}, + "model.language_model.layers.44.input_layernorm.weight": {"dtype":"BF16","file":"model-00012-of-00018.safetensors","shape":[5120]}, + "model.language_model.layers.44.linear_attn.A_log": {"dtype":"BF16","file":"model-00012-of-00018.safetensors","shape":[48]}, + "model.language_model.layers.44.linear_attn.conv1d.weight": {"dtype":"BF16","file":"model-00012-of-00018.safetensors","shape":[10240,1,4]}, + "model.language_model.layers.44.linear_attn.dt_bias": {"dtype":"BF16","file":"model-00012-of-00018.safetensors","shape":[48]}, + "model.language_model.layers.44.linear_attn.in_proj_a.weight": {"dtype":"BF16","file":"model-00012-of-00018.safetensors","shape":[48,5120]}, + "model.language_model.layers.44.linear_attn.in_proj_b.weight": {"dtype":"BF16","file":"model-00012-of-00018.safetensors","shape":[48,5120]}, + "model.language_model.layers.44.linear_attn.in_proj_qkv.weight": {"dtype":"BF16","file":"model-00012-of-00018.safetensors","shape":[10240,5120]}, + "model.language_model.layers.44.linear_attn.in_proj_z.weight": {"dtype":"BF16","file":"model-00012-of-00018.safetensors","shape":[6144,5120]}, + "model.language_model.layers.44.linear_attn.norm.weight": {"dtype":"BF16","file":"model-00012-of-00018.safetensors","shape":[128]}, + "model.language_model.layers.44.linear_attn.out_proj.weight": {"dtype":"BF16","file":"model-00012-of-00018.safetensors","shape":[5120,6144]}, + "model.language_model.layers.44.mlp.down_proj.weight": {"dtype":"BF16","file":"model-00012-of-00018.safetensors","shape":[5120,17408]}, + "model.language_model.layers.44.mlp.gate_proj.weight": {"dtype":"BF16","file":"model-00012-of-00018.safetensors","shape":[17408,5120]}, + "model.language_model.layers.44.mlp.up_proj.weight": {"dtype":"BF16","file":"model-00012-of-00018.safetensors","shape":[17408,5120]}, + "model.language_model.layers.44.post_attention_layernorm.weight": {"dtype":"BF16","file":"model-00012-of-00018.safetensors","shape":[5120]}, + "model.language_model.layers.45.input_layernorm.weight": {"dtype":"BF16","file":"model-00012-of-00018.safetensors","shape":[5120]}, + "model.language_model.layers.45.linear_attn.A_log": {"dtype":"BF16","file":"model-00012-of-00018.safetensors","shape":[48]}, + "model.language_model.layers.45.linear_attn.conv1d.weight": {"dtype":"BF16","file":"model-00012-of-00018.safetensors","shape":[10240,1,4]}, + "model.language_model.layers.45.linear_attn.dt_bias": {"dtype":"BF16","file":"model-00012-of-00018.safetensors","shape":[48]}, + "model.language_model.layers.45.linear_attn.in_proj_a.weight": {"dtype":"BF16","file":"model-00012-of-00018.safetensors","shape":[48,5120]}, + "model.language_model.layers.45.linear_attn.in_proj_b.weight": {"dtype":"BF16","file":"model-00012-of-00018.safetensors","shape":[48,5120]}, + "model.language_model.layers.45.linear_attn.in_proj_qkv.weight": {"dtype":"BF16","file":"model-00012-of-00018.safetensors","shape":[10240,5120]}, + "model.language_model.layers.45.linear_attn.in_proj_z.weight": {"dtype":"BF16","file":"model-00012-of-00018.safetensors","shape":[6144,5120]}, + "model.language_model.layers.45.linear_attn.norm.weight": {"dtype":"BF16","file":"model-00012-of-00018.safetensors","shape":[128]}, + "model.language_model.layers.45.linear_attn.out_proj.weight": {"dtype":"BF16","file":"model-00013-of-00018.safetensors","shape":[5120,6144]}, + "model.language_model.layers.45.mlp.down_proj.weight": {"dtype":"BF16","file":"model-00013-of-00018.safetensors","shape":[5120,17408]}, + "model.language_model.layers.45.mlp.gate_proj.weight": {"dtype":"BF16","file":"model-00013-of-00018.safetensors","shape":[17408,5120]}, + "model.language_model.layers.45.mlp.up_proj.weight": {"dtype":"BF16","file":"model-00013-of-00018.safetensors","shape":[17408,5120]}, + "model.language_model.layers.45.post_attention_layernorm.weight": {"dtype":"BF16","file":"model-00013-of-00018.safetensors","shape":[5120]}, + "model.language_model.layers.46.input_layernorm.weight": {"dtype":"BF16","file":"model-00013-of-00018.safetensors","shape":[5120]}, + "model.language_model.layers.46.linear_attn.A_log": {"dtype":"BF16","file":"model-00013-of-00018.safetensors","shape":[48]}, + "model.language_model.layers.46.linear_attn.conv1d.weight": {"dtype":"BF16","file":"model-00013-of-00018.safetensors","shape":[10240,1,4]}, + "model.language_model.layers.46.linear_attn.dt_bias": {"dtype":"BF16","file":"model-00013-of-00018.safetensors","shape":[48]}, + "model.language_model.layers.46.linear_attn.in_proj_a.weight": {"dtype":"BF16","file":"model-00013-of-00018.safetensors","shape":[48,5120]}, + "model.language_model.layers.46.linear_attn.in_proj_b.weight": {"dtype":"BF16","file":"model-00013-of-00018.safetensors","shape":[48,5120]}, + "model.language_model.layers.46.linear_attn.in_proj_qkv.weight": {"dtype":"BF16","file":"model-00013-of-00018.safetensors","shape":[10240,5120]}, + "model.language_model.layers.46.linear_attn.in_proj_z.weight": {"dtype":"BF16","file":"model-00013-of-00018.safetensors","shape":[6144,5120]}, + "model.language_model.layers.46.linear_attn.norm.weight": {"dtype":"BF16","file":"model-00013-of-00018.safetensors","shape":[128]}, + "model.language_model.layers.46.linear_attn.out_proj.weight": {"dtype":"BF16","file":"model-00013-of-00018.safetensors","shape":[5120,6144]}, + "model.language_model.layers.46.mlp.down_proj.weight": {"dtype":"BF16","file":"model-00013-of-00018.safetensors","shape":[5120,17408]}, + "model.language_model.layers.46.mlp.gate_proj.weight": {"dtype":"BF16","file":"model-00013-of-00018.safetensors","shape":[17408,5120]}, + "model.language_model.layers.46.mlp.up_proj.weight": {"dtype":"BF16","file":"model-00013-of-00018.safetensors","shape":[17408,5120]}, + "model.language_model.layers.46.post_attention_layernorm.weight": {"dtype":"BF16","file":"model-00013-of-00018.safetensors","shape":[5120]}, + "model.language_model.layers.47.input_layernorm.weight": {"dtype":"BF16","file":"model-00013-of-00018.safetensors","shape":[5120]}, + "model.language_model.layers.47.mlp.down_proj.weight": {"dtype":"BF16","file":"model-00013-of-00018.safetensors","shape":[5120,17408]}, + "model.language_model.layers.47.mlp.gate_proj.weight": {"dtype":"BF16","file":"model-00013-of-00018.safetensors","shape":[17408,5120]}, + "model.language_model.layers.47.mlp.up_proj.weight": {"dtype":"BF16","file":"model-00013-of-00018.safetensors","shape":[17408,5120]}, + "model.language_model.layers.47.post_attention_layernorm.weight": {"dtype":"BF16","file":"model-00013-of-00018.safetensors","shape":[5120]}, + "model.language_model.layers.47.self_attn.k_norm.weight": {"dtype":"BF16","file":"model-00013-of-00018.safetensors","shape":[256]}, + "model.language_model.layers.47.self_attn.k_proj.weight": {"dtype":"BF16","file":"model-00013-of-00018.safetensors","shape":[1024,5120]}, + "model.language_model.layers.47.self_attn.o_proj.weight": {"dtype":"BF16","file":"model-00013-of-00018.safetensors","shape":[5120,6144]}, + "model.language_model.layers.47.self_attn.q_norm.weight": {"dtype":"BF16","file":"model-00013-of-00018.safetensors","shape":[256]}, + "model.language_model.layers.47.self_attn.q_proj.weight": {"dtype":"BF16","file":"model-00013-of-00018.safetensors","shape":[12288,5120]}, + "model.language_model.layers.47.self_attn.v_proj.weight": {"dtype":"BF16","file":"model-00013-of-00018.safetensors","shape":[1024,5120]}, + "model.language_model.layers.48.input_layernorm.weight": {"dtype":"BF16","file":"model-00014-of-00018.safetensors","shape":[5120]}, + "model.language_model.layers.48.linear_attn.A_log": {"dtype":"BF16","file":"model-00014-of-00018.safetensors","shape":[48]}, + "model.language_model.layers.48.linear_attn.conv1d.weight": {"dtype":"BF16","file":"model-00014-of-00018.safetensors","shape":[10240,1,4]}, + "model.language_model.layers.48.linear_attn.dt_bias": {"dtype":"BF16","file":"model-00014-of-00018.safetensors","shape":[48]}, + "model.language_model.layers.48.linear_attn.in_proj_a.weight": {"dtype":"BF16","file":"model-00014-of-00018.safetensors","shape":[48,5120]}, + "model.language_model.layers.48.linear_attn.in_proj_b.weight": {"dtype":"BF16","file":"model-00014-of-00018.safetensors","shape":[48,5120]}, + "model.language_model.layers.48.linear_attn.in_proj_qkv.weight": {"dtype":"BF16","file":"model-00014-of-00018.safetensors","shape":[10240,5120]}, + "model.language_model.layers.48.linear_attn.in_proj_z.weight": {"dtype":"BF16","file":"model-00014-of-00018.safetensors","shape":[6144,5120]}, + "model.language_model.layers.48.linear_attn.norm.weight": {"dtype":"BF16","file":"model-00014-of-00018.safetensors","shape":[128]}, + "model.language_model.layers.48.linear_attn.out_proj.weight": {"dtype":"BF16","file":"model-00014-of-00018.safetensors","shape":[5120,6144]}, + "model.language_model.layers.48.mlp.down_proj.weight": {"dtype":"BF16","file":"model-00014-of-00018.safetensors","shape":[5120,17408]}, + "model.language_model.layers.48.mlp.gate_proj.weight": {"dtype":"BF16","file":"model-00014-of-00018.safetensors","shape":[17408,5120]}, + "model.language_model.layers.48.mlp.up_proj.weight": {"dtype":"BF16","file":"model-00014-of-00018.safetensors","shape":[17408,5120]}, + "model.language_model.layers.48.post_attention_layernorm.weight": {"dtype":"BF16","file":"model-00014-of-00018.safetensors","shape":[5120]}, + "model.language_model.layers.49.input_layernorm.weight": {"dtype":"BF16","file":"model-00014-of-00018.safetensors","shape":[5120]}, + "model.language_model.layers.49.linear_attn.A_log": {"dtype":"BF16","file":"model-00014-of-00018.safetensors","shape":[48]}, + "model.language_model.layers.49.linear_attn.conv1d.weight": {"dtype":"BF16","file":"model-00014-of-00018.safetensors","shape":[10240,1,4]}, + "model.language_model.layers.49.linear_attn.dt_bias": {"dtype":"BF16","file":"model-00014-of-00018.safetensors","shape":[48]}, + "model.language_model.layers.49.linear_attn.in_proj_a.weight": {"dtype":"BF16","file":"model-00014-of-00018.safetensors","shape":[48,5120]}, + "model.language_model.layers.49.linear_attn.in_proj_b.weight": {"dtype":"BF16","file":"model-00014-of-00018.safetensors","shape":[48,5120]}, + "model.language_model.layers.49.linear_attn.in_proj_qkv.weight": {"dtype":"BF16","file":"model-00014-of-00018.safetensors","shape":[10240,5120]}, + "model.language_model.layers.49.linear_attn.in_proj_z.weight": {"dtype":"BF16","file":"model-00014-of-00018.safetensors","shape":[6144,5120]}, + "model.language_model.layers.49.linear_attn.norm.weight": {"dtype":"BF16","file":"model-00014-of-00018.safetensors","shape":[128]}, + "model.language_model.layers.49.linear_attn.out_proj.weight": {"dtype":"BF16","file":"model-00014-of-00018.safetensors","shape":[5120,6144]}, + "model.language_model.layers.49.mlp.down_proj.weight": {"dtype":"BF16","file":"model-00014-of-00018.safetensors","shape":[5120,17408]}, + "model.language_model.layers.49.mlp.gate_proj.weight": {"dtype":"BF16","file":"model-00014-of-00018.safetensors","shape":[17408,5120]}, + "model.language_model.layers.49.mlp.up_proj.weight": {"dtype":"BF16","file":"model-00014-of-00018.safetensors","shape":[17408,5120]}, + "model.language_model.layers.49.post_attention_layernorm.weight": {"dtype":"BF16","file":"model-00014-of-00018.safetensors","shape":[5120]}, + "model.language_model.layers.5.input_layernorm.weight": {"dtype":"BF16","file":"model-00002-of-00018.safetensors","shape":[5120]}, + "model.language_model.layers.5.linear_attn.A_log": {"dtype":"BF16","file":"model-00002-of-00018.safetensors","shape":[48]}, + "model.language_model.layers.5.linear_attn.conv1d.weight": {"dtype":"BF16","file":"model-00002-of-00018.safetensors","shape":[10240,1,4]}, + "model.language_model.layers.5.linear_attn.dt_bias": {"dtype":"BF16","file":"model-00002-of-00018.safetensors","shape":[48]}, + "model.language_model.layers.5.linear_attn.in_proj_a.weight": {"dtype":"BF16","file":"model-00002-of-00018.safetensors","shape":[48,5120]}, + "model.language_model.layers.5.linear_attn.in_proj_b.weight": {"dtype":"BF16","file":"model-00002-of-00018.safetensors","shape":[48,5120]}, + "model.language_model.layers.5.linear_attn.in_proj_qkv.weight": {"dtype":"BF16","file":"model-00002-of-00018.safetensors","shape":[10240,5120]}, + "model.language_model.layers.5.linear_attn.in_proj_z.weight": {"dtype":"BF16","file":"model-00002-of-00018.safetensors","shape":[6144,5120]}, + "model.language_model.layers.5.linear_attn.norm.weight": {"dtype":"BF16","file":"model-00002-of-00018.safetensors","shape":[128]}, + "model.language_model.layers.5.linear_attn.out_proj.weight": {"dtype":"BF16","file":"model-00002-of-00018.safetensors","shape":[5120,6144]}, + "model.language_model.layers.5.mlp.down_proj.weight": {"dtype":"BF16","file":"model-00002-of-00018.safetensors","shape":[5120,17408]}, + "model.language_model.layers.5.mlp.gate_proj.weight": {"dtype":"BF16","file":"model-00002-of-00018.safetensors","shape":[17408,5120]}, + "model.language_model.layers.5.mlp.up_proj.weight": {"dtype":"BF16","file":"model-00002-of-00018.safetensors","shape":[17408,5120]}, + "model.language_model.layers.5.post_attention_layernorm.weight": {"dtype":"BF16","file":"model-00002-of-00018.safetensors","shape":[5120]}, + "model.language_model.layers.50.input_layernorm.weight": {"dtype":"BF16","file":"model-00014-of-00018.safetensors","shape":[5120]}, + "model.language_model.layers.50.linear_attn.A_log": {"dtype":"BF16","file":"model-00014-of-00018.safetensors","shape":[48]}, + "model.language_model.layers.50.linear_attn.conv1d.weight": {"dtype":"BF16","file":"model-00014-of-00018.safetensors","shape":[10240,1,4]}, + "model.language_model.layers.50.linear_attn.dt_bias": {"dtype":"BF16","file":"model-00014-of-00018.safetensors","shape":[48]}, + "model.language_model.layers.50.linear_attn.in_proj_a.weight": {"dtype":"BF16","file":"model-00014-of-00018.safetensors","shape":[48,5120]}, + "model.language_model.layers.50.linear_attn.in_proj_b.weight": {"dtype":"BF16","file":"model-00014-of-00018.safetensors","shape":[48,5120]}, + "model.language_model.layers.50.linear_attn.in_proj_qkv.weight": {"dtype":"BF16","file":"model-00014-of-00018.safetensors","shape":[10240,5120]}, + "model.language_model.layers.50.linear_attn.in_proj_z.weight": {"dtype":"BF16","file":"model-00014-of-00018.safetensors","shape":[6144,5120]}, + "model.language_model.layers.50.linear_attn.norm.weight": {"dtype":"BF16","file":"model-00014-of-00018.safetensors","shape":[128]}, + "model.language_model.layers.50.linear_attn.out_proj.weight": {"dtype":"BF16","file":"model-00014-of-00018.safetensors","shape":[5120,6144]}, + "model.language_model.layers.50.mlp.down_proj.weight": {"dtype":"BF16","file":"model-00014-of-00018.safetensors","shape":[5120,17408]}, + "model.language_model.layers.50.mlp.gate_proj.weight": {"dtype":"BF16","file":"model-00014-of-00018.safetensors","shape":[17408,5120]}, + "model.language_model.layers.50.mlp.up_proj.weight": {"dtype":"BF16","file":"model-00014-of-00018.safetensors","shape":[17408,5120]}, + "model.language_model.layers.50.post_attention_layernorm.weight": {"dtype":"BF16","file":"model-00014-of-00018.safetensors","shape":[5120]}, + "model.language_model.layers.51.input_layernorm.weight": {"dtype":"BF16","file":"model-00014-of-00018.safetensors","shape":[5120]}, + "model.language_model.layers.51.mlp.down_proj.weight": {"dtype":"BF16","file":"model-00014-of-00018.safetensors","shape":[5120,17408]}, + "model.language_model.layers.51.mlp.gate_proj.weight": {"dtype":"BF16","file":"model-00014-of-00018.safetensors","shape":[17408,5120]}, + "model.language_model.layers.51.mlp.up_proj.weight": {"dtype":"BF16","file":"model-00014-of-00018.safetensors","shape":[17408,5120]}, + "model.language_model.layers.51.post_attention_layernorm.weight": {"dtype":"BF16","file":"model-00014-of-00018.safetensors","shape":[5120]}, + "model.language_model.layers.51.self_attn.k_norm.weight": {"dtype":"BF16","file":"model-00014-of-00018.safetensors","shape":[256]}, + "model.language_model.layers.51.self_attn.k_proj.weight": {"dtype":"BF16","file":"model-00014-of-00018.safetensors","shape":[1024,5120]}, + "model.language_model.layers.51.self_attn.o_proj.weight": {"dtype":"BF16","file":"model-00014-of-00018.safetensors","shape":[5120,6144]}, + "model.language_model.layers.51.self_attn.q_norm.weight": {"dtype":"BF16","file":"model-00014-of-00018.safetensors","shape":[256]}, + "model.language_model.layers.51.self_attn.q_proj.weight": {"dtype":"BF16","file":"model-00014-of-00018.safetensors","shape":[12288,5120]}, + "model.language_model.layers.51.self_attn.v_proj.weight": {"dtype":"BF16","file":"model-00014-of-00018.safetensors","shape":[1024,5120]}, + "model.language_model.layers.52.input_layernorm.weight": {"dtype":"BF16","file":"model-00014-of-00018.safetensors","shape":[5120]}, + "model.language_model.layers.52.linear_attn.A_log": {"dtype":"BF16","file":"model-00014-of-00018.safetensors","shape":[48]}, + "model.language_model.layers.52.linear_attn.conv1d.weight": {"dtype":"BF16","file":"model-00014-of-00018.safetensors","shape":[10240,1,4]}, + "model.language_model.layers.52.linear_attn.dt_bias": {"dtype":"BF16","file":"model-00014-of-00018.safetensors","shape":[48]}, + "model.language_model.layers.52.linear_attn.in_proj_a.weight": {"dtype":"BF16","file":"model-00014-of-00018.safetensors","shape":[48,5120]}, + "model.language_model.layers.52.linear_attn.in_proj_b.weight": {"dtype":"BF16","file":"model-00014-of-00018.safetensors","shape":[48,5120]}, + "model.language_model.layers.52.linear_attn.in_proj_qkv.weight": {"dtype":"BF16","file":"model-00014-of-00018.safetensors","shape":[10240,5120]}, + "model.language_model.layers.52.linear_attn.in_proj_z.weight": {"dtype":"BF16","file":"model-00014-of-00018.safetensors","shape":[6144,5120]}, + "model.language_model.layers.52.linear_attn.norm.weight": {"dtype":"BF16","file":"model-00014-of-00018.safetensors","shape":[128]}, + "model.language_model.layers.52.linear_attn.out_proj.weight": {"dtype":"BF16","file":"model-00014-of-00018.safetensors","shape":[5120,6144]}, + "model.language_model.layers.52.mlp.down_proj.weight": {"dtype":"BF16","file":"model-00014-of-00018.safetensors","shape":[5120,17408]}, + "model.language_model.layers.52.mlp.gate_proj.weight": {"dtype":"BF16","file":"model-00014-of-00018.safetensors","shape":[17408,5120]}, + "model.language_model.layers.52.mlp.up_proj.weight": {"dtype":"BF16","file":"model-00014-of-00018.safetensors","shape":[17408,5120]}, + "model.language_model.layers.52.post_attention_layernorm.weight": {"dtype":"BF16","file":"model-00014-of-00018.safetensors","shape":[5120]}, + "model.language_model.layers.53.input_layernorm.weight": {"dtype":"BF16","file":"model-00014-of-00018.safetensors","shape":[5120]}, + "model.language_model.layers.53.linear_attn.A_log": {"dtype":"BF16","file":"model-00014-of-00018.safetensors","shape":[48]}, + "model.language_model.layers.53.linear_attn.conv1d.weight": {"dtype":"BF16","file":"model-00014-of-00018.safetensors","shape":[10240,1,4]}, + "model.language_model.layers.53.linear_attn.dt_bias": {"dtype":"BF16","file":"model-00014-of-00018.safetensors","shape":[48]}, + "model.language_model.layers.53.linear_attn.in_proj_a.weight": {"dtype":"BF16","file":"model-00014-of-00018.safetensors","shape":[48,5120]}, + "model.language_model.layers.53.linear_attn.in_proj_b.weight": {"dtype":"BF16","file":"model-00014-of-00018.safetensors","shape":[48,5120]}, + "model.language_model.layers.53.linear_attn.in_proj_qkv.weight": {"dtype":"BF16","file":"model-00014-of-00018.safetensors","shape":[10240,5120]}, + "model.language_model.layers.53.linear_attn.in_proj_z.weight": {"dtype":"BF16","file":"model-00014-of-00018.safetensors","shape":[6144,5120]}, + "model.language_model.layers.53.linear_attn.norm.weight": {"dtype":"BF16","file":"model-00014-of-00018.safetensors","shape":[128]}, + "model.language_model.layers.53.linear_attn.out_proj.weight": {"dtype":"BF16","file":"model-00015-of-00018.safetensors","shape":[5120,6144]}, + "model.language_model.layers.53.mlp.down_proj.weight": {"dtype":"BF16","file":"model-00015-of-00018.safetensors","shape":[5120,17408]}, + "model.language_model.layers.53.mlp.gate_proj.weight": {"dtype":"BF16","file":"model-00015-of-00018.safetensors","shape":[17408,5120]}, + "model.language_model.layers.53.mlp.up_proj.weight": {"dtype":"BF16","file":"model-00015-of-00018.safetensors","shape":[17408,5120]}, + "model.language_model.layers.53.post_attention_layernorm.weight": {"dtype":"BF16","file":"model-00015-of-00018.safetensors","shape":[5120]}, + "model.language_model.layers.54.input_layernorm.weight": {"dtype":"BF16","file":"model-00015-of-00018.safetensors","shape":[5120]}, + "model.language_model.layers.54.linear_attn.A_log": {"dtype":"BF16","file":"model-00015-of-00018.safetensors","shape":[48]}, + "model.language_model.layers.54.linear_attn.conv1d.weight": {"dtype":"BF16","file":"model-00015-of-00018.safetensors","shape":[10240,1,4]}, + "model.language_model.layers.54.linear_attn.dt_bias": {"dtype":"BF16","file":"model-00015-of-00018.safetensors","shape":[48]}, + "model.language_model.layers.54.linear_attn.in_proj_a.weight": {"dtype":"BF16","file":"model-00015-of-00018.safetensors","shape":[48,5120]}, + "model.language_model.layers.54.linear_attn.in_proj_b.weight": {"dtype":"BF16","file":"model-00015-of-00018.safetensors","shape":[48,5120]}, + "model.language_model.layers.54.linear_attn.in_proj_qkv.weight": {"dtype":"BF16","file":"model-00015-of-00018.safetensors","shape":[10240,5120]}, + "model.language_model.layers.54.linear_attn.in_proj_z.weight": {"dtype":"BF16","file":"model-00015-of-00018.safetensors","shape":[6144,5120]}, + "model.language_model.layers.54.linear_attn.norm.weight": {"dtype":"BF16","file":"model-00015-of-00018.safetensors","shape":[128]}, + "model.language_model.layers.54.linear_attn.out_proj.weight": {"dtype":"BF16","file":"model-00015-of-00018.safetensors","shape":[5120,6144]}, + "model.language_model.layers.54.mlp.down_proj.weight": {"dtype":"BF16","file":"model-00015-of-00018.safetensors","shape":[5120,17408]}, + "model.language_model.layers.54.mlp.gate_proj.weight": {"dtype":"BF16","file":"model-00015-of-00018.safetensors","shape":[17408,5120]}, + "model.language_model.layers.54.mlp.up_proj.weight": {"dtype":"BF16","file":"model-00015-of-00018.safetensors","shape":[17408,5120]}, + "model.language_model.layers.54.post_attention_layernorm.weight": {"dtype":"BF16","file":"model-00015-of-00018.safetensors","shape":[5120]}, + "model.language_model.layers.55.input_layernorm.weight": {"dtype":"BF16","file":"model-00015-of-00018.safetensors","shape":[5120]}, + "model.language_model.layers.55.mlp.down_proj.weight": {"dtype":"BF16","file":"model-00015-of-00018.safetensors","shape":[5120,17408]}, + "model.language_model.layers.55.mlp.gate_proj.weight": {"dtype":"BF16","file":"model-00015-of-00018.safetensors","shape":[17408,5120]}, + "model.language_model.layers.55.mlp.up_proj.weight": {"dtype":"BF16","file":"model-00015-of-00018.safetensors","shape":[17408,5120]}, + "model.language_model.layers.55.post_attention_layernorm.weight": {"dtype":"BF16","file":"model-00015-of-00018.safetensors","shape":[5120]}, + "model.language_model.layers.55.self_attn.k_norm.weight": {"dtype":"BF16","file":"model-00015-of-00018.safetensors","shape":[256]}, + "model.language_model.layers.55.self_attn.k_proj.weight": {"dtype":"BF16","file":"model-00015-of-00018.safetensors","shape":[1024,5120]}, + "model.language_model.layers.55.self_attn.o_proj.weight": {"dtype":"BF16","file":"model-00015-of-00018.safetensors","shape":[5120,6144]}, + "model.language_model.layers.55.self_attn.q_norm.weight": {"dtype":"BF16","file":"model-00015-of-00018.safetensors","shape":[256]}, + "model.language_model.layers.55.self_attn.q_proj.weight": {"dtype":"BF16","file":"model-00015-of-00018.safetensors","shape":[12288,5120]}, + "model.language_model.layers.55.self_attn.v_proj.weight": {"dtype":"BF16","file":"model-00015-of-00018.safetensors","shape":[1024,5120]}, + "model.language_model.layers.56.input_layernorm.weight": {"dtype":"BF16","file":"model-00016-of-00018.safetensors","shape":[5120]}, + "model.language_model.layers.56.linear_attn.A_log": {"dtype":"BF16","file":"model-00016-of-00018.safetensors","shape":[48]}, + "model.language_model.layers.56.linear_attn.conv1d.weight": {"dtype":"BF16","file":"model-00016-of-00018.safetensors","shape":[10240,1,4]}, + "model.language_model.layers.56.linear_attn.dt_bias": {"dtype":"BF16","file":"model-00016-of-00018.safetensors","shape":[48]}, + "model.language_model.layers.56.linear_attn.in_proj_a.weight": {"dtype":"BF16","file":"model-00016-of-00018.safetensors","shape":[48,5120]}, + "model.language_model.layers.56.linear_attn.in_proj_b.weight": {"dtype":"BF16","file":"model-00016-of-00018.safetensors","shape":[48,5120]}, + "model.language_model.layers.56.linear_attn.in_proj_qkv.weight": {"dtype":"BF16","file":"model-00016-of-00018.safetensors","shape":[10240,5120]}, + "model.language_model.layers.56.linear_attn.in_proj_z.weight": {"dtype":"BF16","file":"model-00016-of-00018.safetensors","shape":[6144,5120]}, + "model.language_model.layers.56.linear_attn.norm.weight": {"dtype":"BF16","file":"model-00016-of-00018.safetensors","shape":[128]}, + "model.language_model.layers.56.linear_attn.out_proj.weight": {"dtype":"BF16","file":"model-00016-of-00018.safetensors","shape":[5120,6144]}, + "model.language_model.layers.56.mlp.down_proj.weight": {"dtype":"BF16","file":"model-00016-of-00018.safetensors","shape":[5120,17408]}, + "model.language_model.layers.56.mlp.gate_proj.weight": {"dtype":"BF16","file":"model-00016-of-00018.safetensors","shape":[17408,5120]}, + "model.language_model.layers.56.mlp.up_proj.weight": {"dtype":"BF16","file":"model-00016-of-00018.safetensors","shape":[17408,5120]}, + "model.language_model.layers.56.post_attention_layernorm.weight": {"dtype":"BF16","file":"model-00016-of-00018.safetensors","shape":[5120]}, + "model.language_model.layers.57.input_layernorm.weight": {"dtype":"BF16","file":"model-00016-of-00018.safetensors","shape":[5120]}, + "model.language_model.layers.57.linear_attn.A_log": {"dtype":"BF16","file":"model-00016-of-00018.safetensors","shape":[48]}, + "model.language_model.layers.57.linear_attn.conv1d.weight": {"dtype":"BF16","file":"model-00016-of-00018.safetensors","shape":[10240,1,4]}, + "model.language_model.layers.57.linear_attn.dt_bias": {"dtype":"BF16","file":"model-00016-of-00018.safetensors","shape":[48]}, + "model.language_model.layers.57.linear_attn.in_proj_a.weight": {"dtype":"BF16","file":"model-00016-of-00018.safetensors","shape":[48,5120]}, + "model.language_model.layers.57.linear_attn.in_proj_b.weight": {"dtype":"BF16","file":"model-00016-of-00018.safetensors","shape":[48,5120]}, + "model.language_model.layers.57.linear_attn.in_proj_qkv.weight": {"dtype":"BF16","file":"model-00016-of-00018.safetensors","shape":[10240,5120]}, + "model.language_model.layers.57.linear_attn.in_proj_z.weight": {"dtype":"BF16","file":"model-00016-of-00018.safetensors","shape":[6144,5120]}, + "model.language_model.layers.57.linear_attn.norm.weight": {"dtype":"BF16","file":"model-00016-of-00018.safetensors","shape":[128]}, + "model.language_model.layers.57.linear_attn.out_proj.weight": {"dtype":"BF16","file":"model-00016-of-00018.safetensors","shape":[5120,6144]}, + "model.language_model.layers.57.mlp.down_proj.weight": {"dtype":"BF16","file":"model-00016-of-00018.safetensors","shape":[5120,17408]}, + "model.language_model.layers.57.mlp.gate_proj.weight": {"dtype":"BF16","file":"model-00016-of-00018.safetensors","shape":[17408,5120]}, + "model.language_model.layers.57.mlp.up_proj.weight": {"dtype":"BF16","file":"model-00016-of-00018.safetensors","shape":[17408,5120]}, + "model.language_model.layers.57.post_attention_layernorm.weight": {"dtype":"BF16","file":"model-00016-of-00018.safetensors","shape":[5120]}, + "model.language_model.layers.58.input_layernorm.weight": {"dtype":"BF16","file":"model-00016-of-00018.safetensors","shape":[5120]}, + "model.language_model.layers.58.linear_attn.A_log": {"dtype":"BF16","file":"model-00016-of-00018.safetensors","shape":[48]}, + "model.language_model.layers.58.linear_attn.conv1d.weight": {"dtype":"BF16","file":"model-00016-of-00018.safetensors","shape":[10240,1,4]}, + "model.language_model.layers.58.linear_attn.dt_bias": {"dtype":"BF16","file":"model-00016-of-00018.safetensors","shape":[48]}, + "model.language_model.layers.58.linear_attn.in_proj_a.weight": {"dtype":"BF16","file":"model-00016-of-00018.safetensors","shape":[48,5120]}, + "model.language_model.layers.58.linear_attn.in_proj_b.weight": {"dtype":"BF16","file":"model-00016-of-00018.safetensors","shape":[48,5120]}, + "model.language_model.layers.58.linear_attn.in_proj_qkv.weight": {"dtype":"BF16","file":"model-00016-of-00018.safetensors","shape":[10240,5120]}, + "model.language_model.layers.58.linear_attn.in_proj_z.weight": {"dtype":"BF16","file":"model-00016-of-00018.safetensors","shape":[6144,5120]}, + "model.language_model.layers.58.linear_attn.norm.weight": {"dtype":"BF16","file":"model-00016-of-00018.safetensors","shape":[128]}, + "model.language_model.layers.58.linear_attn.out_proj.weight": {"dtype":"BF16","file":"model-00016-of-00018.safetensors","shape":[5120,6144]}, + "model.language_model.layers.58.mlp.down_proj.weight": {"dtype":"BF16","file":"model-00016-of-00018.safetensors","shape":[5120,17408]}, + "model.language_model.layers.58.mlp.gate_proj.weight": {"dtype":"BF16","file":"model-00016-of-00018.safetensors","shape":[17408,5120]}, + "model.language_model.layers.58.mlp.up_proj.weight": {"dtype":"BF16","file":"model-00016-of-00018.safetensors","shape":[17408,5120]}, + "model.language_model.layers.58.post_attention_layernorm.weight": {"dtype":"BF16","file":"model-00016-of-00018.safetensors","shape":[5120]}, + "model.language_model.layers.59.input_layernorm.weight": {"dtype":"BF16","file":"model-00016-of-00018.safetensors","shape":[5120]}, + "model.language_model.layers.59.mlp.down_proj.weight": {"dtype":"BF16","file":"model-00016-of-00018.safetensors","shape":[5120,17408]}, + "model.language_model.layers.59.mlp.gate_proj.weight": {"dtype":"BF16","file":"model-00016-of-00018.safetensors","shape":[17408,5120]}, + "model.language_model.layers.59.mlp.up_proj.weight": {"dtype":"BF16","file":"model-00016-of-00018.safetensors","shape":[17408,5120]}, + "model.language_model.layers.59.post_attention_layernorm.weight": {"dtype":"BF16","file":"model-00016-of-00018.safetensors","shape":[5120]}, + "model.language_model.layers.59.self_attn.k_norm.weight": {"dtype":"BF16","file":"model-00016-of-00018.safetensors","shape":[256]}, + "model.language_model.layers.59.self_attn.k_proj.weight": {"dtype":"BF16","file":"model-00016-of-00018.safetensors","shape":[1024,5120]}, + "model.language_model.layers.59.self_attn.o_proj.weight": {"dtype":"BF16","file":"model-00016-of-00018.safetensors","shape":[5120,6144]}, + "model.language_model.layers.59.self_attn.q_norm.weight": {"dtype":"BF16","file":"model-00016-of-00018.safetensors","shape":[256]}, + "model.language_model.layers.59.self_attn.q_proj.weight": {"dtype":"BF16","file":"model-00016-of-00018.safetensors","shape":[12288,5120]}, + "model.language_model.layers.59.self_attn.v_proj.weight": {"dtype":"BF16","file":"model-00016-of-00018.safetensors","shape":[1024,5120]}, + "model.language_model.layers.6.input_layernorm.weight": {"dtype":"BF16","file":"model-00002-of-00018.safetensors","shape":[5120]}, + "model.language_model.layers.6.linear_attn.A_log": {"dtype":"BF16","file":"model-00002-of-00018.safetensors","shape":[48]}, + "model.language_model.layers.6.linear_attn.conv1d.weight": {"dtype":"BF16","file":"model-00002-of-00018.safetensors","shape":[10240,1,4]}, + "model.language_model.layers.6.linear_attn.dt_bias": {"dtype":"BF16","file":"model-00002-of-00018.safetensors","shape":[48]}, + "model.language_model.layers.6.linear_attn.in_proj_a.weight": {"dtype":"BF16","file":"model-00002-of-00018.safetensors","shape":[48,5120]}, + "model.language_model.layers.6.linear_attn.in_proj_b.weight": {"dtype":"BF16","file":"model-00002-of-00018.safetensors","shape":[48,5120]}, + "model.language_model.layers.6.linear_attn.in_proj_qkv.weight": {"dtype":"BF16","file":"model-00002-of-00018.safetensors","shape":[10240,5120]}, + "model.language_model.layers.6.linear_attn.in_proj_z.weight": {"dtype":"BF16","file":"model-00002-of-00018.safetensors","shape":[6144,5120]}, + "model.language_model.layers.6.linear_attn.norm.weight": {"dtype":"BF16","file":"model-00002-of-00018.safetensors","shape":[128]}, + "model.language_model.layers.6.linear_attn.out_proj.weight": {"dtype":"BF16","file":"model-00002-of-00018.safetensors","shape":[5120,6144]}, + "model.language_model.layers.6.mlp.down_proj.weight": {"dtype":"BF16","file":"model-00002-of-00018.safetensors","shape":[5120,17408]}, + "model.language_model.layers.6.mlp.gate_proj.weight": {"dtype":"BF16","file":"model-00002-of-00018.safetensors","shape":[17408,5120]}, + "model.language_model.layers.6.mlp.up_proj.weight": {"dtype":"BF16","file":"model-00002-of-00018.safetensors","shape":[17408,5120]}, + "model.language_model.layers.6.post_attention_layernorm.weight": {"dtype":"BF16","file":"model-00002-of-00018.safetensors","shape":[5120]}, + "model.language_model.layers.60.input_layernorm.weight": {"dtype":"BF16","file":"model-00016-of-00018.safetensors","shape":[5120]}, + "model.language_model.layers.60.linear_attn.A_log": {"dtype":"BF16","file":"model-00016-of-00018.safetensors","shape":[48]}, + "model.language_model.layers.60.linear_attn.conv1d.weight": {"dtype":"BF16","file":"model-00016-of-00018.safetensors","shape":[10240,1,4]}, + "model.language_model.layers.60.linear_attn.dt_bias": {"dtype":"BF16","file":"model-00016-of-00018.safetensors","shape":[48]}, + "model.language_model.layers.60.linear_attn.in_proj_a.weight": {"dtype":"BF16","file":"model-00016-of-00018.safetensors","shape":[48,5120]}, + "model.language_model.layers.60.linear_attn.in_proj_b.weight": {"dtype":"BF16","file":"model-00016-of-00018.safetensors","shape":[48,5120]}, + "model.language_model.layers.60.linear_attn.in_proj_qkv.weight": {"dtype":"BF16","file":"model-00016-of-00018.safetensors","shape":[10240,5120]}, + "model.language_model.layers.60.linear_attn.in_proj_z.weight": {"dtype":"BF16","file":"model-00016-of-00018.safetensors","shape":[6144,5120]}, + "model.language_model.layers.60.linear_attn.norm.weight": {"dtype":"BF16","file":"model-00016-of-00018.safetensors","shape":[128]}, + "model.language_model.layers.60.linear_attn.out_proj.weight": {"dtype":"BF16","file":"model-00016-of-00018.safetensors","shape":[5120,6144]}, + "model.language_model.layers.60.mlp.down_proj.weight": {"dtype":"BF16","file":"model-00016-of-00018.safetensors","shape":[5120,17408]}, + "model.language_model.layers.60.mlp.gate_proj.weight": {"dtype":"BF16","file":"model-00016-of-00018.safetensors","shape":[17408,5120]}, + "model.language_model.layers.60.mlp.up_proj.weight": {"dtype":"BF16","file":"model-00016-of-00018.safetensors","shape":[17408,5120]}, + "model.language_model.layers.60.post_attention_layernorm.weight": {"dtype":"BF16","file":"model-00016-of-00018.safetensors","shape":[5120]}, + "model.language_model.layers.61.input_layernorm.weight": {"dtype":"BF16","file":"model-00016-of-00018.safetensors","shape":[5120]}, + "model.language_model.layers.61.linear_attn.A_log": {"dtype":"BF16","file":"model-00016-of-00018.safetensors","shape":[48]}, + "model.language_model.layers.61.linear_attn.conv1d.weight": {"dtype":"BF16","file":"model-00016-of-00018.safetensors","shape":[10240,1,4]}, + "model.language_model.layers.61.linear_attn.dt_bias": {"dtype":"BF16","file":"model-00016-of-00018.safetensors","shape":[48]}, + "model.language_model.layers.61.linear_attn.in_proj_a.weight": {"dtype":"BF16","file":"model-00016-of-00018.safetensors","shape":[48,5120]}, + "model.language_model.layers.61.linear_attn.in_proj_b.weight": {"dtype":"BF16","file":"model-00016-of-00018.safetensors","shape":[48,5120]}, + "model.language_model.layers.61.linear_attn.in_proj_qkv.weight": {"dtype":"BF16","file":"model-00016-of-00018.safetensors","shape":[10240,5120]}, + "model.language_model.layers.61.linear_attn.in_proj_z.weight": {"dtype":"BF16","file":"model-00016-of-00018.safetensors","shape":[6144,5120]}, + "model.language_model.layers.61.linear_attn.norm.weight": {"dtype":"BF16","file":"model-00016-of-00018.safetensors","shape":[128]}, + "model.language_model.layers.61.linear_attn.out_proj.weight": {"dtype":"BF16","file":"model-00017-of-00018.safetensors","shape":[5120,6144]}, + "model.language_model.layers.61.mlp.down_proj.weight": {"dtype":"BF16","file":"model-00017-of-00018.safetensors","shape":[5120,17408]}, + "model.language_model.layers.61.mlp.gate_proj.weight": {"dtype":"BF16","file":"model-00017-of-00018.safetensors","shape":[17408,5120]}, + "model.language_model.layers.61.mlp.up_proj.weight": {"dtype":"BF16","file":"model-00017-of-00018.safetensors","shape":[17408,5120]}, + "model.language_model.layers.61.post_attention_layernorm.weight": {"dtype":"BF16","file":"model-00017-of-00018.safetensors","shape":[5120]}, + "model.language_model.layers.62.input_layernorm.weight": {"dtype":"BF16","file":"model-00017-of-00018.safetensors","shape":[5120]}, + "model.language_model.layers.62.linear_attn.A_log": {"dtype":"BF16","file":"model-00017-of-00018.safetensors","shape":[48]}, + "model.language_model.layers.62.linear_attn.conv1d.weight": {"dtype":"BF16","file":"model-00017-of-00018.safetensors","shape":[10240,1,4]}, + "model.language_model.layers.62.linear_attn.dt_bias": {"dtype":"BF16","file":"model-00017-of-00018.safetensors","shape":[48]}, + "model.language_model.layers.62.linear_attn.in_proj_a.weight": {"dtype":"BF16","file":"model-00017-of-00018.safetensors","shape":[48,5120]}, + "model.language_model.layers.62.linear_attn.in_proj_b.weight": {"dtype":"BF16","file":"model-00017-of-00018.safetensors","shape":[48,5120]}, + "model.language_model.layers.62.linear_attn.in_proj_qkv.weight": {"dtype":"BF16","file":"model-00017-of-00018.safetensors","shape":[10240,5120]}, + "model.language_model.layers.62.linear_attn.in_proj_z.weight": {"dtype":"BF16","file":"model-00017-of-00018.safetensors","shape":[6144,5120]}, + "model.language_model.layers.62.linear_attn.norm.weight": {"dtype":"BF16","file":"model-00017-of-00018.safetensors","shape":[128]}, + "model.language_model.layers.62.linear_attn.out_proj.weight": {"dtype":"BF16","file":"model-00017-of-00018.safetensors","shape":[5120,6144]}, + "model.language_model.layers.62.mlp.down_proj.weight": {"dtype":"BF16","file":"model-00017-of-00018.safetensors","shape":[5120,17408]}, + "model.language_model.layers.62.mlp.gate_proj.weight": {"dtype":"BF16","file":"model-00017-of-00018.safetensors","shape":[17408,5120]}, + "model.language_model.layers.62.mlp.up_proj.weight": {"dtype":"BF16","file":"model-00017-of-00018.safetensors","shape":[17408,5120]}, + "model.language_model.layers.62.post_attention_layernorm.weight": {"dtype":"BF16","file":"model-00017-of-00018.safetensors","shape":[5120]}, + "model.language_model.layers.63.input_layernorm.weight": {"dtype":"BF16","file":"model-00017-of-00018.safetensors","shape":[5120]}, + "model.language_model.layers.63.mlp.down_proj.weight": {"dtype":"BF16","file":"model-00017-of-00018.safetensors","shape":[5120,17408]}, + "model.language_model.layers.63.mlp.gate_proj.weight": {"dtype":"BF16","file":"model-00017-of-00018.safetensors","shape":[17408,5120]}, + "model.language_model.layers.63.mlp.up_proj.weight": {"dtype":"BF16","file":"model-00017-of-00018.safetensors","shape":[17408,5120]}, + "model.language_model.layers.63.post_attention_layernorm.weight": {"dtype":"BF16","file":"model-00017-of-00018.safetensors","shape":[5120]}, + "model.language_model.layers.63.self_attn.k_norm.weight": {"dtype":"BF16","file":"model-00017-of-00018.safetensors","shape":[256]}, + "model.language_model.layers.63.self_attn.k_proj.weight": {"dtype":"BF16","file":"model-00017-of-00018.safetensors","shape":[1024,5120]}, + "model.language_model.layers.63.self_attn.o_proj.weight": {"dtype":"BF16","file":"model-00017-of-00018.safetensors","shape":[5120,6144]}, + "model.language_model.layers.63.self_attn.q_norm.weight": {"dtype":"BF16","file":"model-00017-of-00018.safetensors","shape":[256]}, + "model.language_model.layers.63.self_attn.q_proj.weight": {"dtype":"BF16","file":"model-00017-of-00018.safetensors","shape":[12288,5120]}, + "model.language_model.layers.63.self_attn.v_proj.weight": {"dtype":"BF16","file":"model-00017-of-00018.safetensors","shape":[1024,5120]}, + "model.language_model.layers.7.input_layernorm.weight": {"dtype":"BF16","file":"model-00002-of-00018.safetensors","shape":[5120]}, + "model.language_model.layers.7.mlp.down_proj.weight": {"dtype":"BF16","file":"model-00002-of-00018.safetensors","shape":[5120,17408]}, + "model.language_model.layers.7.mlp.gate_proj.weight": {"dtype":"BF16","file":"model-00002-of-00018.safetensors","shape":[17408,5120]}, + "model.language_model.layers.7.mlp.up_proj.weight": {"dtype":"BF16","file":"model-00002-of-00018.safetensors","shape":[17408,5120]}, + "model.language_model.layers.7.post_attention_layernorm.weight": {"dtype":"BF16","file":"model-00002-of-00018.safetensors","shape":[5120]}, + "model.language_model.layers.7.self_attn.k_norm.weight": {"dtype":"BF16","file":"model-00002-of-00018.safetensors","shape":[256]}, + "model.language_model.layers.7.self_attn.k_proj.weight": {"dtype":"BF16","file":"model-00002-of-00018.safetensors","shape":[1024,5120]}, + "model.language_model.layers.7.self_attn.o_proj.weight": {"dtype":"BF16","file":"model-00002-of-00018.safetensors","shape":[5120,6144]}, + "model.language_model.layers.7.self_attn.q_norm.weight": {"dtype":"BF16","file":"model-00002-of-00018.safetensors","shape":[256]}, + "model.language_model.layers.7.self_attn.q_proj.weight": {"dtype":"BF16","file":"model-00002-of-00018.safetensors","shape":[12288,5120]}, + "model.language_model.layers.7.self_attn.v_proj.weight": {"dtype":"BF16","file":"model-00002-of-00018.safetensors","shape":[1024,5120]}, + "model.language_model.layers.8.input_layernorm.weight": {"dtype":"BF16","file":"model-00005-of-00018.safetensors","shape":[5120]}, + "model.language_model.layers.8.linear_attn.A_log": {"dtype":"BF16","file":"model-00005-of-00018.safetensors","shape":[48]}, + "model.language_model.layers.8.linear_attn.conv1d.weight": {"dtype":"BF16","file":"model-00005-of-00018.safetensors","shape":[10240,1,4]}, + "model.language_model.layers.8.linear_attn.dt_bias": {"dtype":"BF16","file":"model-00005-of-00018.safetensors","shape":[48]}, + "model.language_model.layers.8.linear_attn.in_proj_a.weight": {"dtype":"BF16","file":"model-00005-of-00018.safetensors","shape":[48,5120]}, + "model.language_model.layers.8.linear_attn.in_proj_b.weight": {"dtype":"BF16","file":"model-00005-of-00018.safetensors","shape":[48,5120]}, + "model.language_model.layers.8.linear_attn.in_proj_qkv.weight": {"dtype":"BF16","file":"model-00005-of-00018.safetensors","shape":[10240,5120]}, + "model.language_model.layers.8.linear_attn.in_proj_z.weight": {"dtype":"BF16","file":"model-00005-of-00018.safetensors","shape":[6144,5120]}, + "model.language_model.layers.8.linear_attn.norm.weight": {"dtype":"BF16","file":"model-00005-of-00018.safetensors","shape":[128]}, + "model.language_model.layers.8.linear_attn.out_proj.weight": {"dtype":"BF16","file":"model-00005-of-00018.safetensors","shape":[5120,6144]}, + "model.language_model.layers.8.mlp.down_proj.weight": {"dtype":"BF16","file":"model-00005-of-00018.safetensors","shape":[5120,17408]}, + "model.language_model.layers.8.mlp.gate_proj.weight": {"dtype":"BF16","file":"model-00005-of-00018.safetensors","shape":[17408,5120]}, + "model.language_model.layers.8.mlp.up_proj.weight": {"dtype":"BF16","file":"model-00005-of-00018.safetensors","shape":[17408,5120]}, + "model.language_model.layers.8.post_attention_layernorm.weight": {"dtype":"BF16","file":"model-00005-of-00018.safetensors","shape":[5120]}, + "model.language_model.layers.9.input_layernorm.weight": {"dtype":"BF16","file":"model-00005-of-00018.safetensors","shape":[5120]}, + "model.language_model.layers.9.linear_attn.A_log": {"dtype":"BF16","file":"model-00005-of-00018.safetensors","shape":[48]}, + "model.language_model.layers.9.linear_attn.conv1d.weight": {"dtype":"BF16","file":"model-00005-of-00018.safetensors","shape":[10240,1,4]}, + "model.language_model.layers.9.linear_attn.dt_bias": {"dtype":"BF16","file":"model-00005-of-00018.safetensors","shape":[48]}, + "model.language_model.layers.9.linear_attn.in_proj_a.weight": {"dtype":"BF16","file":"model-00005-of-00018.safetensors","shape":[48,5120]}, + "model.language_model.layers.9.linear_attn.in_proj_b.weight": {"dtype":"BF16","file":"model-00005-of-00018.safetensors","shape":[48,5120]}, + "model.language_model.layers.9.linear_attn.in_proj_qkv.weight": {"dtype":"BF16","file":"model-00005-of-00018.safetensors","shape":[10240,5120]}, + "model.language_model.layers.9.linear_attn.in_proj_z.weight": {"dtype":"BF16","file":"model-00005-of-00018.safetensors","shape":[6144,5120]}, + "model.language_model.layers.9.linear_attn.norm.weight": {"dtype":"BF16","file":"model-00005-of-00018.safetensors","shape":[128]}, + "model.language_model.layers.9.linear_attn.out_proj.weight": {"dtype":"BF16","file":"model-00005-of-00018.safetensors","shape":[5120,6144]}, + "model.language_model.layers.9.mlp.down_proj.weight": {"dtype":"BF16","file":"model-00005-of-00018.safetensors","shape":[5120,17408]}, + "model.language_model.layers.9.mlp.gate_proj.weight": {"dtype":"BF16","file":"model-00005-of-00018.safetensors","shape":[17408,5120]}, + "model.language_model.layers.9.mlp.up_proj.weight": {"dtype":"BF16","file":"model-00005-of-00018.safetensors","shape":[17408,5120]}, + "model.language_model.layers.9.post_attention_layernorm.weight": {"dtype":"BF16","file":"model-00005-of-00018.safetensors","shape":[5120]}, + "model.language_model.norm.weight": {"dtype":"BF16","file":"model-00016-of-00018.safetensors","shape":[5120]}, + "model.visual.blocks.0.attn.proj.bias": {"dtype":"BF16","file":"model-00001-of-00018.safetensors","shape":[1152]}, + "model.visual.blocks.0.attn.proj.weight": {"dtype":"BF16","file":"model-00001-of-00018.safetensors","shape":[1152,1152]}, + "model.visual.blocks.0.attn.qkv.bias": {"dtype":"BF16","file":"model-00001-of-00018.safetensors","shape":[3456]}, + "model.visual.blocks.0.attn.qkv.weight": {"dtype":"BF16","file":"model-00001-of-00018.safetensors","shape":[3456,1152]}, + "model.visual.blocks.0.mlp.linear_fc1.bias": {"dtype":"BF16","file":"model-00001-of-00018.safetensors","shape":[4304]}, + "model.visual.blocks.0.mlp.linear_fc1.weight": {"dtype":"BF16","file":"model-00001-of-00018.safetensors","shape":[4304,1152]}, + "model.visual.blocks.0.mlp.linear_fc2.bias": {"dtype":"BF16","file":"model-00001-of-00018.safetensors","shape":[1152]}, + "model.visual.blocks.0.mlp.linear_fc2.weight": {"dtype":"BF16","file":"model-00001-of-00018.safetensors","shape":[1152,4304]}, + "model.visual.blocks.0.norm1.bias": {"dtype":"BF16","file":"model-00001-of-00018.safetensors","shape":[1152]}, + "model.visual.blocks.0.norm1.weight": {"dtype":"BF16","file":"model-00001-of-00018.safetensors","shape":[1152]}, + "model.visual.blocks.0.norm2.bias": {"dtype":"BF16","file":"model-00001-of-00018.safetensors","shape":[1152]}, + "model.visual.blocks.0.norm2.weight": {"dtype":"BF16","file":"model-00001-of-00018.safetensors","shape":[1152]}, + "model.visual.blocks.1.attn.proj.bias": {"dtype":"BF16","file":"model-00001-of-00018.safetensors","shape":[1152]}, + "model.visual.blocks.1.attn.proj.weight": {"dtype":"BF16","file":"model-00001-of-00018.safetensors","shape":[1152,1152]}, + "model.visual.blocks.1.attn.qkv.bias": {"dtype":"BF16","file":"model-00001-of-00018.safetensors","shape":[3456]}, + "model.visual.blocks.1.attn.qkv.weight": {"dtype":"BF16","file":"model-00001-of-00018.safetensors","shape":[3456,1152]}, + "model.visual.blocks.1.mlp.linear_fc1.bias": {"dtype":"BF16","file":"model-00001-of-00018.safetensors","shape":[4304]}, + "model.visual.blocks.1.mlp.linear_fc1.weight": {"dtype":"BF16","file":"model-00001-of-00018.safetensors","shape":[4304,1152]}, + "model.visual.blocks.1.mlp.linear_fc2.bias": {"dtype":"BF16","file":"model-00001-of-00018.safetensors","shape":[1152]}, + "model.visual.blocks.1.mlp.linear_fc2.weight": {"dtype":"BF16","file":"model-00001-of-00018.safetensors","shape":[1152,4304]}, + "model.visual.blocks.1.norm1.bias": {"dtype":"BF16","file":"model-00001-of-00018.safetensors","shape":[1152]}, + "model.visual.blocks.1.norm1.weight": {"dtype":"BF16","file":"model-00001-of-00018.safetensors","shape":[1152]}, + "model.visual.blocks.1.norm2.bias": {"dtype":"BF16","file":"model-00001-of-00018.safetensors","shape":[1152]}, + "model.visual.blocks.1.norm2.weight": {"dtype":"BF16","file":"model-00001-of-00018.safetensors","shape":[1152]}, + "model.visual.blocks.10.attn.proj.bias": {"dtype":"BF16","file":"model-00001-of-00018.safetensors","shape":[1152]}, + "model.visual.blocks.10.attn.proj.weight": {"dtype":"BF16","file":"model-00001-of-00018.safetensors","shape":[1152,1152]}, + "model.visual.blocks.10.attn.qkv.bias": {"dtype":"BF16","file":"model-00001-of-00018.safetensors","shape":[3456]}, + "model.visual.blocks.10.attn.qkv.weight": {"dtype":"BF16","file":"model-00001-of-00018.safetensors","shape":[3456,1152]}, + "model.visual.blocks.10.mlp.linear_fc1.bias": {"dtype":"BF16","file":"model-00001-of-00018.safetensors","shape":[4304]}, + "model.visual.blocks.10.mlp.linear_fc1.weight": {"dtype":"BF16","file":"model-00001-of-00018.safetensors","shape":[4304,1152]}, + "model.visual.blocks.10.mlp.linear_fc2.bias": {"dtype":"BF16","file":"model-00001-of-00018.safetensors","shape":[1152]}, + "model.visual.blocks.10.mlp.linear_fc2.weight": {"dtype":"BF16","file":"model-00001-of-00018.safetensors","shape":[1152,4304]}, + "model.visual.blocks.10.norm1.bias": {"dtype":"BF16","file":"model-00001-of-00018.safetensors","shape":[1152]}, + "model.visual.blocks.10.norm1.weight": {"dtype":"BF16","file":"model-00001-of-00018.safetensors","shape":[1152]}, + "model.visual.blocks.10.norm2.bias": {"dtype":"BF16","file":"model-00001-of-00018.safetensors","shape":[1152]}, + "model.visual.blocks.10.norm2.weight": {"dtype":"BF16","file":"model-00001-of-00018.safetensors","shape":[1152]}, + "model.visual.blocks.11.attn.proj.bias": {"dtype":"BF16","file":"model-00001-of-00018.safetensors","shape":[1152]}, + "model.visual.blocks.11.attn.proj.weight": {"dtype":"BF16","file":"model-00001-of-00018.safetensors","shape":[1152,1152]}, + "model.visual.blocks.11.attn.qkv.bias": {"dtype":"BF16","file":"model-00001-of-00018.safetensors","shape":[3456]}, + "model.visual.blocks.11.attn.qkv.weight": {"dtype":"BF16","file":"model-00001-of-00018.safetensors","shape":[3456,1152]}, + "model.visual.blocks.11.mlp.linear_fc1.bias": {"dtype":"BF16","file":"model-00001-of-00018.safetensors","shape":[4304]}, + "model.visual.blocks.11.mlp.linear_fc1.weight": {"dtype":"BF16","file":"model-00001-of-00018.safetensors","shape":[4304,1152]}, + "model.visual.blocks.11.mlp.linear_fc2.bias": {"dtype":"BF16","file":"model-00001-of-00018.safetensors","shape":[1152]}, + "model.visual.blocks.11.mlp.linear_fc2.weight": {"dtype":"BF16","file":"model-00001-of-00018.safetensors","shape":[1152,4304]}, + "model.visual.blocks.11.norm1.bias": {"dtype":"BF16","file":"model-00001-of-00018.safetensors","shape":[1152]}, + "model.visual.blocks.11.norm1.weight": {"dtype":"BF16","file":"model-00001-of-00018.safetensors","shape":[1152]}, + "model.visual.blocks.11.norm2.bias": {"dtype":"BF16","file":"model-00001-of-00018.safetensors","shape":[1152]}, + "model.visual.blocks.11.norm2.weight": {"dtype":"BF16","file":"model-00001-of-00018.safetensors","shape":[1152]}, + "model.visual.blocks.12.attn.proj.bias": {"dtype":"BF16","file":"model-00001-of-00018.safetensors","shape":[1152]}, + "model.visual.blocks.12.attn.proj.weight": {"dtype":"BF16","file":"model-00001-of-00018.safetensors","shape":[1152,1152]}, + "model.visual.blocks.12.attn.qkv.bias": {"dtype":"BF16","file":"model-00001-of-00018.safetensors","shape":[3456]}, + "model.visual.blocks.12.attn.qkv.weight": {"dtype":"BF16","file":"model-00001-of-00018.safetensors","shape":[3456,1152]}, + "model.visual.blocks.12.mlp.linear_fc1.bias": {"dtype":"BF16","file":"model-00001-of-00018.safetensors","shape":[4304]}, + "model.visual.blocks.12.mlp.linear_fc1.weight": {"dtype":"BF16","file":"model-00001-of-00018.safetensors","shape":[4304,1152]}, + "model.visual.blocks.12.mlp.linear_fc2.bias": {"dtype":"BF16","file":"model-00001-of-00018.safetensors","shape":[1152]}, + "model.visual.blocks.12.mlp.linear_fc2.weight": {"dtype":"BF16","file":"model-00001-of-00018.safetensors","shape":[1152,4304]}, + "model.visual.blocks.12.norm1.bias": {"dtype":"BF16","file":"model-00001-of-00018.safetensors","shape":[1152]}, + "model.visual.blocks.12.norm1.weight": {"dtype":"BF16","file":"model-00001-of-00018.safetensors","shape":[1152]}, + "model.visual.blocks.12.norm2.bias": {"dtype":"BF16","file":"model-00001-of-00018.safetensors","shape":[1152]}, + "model.visual.blocks.12.norm2.weight": {"dtype":"BF16","file":"model-00001-of-00018.safetensors","shape":[1152]}, + "model.visual.blocks.13.attn.proj.bias": {"dtype":"BF16","file":"model-00001-of-00018.safetensors","shape":[1152]}, + "model.visual.blocks.13.attn.proj.weight": {"dtype":"BF16","file":"model-00001-of-00018.safetensors","shape":[1152,1152]}, + "model.visual.blocks.13.attn.qkv.bias": {"dtype":"BF16","file":"model-00001-of-00018.safetensors","shape":[3456]}, + "model.visual.blocks.13.attn.qkv.weight": {"dtype":"BF16","file":"model-00001-of-00018.safetensors","shape":[3456,1152]}, + "model.visual.blocks.13.mlp.linear_fc1.bias": {"dtype":"BF16","file":"model-00001-of-00018.safetensors","shape":[4304]}, + "model.visual.blocks.13.mlp.linear_fc1.weight": {"dtype":"BF16","file":"model-00001-of-00018.safetensors","shape":[4304,1152]}, + "model.visual.blocks.13.mlp.linear_fc2.bias": {"dtype":"BF16","file":"model-00001-of-00018.safetensors","shape":[1152]}, + "model.visual.blocks.13.mlp.linear_fc2.weight": {"dtype":"BF16","file":"model-00001-of-00018.safetensors","shape":[1152,4304]}, + "model.visual.blocks.13.norm1.bias": {"dtype":"BF16","file":"model-00001-of-00018.safetensors","shape":[1152]}, + "model.visual.blocks.13.norm1.weight": {"dtype":"BF16","file":"model-00001-of-00018.safetensors","shape":[1152]}, + "model.visual.blocks.13.norm2.bias": {"dtype":"BF16","file":"model-00001-of-00018.safetensors","shape":[1152]}, + "model.visual.blocks.13.norm2.weight": {"dtype":"BF16","file":"model-00001-of-00018.safetensors","shape":[1152]}, + "model.visual.blocks.14.attn.proj.bias": {"dtype":"BF16","file":"model-00001-of-00018.safetensors","shape":[1152]}, + "model.visual.blocks.14.attn.proj.weight": {"dtype":"BF16","file":"model-00001-of-00018.safetensors","shape":[1152,1152]}, + "model.visual.blocks.14.attn.qkv.bias": {"dtype":"BF16","file":"model-00001-of-00018.safetensors","shape":[3456]}, + "model.visual.blocks.14.attn.qkv.weight": {"dtype":"BF16","file":"model-00001-of-00018.safetensors","shape":[3456,1152]}, + "model.visual.blocks.14.mlp.linear_fc1.bias": {"dtype":"BF16","file":"model-00001-of-00018.safetensors","shape":[4304]}, + "model.visual.blocks.14.mlp.linear_fc1.weight": {"dtype":"BF16","file":"model-00001-of-00018.safetensors","shape":[4304,1152]}, + "model.visual.blocks.14.mlp.linear_fc2.bias": {"dtype":"BF16","file":"model-00001-of-00018.safetensors","shape":[1152]}, + "model.visual.blocks.14.mlp.linear_fc2.weight": {"dtype":"BF16","file":"model-00001-of-00018.safetensors","shape":[1152,4304]}, + "model.visual.blocks.14.norm1.bias": {"dtype":"BF16","file":"model-00001-of-00018.safetensors","shape":[1152]}, + "model.visual.blocks.14.norm1.weight": {"dtype":"BF16","file":"model-00001-of-00018.safetensors","shape":[1152]}, + "model.visual.blocks.14.norm2.bias": {"dtype":"BF16","file":"model-00001-of-00018.safetensors","shape":[1152]}, + "model.visual.blocks.14.norm2.weight": {"dtype":"BF16","file":"model-00001-of-00018.safetensors","shape":[1152]}, + "model.visual.blocks.15.attn.proj.bias": {"dtype":"BF16","file":"model-00001-of-00018.safetensors","shape":[1152]}, + "model.visual.blocks.15.attn.proj.weight": {"dtype":"BF16","file":"model-00001-of-00018.safetensors","shape":[1152,1152]}, + "model.visual.blocks.15.attn.qkv.bias": {"dtype":"BF16","file":"model-00001-of-00018.safetensors","shape":[3456]}, + "model.visual.blocks.15.attn.qkv.weight": {"dtype":"BF16","file":"model-00001-of-00018.safetensors","shape":[3456,1152]}, + "model.visual.blocks.15.mlp.linear_fc1.bias": {"dtype":"BF16","file":"model-00001-of-00018.safetensors","shape":[4304]}, + "model.visual.blocks.15.mlp.linear_fc1.weight": {"dtype":"BF16","file":"model-00001-of-00018.safetensors","shape":[4304,1152]}, + "model.visual.blocks.15.mlp.linear_fc2.bias": {"dtype":"BF16","file":"model-00001-of-00018.safetensors","shape":[1152]}, + "model.visual.blocks.15.mlp.linear_fc2.weight": {"dtype":"BF16","file":"model-00001-of-00018.safetensors","shape":[1152,4304]}, + "model.visual.blocks.15.norm1.bias": {"dtype":"BF16","file":"model-00001-of-00018.safetensors","shape":[1152]}, + "model.visual.blocks.15.norm1.weight": {"dtype":"BF16","file":"model-00001-of-00018.safetensors","shape":[1152]}, + "model.visual.blocks.15.norm2.bias": {"dtype":"BF16","file":"model-00001-of-00018.safetensors","shape":[1152]}, + "model.visual.blocks.15.norm2.weight": {"dtype":"BF16","file":"model-00001-of-00018.safetensors","shape":[1152]}, + "model.visual.blocks.16.attn.proj.bias": {"dtype":"BF16","file":"model-00001-of-00018.safetensors","shape":[1152]}, + "model.visual.blocks.16.attn.proj.weight": {"dtype":"BF16","file":"model-00001-of-00018.safetensors","shape":[1152,1152]}, + "model.visual.blocks.16.attn.qkv.bias": {"dtype":"BF16","file":"model-00001-of-00018.safetensors","shape":[3456]}, + "model.visual.blocks.16.attn.qkv.weight": {"dtype":"BF16","file":"model-00001-of-00018.safetensors","shape":[3456,1152]}, + "model.visual.blocks.16.mlp.linear_fc1.bias": {"dtype":"BF16","file":"model-00001-of-00018.safetensors","shape":[4304]}, + "model.visual.blocks.16.mlp.linear_fc1.weight": {"dtype":"BF16","file":"model-00001-of-00018.safetensors","shape":[4304,1152]}, + "model.visual.blocks.16.mlp.linear_fc2.bias": {"dtype":"BF16","file":"model-00001-of-00018.safetensors","shape":[1152]}, + "model.visual.blocks.16.mlp.linear_fc2.weight": {"dtype":"BF16","file":"model-00001-of-00018.safetensors","shape":[1152,4304]}, + "model.visual.blocks.16.norm1.bias": {"dtype":"BF16","file":"model-00001-of-00018.safetensors","shape":[1152]}, + "model.visual.blocks.16.norm1.weight": {"dtype":"BF16","file":"model-00001-of-00018.safetensors","shape":[1152]}, + "model.visual.blocks.16.norm2.bias": {"dtype":"BF16","file":"model-00001-of-00018.safetensors","shape":[1152]}, + "model.visual.blocks.16.norm2.weight": {"dtype":"BF16","file":"model-00001-of-00018.safetensors","shape":[1152]}, + "model.visual.blocks.17.attn.proj.bias": {"dtype":"BF16","file":"model-00001-of-00018.safetensors","shape":[1152]}, + "model.visual.blocks.17.attn.proj.weight": {"dtype":"BF16","file":"model-00001-of-00018.safetensors","shape":[1152,1152]}, + "model.visual.blocks.17.attn.qkv.bias": {"dtype":"BF16","file":"model-00001-of-00018.safetensors","shape":[3456]}, + "model.visual.blocks.17.attn.qkv.weight": {"dtype":"BF16","file":"model-00001-of-00018.safetensors","shape":[3456,1152]}, + "model.visual.blocks.17.mlp.linear_fc1.bias": {"dtype":"BF16","file":"model-00001-of-00018.safetensors","shape":[4304]}, + "model.visual.blocks.17.mlp.linear_fc1.weight": {"dtype":"BF16","file":"model-00001-of-00018.safetensors","shape":[4304,1152]}, + "model.visual.blocks.17.mlp.linear_fc2.bias": {"dtype":"BF16","file":"model-00001-of-00018.safetensors","shape":[1152]}, + "model.visual.blocks.17.mlp.linear_fc2.weight": {"dtype":"BF16","file":"model-00001-of-00018.safetensors","shape":[1152,4304]}, + "model.visual.blocks.17.norm1.bias": {"dtype":"BF16","file":"model-00001-of-00018.safetensors","shape":[1152]}, + "model.visual.blocks.17.norm1.weight": {"dtype":"BF16","file":"model-00001-of-00018.safetensors","shape":[1152]}, + "model.visual.blocks.17.norm2.bias": {"dtype":"BF16","file":"model-00001-of-00018.safetensors","shape":[1152]}, + "model.visual.blocks.17.norm2.weight": {"dtype":"BF16","file":"model-00001-of-00018.safetensors","shape":[1152]}, + "model.visual.blocks.18.attn.proj.bias": {"dtype":"BF16","file":"model-00001-of-00018.safetensors","shape":[1152]}, + "model.visual.blocks.18.attn.proj.weight": {"dtype":"BF16","file":"model-00001-of-00018.safetensors","shape":[1152,1152]}, + "model.visual.blocks.18.attn.qkv.bias": {"dtype":"BF16","file":"model-00001-of-00018.safetensors","shape":[3456]}, + "model.visual.blocks.18.attn.qkv.weight": {"dtype":"BF16","file":"model-00001-of-00018.safetensors","shape":[3456,1152]}, + "model.visual.blocks.18.mlp.linear_fc1.bias": {"dtype":"BF16","file":"model-00001-of-00018.safetensors","shape":[4304]}, + "model.visual.blocks.18.mlp.linear_fc1.weight": {"dtype":"BF16","file":"model-00001-of-00018.safetensors","shape":[4304,1152]}, + "model.visual.blocks.18.mlp.linear_fc2.bias": {"dtype":"BF16","file":"model-00001-of-00018.safetensors","shape":[1152]}, + "model.visual.blocks.18.mlp.linear_fc2.weight": {"dtype":"BF16","file":"model-00001-of-00018.safetensors","shape":[1152,4304]}, + "model.visual.blocks.18.norm1.bias": {"dtype":"BF16","file":"model-00001-of-00018.safetensors","shape":[1152]}, + "model.visual.blocks.18.norm1.weight": {"dtype":"BF16","file":"model-00001-of-00018.safetensors","shape":[1152]}, + "model.visual.blocks.18.norm2.bias": {"dtype":"BF16","file":"model-00001-of-00018.safetensors","shape":[1152]}, + "model.visual.blocks.18.norm2.weight": {"dtype":"BF16","file":"model-00001-of-00018.safetensors","shape":[1152]}, + "model.visual.blocks.19.attn.proj.bias": {"dtype":"BF16","file":"model-00001-of-00018.safetensors","shape":[1152]}, + "model.visual.blocks.19.attn.proj.weight": {"dtype":"BF16","file":"model-00001-of-00018.safetensors","shape":[1152,1152]}, + "model.visual.blocks.19.attn.qkv.bias": {"dtype":"BF16","file":"model-00001-of-00018.safetensors","shape":[3456]}, + "model.visual.blocks.19.attn.qkv.weight": {"dtype":"BF16","file":"model-00001-of-00018.safetensors","shape":[3456,1152]}, + "model.visual.blocks.19.mlp.linear_fc1.bias": {"dtype":"BF16","file":"model-00001-of-00018.safetensors","shape":[4304]}, + "model.visual.blocks.19.mlp.linear_fc1.weight": {"dtype":"BF16","file":"model-00001-of-00018.safetensors","shape":[4304,1152]}, + "model.visual.blocks.19.mlp.linear_fc2.bias": {"dtype":"BF16","file":"model-00001-of-00018.safetensors","shape":[1152]}, + "model.visual.blocks.19.mlp.linear_fc2.weight": {"dtype":"BF16","file":"model-00001-of-00018.safetensors","shape":[1152,4304]}, + "model.visual.blocks.19.norm1.bias": {"dtype":"BF16","file":"model-00001-of-00018.safetensors","shape":[1152]}, + "model.visual.blocks.19.norm1.weight": {"dtype":"BF16","file":"model-00001-of-00018.safetensors","shape":[1152]}, + "model.visual.blocks.19.norm2.bias": {"dtype":"BF16","file":"model-00001-of-00018.safetensors","shape":[1152]}, + "model.visual.blocks.19.norm2.weight": {"dtype":"BF16","file":"model-00001-of-00018.safetensors","shape":[1152]}, + "model.visual.blocks.2.attn.proj.bias": {"dtype":"BF16","file":"model-00001-of-00018.safetensors","shape":[1152]}, + "model.visual.blocks.2.attn.proj.weight": {"dtype":"BF16","file":"model-00001-of-00018.safetensors","shape":[1152,1152]}, + "model.visual.blocks.2.attn.qkv.bias": {"dtype":"BF16","file":"model-00001-of-00018.safetensors","shape":[3456]}, + "model.visual.blocks.2.attn.qkv.weight": {"dtype":"BF16","file":"model-00001-of-00018.safetensors","shape":[3456,1152]}, + "model.visual.blocks.2.mlp.linear_fc1.bias": {"dtype":"BF16","file":"model-00001-of-00018.safetensors","shape":[4304]}, + "model.visual.blocks.2.mlp.linear_fc1.weight": {"dtype":"BF16","file":"model-00001-of-00018.safetensors","shape":[4304,1152]}, + "model.visual.blocks.2.mlp.linear_fc2.bias": {"dtype":"BF16","file":"model-00001-of-00018.safetensors","shape":[1152]}, + "model.visual.blocks.2.mlp.linear_fc2.weight": {"dtype":"BF16","file":"model-00001-of-00018.safetensors","shape":[1152,4304]}, + "model.visual.blocks.2.norm1.bias": {"dtype":"BF16","file":"model-00001-of-00018.safetensors","shape":[1152]}, + "model.visual.blocks.2.norm1.weight": {"dtype":"BF16","file":"model-00001-of-00018.safetensors","shape":[1152]}, + "model.visual.blocks.2.norm2.bias": {"dtype":"BF16","file":"model-00001-of-00018.safetensors","shape":[1152]}, + "model.visual.blocks.2.norm2.weight": {"dtype":"BF16","file":"model-00001-of-00018.safetensors","shape":[1152]}, + "model.visual.blocks.20.attn.proj.bias": {"dtype":"BF16","file":"model-00001-of-00018.safetensors","shape":[1152]}, + "model.visual.blocks.20.attn.proj.weight": {"dtype":"BF16","file":"model-00001-of-00018.safetensors","shape":[1152,1152]}, + "model.visual.blocks.20.attn.qkv.bias": {"dtype":"BF16","file":"model-00001-of-00018.safetensors","shape":[3456]}, + "model.visual.blocks.20.attn.qkv.weight": {"dtype":"BF16","file":"model-00001-of-00018.safetensors","shape":[3456,1152]}, + "model.visual.blocks.20.mlp.linear_fc1.bias": {"dtype":"BF16","file":"model-00001-of-00018.safetensors","shape":[4304]}, + "model.visual.blocks.20.mlp.linear_fc1.weight": {"dtype":"BF16","file":"model-00001-of-00018.safetensors","shape":[4304,1152]}, + "model.visual.blocks.20.mlp.linear_fc2.bias": {"dtype":"BF16","file":"model-00001-of-00018.safetensors","shape":[1152]}, + "model.visual.blocks.20.mlp.linear_fc2.weight": {"dtype":"BF16","file":"model-00001-of-00018.safetensors","shape":[1152,4304]}, + "model.visual.blocks.20.norm1.bias": {"dtype":"BF16","file":"model-00001-of-00018.safetensors","shape":[1152]}, + "model.visual.blocks.20.norm1.weight": {"dtype":"BF16","file":"model-00001-of-00018.safetensors","shape":[1152]}, + "model.visual.blocks.20.norm2.bias": {"dtype":"BF16","file":"model-00001-of-00018.safetensors","shape":[1152]}, + "model.visual.blocks.20.norm2.weight": {"dtype":"BF16","file":"model-00001-of-00018.safetensors","shape":[1152]}, + "model.visual.blocks.21.attn.proj.bias": {"dtype":"BF16","file":"model-00001-of-00018.safetensors","shape":[1152]}, + "model.visual.blocks.21.attn.proj.weight": {"dtype":"BF16","file":"model-00001-of-00018.safetensors","shape":[1152,1152]}, + "model.visual.blocks.21.attn.qkv.bias": {"dtype":"BF16","file":"model-00001-of-00018.safetensors","shape":[3456]}, + "model.visual.blocks.21.attn.qkv.weight": {"dtype":"BF16","file":"model-00001-of-00018.safetensors","shape":[3456,1152]}, + "model.visual.blocks.21.mlp.linear_fc1.bias": {"dtype":"BF16","file":"model-00001-of-00018.safetensors","shape":[4304]}, + "model.visual.blocks.21.mlp.linear_fc1.weight": {"dtype":"BF16","file":"model-00001-of-00018.safetensors","shape":[4304,1152]}, + "model.visual.blocks.21.mlp.linear_fc2.bias": {"dtype":"BF16","file":"model-00001-of-00018.safetensors","shape":[1152]}, + "model.visual.blocks.21.mlp.linear_fc2.weight": {"dtype":"BF16","file":"model-00001-of-00018.safetensors","shape":[1152,4304]}, + "model.visual.blocks.21.norm1.bias": {"dtype":"BF16","file":"model-00001-of-00018.safetensors","shape":[1152]}, + "model.visual.blocks.21.norm1.weight": {"dtype":"BF16","file":"model-00001-of-00018.safetensors","shape":[1152]}, + "model.visual.blocks.21.norm2.bias": {"dtype":"BF16","file":"model-00001-of-00018.safetensors","shape":[1152]}, + "model.visual.blocks.21.norm2.weight": {"dtype":"BF16","file":"model-00001-of-00018.safetensors","shape":[1152]}, + "model.visual.blocks.22.attn.proj.bias": {"dtype":"BF16","file":"model-00001-of-00018.safetensors","shape":[1152]}, + "model.visual.blocks.22.attn.proj.weight": {"dtype":"BF16","file":"model-00001-of-00018.safetensors","shape":[1152,1152]}, + "model.visual.blocks.22.attn.qkv.bias": {"dtype":"BF16","file":"model-00001-of-00018.safetensors","shape":[3456]}, + "model.visual.blocks.22.attn.qkv.weight": {"dtype":"BF16","file":"model-00001-of-00018.safetensors","shape":[3456,1152]}, + "model.visual.blocks.22.mlp.linear_fc1.bias": {"dtype":"BF16","file":"model-00001-of-00018.safetensors","shape":[4304]}, + "model.visual.blocks.22.mlp.linear_fc1.weight": {"dtype":"BF16","file":"model-00001-of-00018.safetensors","shape":[4304,1152]}, + "model.visual.blocks.22.mlp.linear_fc2.bias": {"dtype":"BF16","file":"model-00001-of-00018.safetensors","shape":[1152]}, + "model.visual.blocks.22.mlp.linear_fc2.weight": {"dtype":"BF16","file":"model-00001-of-00018.safetensors","shape":[1152,4304]}, + "model.visual.blocks.22.norm1.bias": {"dtype":"BF16","file":"model-00001-of-00018.safetensors","shape":[1152]}, + "model.visual.blocks.22.norm1.weight": {"dtype":"BF16","file":"model-00001-of-00018.safetensors","shape":[1152]}, + "model.visual.blocks.22.norm2.bias": {"dtype":"BF16","file":"model-00001-of-00018.safetensors","shape":[1152]}, + "model.visual.blocks.22.norm2.weight": {"dtype":"BF16","file":"model-00001-of-00018.safetensors","shape":[1152]}, + "model.visual.blocks.23.attn.proj.bias": {"dtype":"BF16","file":"model-00001-of-00018.safetensors","shape":[1152]}, + "model.visual.blocks.23.attn.proj.weight": {"dtype":"BF16","file":"model-00001-of-00018.safetensors","shape":[1152,1152]}, + "model.visual.blocks.23.attn.qkv.bias": {"dtype":"BF16","file":"model-00001-of-00018.safetensors","shape":[3456]}, + "model.visual.blocks.23.attn.qkv.weight": {"dtype":"BF16","file":"model-00001-of-00018.safetensors","shape":[3456,1152]}, + "model.visual.blocks.23.mlp.linear_fc1.bias": {"dtype":"BF16","file":"model-00001-of-00018.safetensors","shape":[4304]}, + "model.visual.blocks.23.mlp.linear_fc1.weight": {"dtype":"BF16","file":"model-00001-of-00018.safetensors","shape":[4304,1152]}, + "model.visual.blocks.23.mlp.linear_fc2.bias": {"dtype":"BF16","file":"model-00001-of-00018.safetensors","shape":[1152]}, + "model.visual.blocks.23.mlp.linear_fc2.weight": {"dtype":"BF16","file":"model-00001-of-00018.safetensors","shape":[1152,4304]}, + "model.visual.blocks.23.norm1.bias": {"dtype":"BF16","file":"model-00001-of-00018.safetensors","shape":[1152]}, + "model.visual.blocks.23.norm1.weight": {"dtype":"BF16","file":"model-00001-of-00018.safetensors","shape":[1152]}, + "model.visual.blocks.23.norm2.bias": {"dtype":"BF16","file":"model-00001-of-00018.safetensors","shape":[1152]}, + "model.visual.blocks.23.norm2.weight": {"dtype":"BF16","file":"model-00001-of-00018.safetensors","shape":[1152]}, + "model.visual.blocks.24.attn.proj.bias": {"dtype":"BF16","file":"model-00001-of-00018.safetensors","shape":[1152]}, + "model.visual.blocks.24.attn.proj.weight": {"dtype":"BF16","file":"model-00001-of-00018.safetensors","shape":[1152,1152]}, + "model.visual.blocks.24.attn.qkv.bias": {"dtype":"BF16","file":"model-00001-of-00018.safetensors","shape":[3456]}, + "model.visual.blocks.24.attn.qkv.weight": {"dtype":"BF16","file":"model-00001-of-00018.safetensors","shape":[3456,1152]}, + "model.visual.blocks.24.mlp.linear_fc1.bias": {"dtype":"BF16","file":"model-00001-of-00018.safetensors","shape":[4304]}, + "model.visual.blocks.24.mlp.linear_fc1.weight": {"dtype":"BF16","file":"model-00001-of-00018.safetensors","shape":[4304,1152]}, + "model.visual.blocks.24.mlp.linear_fc2.bias": {"dtype":"BF16","file":"model-00001-of-00018.safetensors","shape":[1152]}, + "model.visual.blocks.24.mlp.linear_fc2.weight": {"dtype":"BF16","file":"model-00001-of-00018.safetensors","shape":[1152,4304]}, + "model.visual.blocks.24.norm1.bias": {"dtype":"BF16","file":"model-00001-of-00018.safetensors","shape":[1152]}, + "model.visual.blocks.24.norm1.weight": {"dtype":"BF16","file":"model-00001-of-00018.safetensors","shape":[1152]}, + "model.visual.blocks.24.norm2.bias": {"dtype":"BF16","file":"model-00001-of-00018.safetensors","shape":[1152]}, + "model.visual.blocks.24.norm2.weight": {"dtype":"BF16","file":"model-00001-of-00018.safetensors","shape":[1152]}, + "model.visual.blocks.25.attn.proj.bias": {"dtype":"BF16","file":"model-00001-of-00018.safetensors","shape":[1152]}, + "model.visual.blocks.25.attn.proj.weight": {"dtype":"BF16","file":"model-00001-of-00018.safetensors","shape":[1152,1152]}, + "model.visual.blocks.25.attn.qkv.bias": {"dtype":"BF16","file":"model-00001-of-00018.safetensors","shape":[3456]}, + "model.visual.blocks.25.attn.qkv.weight": {"dtype":"BF16","file":"model-00001-of-00018.safetensors","shape":[3456,1152]}, + "model.visual.blocks.25.mlp.linear_fc1.bias": {"dtype":"BF16","file":"model-00001-of-00018.safetensors","shape":[4304]}, + "model.visual.blocks.25.mlp.linear_fc1.weight": {"dtype":"BF16","file":"model-00001-of-00018.safetensors","shape":[4304,1152]}, + "model.visual.blocks.25.mlp.linear_fc2.bias": {"dtype":"BF16","file":"model-00001-of-00018.safetensors","shape":[1152]}, + "model.visual.blocks.25.mlp.linear_fc2.weight": {"dtype":"BF16","file":"model-00001-of-00018.safetensors","shape":[1152,4304]}, + "model.visual.blocks.25.norm1.bias": {"dtype":"BF16","file":"model-00001-of-00018.safetensors","shape":[1152]}, + "model.visual.blocks.25.norm1.weight": {"dtype":"BF16","file":"model-00001-of-00018.safetensors","shape":[1152]}, + "model.visual.blocks.25.norm2.bias": {"dtype":"BF16","file":"model-00001-of-00018.safetensors","shape":[1152]}, + "model.visual.blocks.25.norm2.weight": {"dtype":"BF16","file":"model-00001-of-00018.safetensors","shape":[1152]}, + "model.visual.blocks.26.attn.proj.bias": {"dtype":"BF16","file":"model-00001-of-00018.safetensors","shape":[1152]}, + "model.visual.blocks.26.attn.proj.weight": {"dtype":"BF16","file":"model-00001-of-00018.safetensors","shape":[1152,1152]}, + "model.visual.blocks.26.attn.qkv.bias": {"dtype":"BF16","file":"model-00001-of-00018.safetensors","shape":[3456]}, + "model.visual.blocks.26.attn.qkv.weight": {"dtype":"BF16","file":"model-00001-of-00018.safetensors","shape":[3456,1152]}, + "model.visual.blocks.26.mlp.linear_fc1.bias": {"dtype":"BF16","file":"model-00001-of-00018.safetensors","shape":[4304]}, + "model.visual.blocks.26.mlp.linear_fc1.weight": {"dtype":"BF16","file":"model-00001-of-00018.safetensors","shape":[4304,1152]}, + "model.visual.blocks.26.mlp.linear_fc2.bias": {"dtype":"BF16","file":"model-00001-of-00018.safetensors","shape":[1152]}, + "model.visual.blocks.26.mlp.linear_fc2.weight": {"dtype":"BF16","file":"model-00001-of-00018.safetensors","shape":[1152,4304]}, + "model.visual.blocks.26.norm1.bias": {"dtype":"BF16","file":"model-00001-of-00018.safetensors","shape":[1152]}, + "model.visual.blocks.26.norm1.weight": {"dtype":"BF16","file":"model-00001-of-00018.safetensors","shape":[1152]}, + "model.visual.blocks.26.norm2.bias": {"dtype":"BF16","file":"model-00001-of-00018.safetensors","shape":[1152]}, + "model.visual.blocks.26.norm2.weight": {"dtype":"BF16","file":"model-00001-of-00018.safetensors","shape":[1152]}, + "model.visual.blocks.3.attn.proj.bias": {"dtype":"BF16","file":"model-00001-of-00018.safetensors","shape":[1152]}, + "model.visual.blocks.3.attn.proj.weight": {"dtype":"BF16","file":"model-00001-of-00018.safetensors","shape":[1152,1152]}, + "model.visual.blocks.3.attn.qkv.bias": {"dtype":"BF16","file":"model-00001-of-00018.safetensors","shape":[3456]}, + "model.visual.blocks.3.attn.qkv.weight": {"dtype":"BF16","file":"model-00001-of-00018.safetensors","shape":[3456,1152]}, + "model.visual.blocks.3.mlp.linear_fc1.bias": {"dtype":"BF16","file":"model-00001-of-00018.safetensors","shape":[4304]}, + "model.visual.blocks.3.mlp.linear_fc1.weight": {"dtype":"BF16","file":"model-00001-of-00018.safetensors","shape":[4304,1152]}, + "model.visual.blocks.3.mlp.linear_fc2.bias": {"dtype":"BF16","file":"model-00001-of-00018.safetensors","shape":[1152]}, + "model.visual.blocks.3.mlp.linear_fc2.weight": {"dtype":"BF16","file":"model-00001-of-00018.safetensors","shape":[1152,4304]}, + "model.visual.blocks.3.norm1.bias": {"dtype":"BF16","file":"model-00001-of-00018.safetensors","shape":[1152]}, + "model.visual.blocks.3.norm1.weight": {"dtype":"BF16","file":"model-00001-of-00018.safetensors","shape":[1152]}, + "model.visual.blocks.3.norm2.bias": {"dtype":"BF16","file":"model-00001-of-00018.safetensors","shape":[1152]}, + "model.visual.blocks.3.norm2.weight": {"dtype":"BF16","file":"model-00001-of-00018.safetensors","shape":[1152]}, + "model.visual.blocks.4.attn.proj.bias": {"dtype":"BF16","file":"model-00001-of-00018.safetensors","shape":[1152]}, + "model.visual.blocks.4.attn.proj.weight": {"dtype":"BF16","file":"model-00001-of-00018.safetensors","shape":[1152,1152]}, + "model.visual.blocks.4.attn.qkv.bias": {"dtype":"BF16","file":"model-00001-of-00018.safetensors","shape":[3456]}, + "model.visual.blocks.4.attn.qkv.weight": {"dtype":"BF16","file":"model-00001-of-00018.safetensors","shape":[3456,1152]}, + "model.visual.blocks.4.mlp.linear_fc1.bias": {"dtype":"BF16","file":"model-00001-of-00018.safetensors","shape":[4304]}, + "model.visual.blocks.4.mlp.linear_fc1.weight": {"dtype":"BF16","file":"model-00001-of-00018.safetensors","shape":[4304,1152]}, + "model.visual.blocks.4.mlp.linear_fc2.bias": {"dtype":"BF16","file":"model-00001-of-00018.safetensors","shape":[1152]}, + "model.visual.blocks.4.mlp.linear_fc2.weight": {"dtype":"BF16","file":"model-00001-of-00018.safetensors","shape":[1152,4304]}, + "model.visual.blocks.4.norm1.bias": {"dtype":"BF16","file":"model-00001-of-00018.safetensors","shape":[1152]}, + "model.visual.blocks.4.norm1.weight": {"dtype":"BF16","file":"model-00001-of-00018.safetensors","shape":[1152]}, + "model.visual.blocks.4.norm2.bias": {"dtype":"BF16","file":"model-00001-of-00018.safetensors","shape":[1152]}, + "model.visual.blocks.4.norm2.weight": {"dtype":"BF16","file":"model-00001-of-00018.safetensors","shape":[1152]}, + "model.visual.blocks.5.attn.proj.bias": {"dtype":"BF16","file":"model-00001-of-00018.safetensors","shape":[1152]}, + "model.visual.blocks.5.attn.proj.weight": {"dtype":"BF16","file":"model-00001-of-00018.safetensors","shape":[1152,1152]}, + "model.visual.blocks.5.attn.qkv.bias": {"dtype":"BF16","file":"model-00001-of-00018.safetensors","shape":[3456]}, + "model.visual.blocks.5.attn.qkv.weight": {"dtype":"BF16","file":"model-00001-of-00018.safetensors","shape":[3456,1152]}, + "model.visual.blocks.5.mlp.linear_fc1.bias": {"dtype":"BF16","file":"model-00001-of-00018.safetensors","shape":[4304]}, + "model.visual.blocks.5.mlp.linear_fc1.weight": {"dtype":"BF16","file":"model-00001-of-00018.safetensors","shape":[4304,1152]}, + "model.visual.blocks.5.mlp.linear_fc2.bias": {"dtype":"BF16","file":"model-00001-of-00018.safetensors","shape":[1152]}, + "model.visual.blocks.5.mlp.linear_fc2.weight": {"dtype":"BF16","file":"model-00001-of-00018.safetensors","shape":[1152,4304]}, + "model.visual.blocks.5.norm1.bias": {"dtype":"BF16","file":"model-00001-of-00018.safetensors","shape":[1152]}, + "model.visual.blocks.5.norm1.weight": {"dtype":"BF16","file":"model-00001-of-00018.safetensors","shape":[1152]}, + "model.visual.blocks.5.norm2.bias": {"dtype":"BF16","file":"model-00001-of-00018.safetensors","shape":[1152]}, + "model.visual.blocks.5.norm2.weight": {"dtype":"BF16","file":"model-00001-of-00018.safetensors","shape":[1152]}, + "model.visual.blocks.6.attn.proj.bias": {"dtype":"BF16","file":"model-00001-of-00018.safetensors","shape":[1152]}, + "model.visual.blocks.6.attn.proj.weight": {"dtype":"BF16","file":"model-00001-of-00018.safetensors","shape":[1152,1152]}, + "model.visual.blocks.6.attn.qkv.bias": {"dtype":"BF16","file":"model-00001-of-00018.safetensors","shape":[3456]}, + "model.visual.blocks.6.attn.qkv.weight": {"dtype":"BF16","file":"model-00001-of-00018.safetensors","shape":[3456,1152]}, + "model.visual.blocks.6.mlp.linear_fc1.bias": {"dtype":"BF16","file":"model-00001-of-00018.safetensors","shape":[4304]}, + "model.visual.blocks.6.mlp.linear_fc1.weight": {"dtype":"BF16","file":"model-00001-of-00018.safetensors","shape":[4304,1152]}, + "model.visual.blocks.6.mlp.linear_fc2.bias": {"dtype":"BF16","file":"model-00001-of-00018.safetensors","shape":[1152]}, + "model.visual.blocks.6.mlp.linear_fc2.weight": {"dtype":"BF16","file":"model-00001-of-00018.safetensors","shape":[1152,4304]}, + "model.visual.blocks.6.norm1.bias": {"dtype":"BF16","file":"model-00001-of-00018.safetensors","shape":[1152]}, + "model.visual.blocks.6.norm1.weight": {"dtype":"BF16","file":"model-00001-of-00018.safetensors","shape":[1152]}, + "model.visual.blocks.6.norm2.bias": {"dtype":"BF16","file":"model-00001-of-00018.safetensors","shape":[1152]}, + "model.visual.blocks.6.norm2.weight": {"dtype":"BF16","file":"model-00001-of-00018.safetensors","shape":[1152]}, + "model.visual.blocks.7.attn.proj.bias": {"dtype":"BF16","file":"model-00001-of-00018.safetensors","shape":[1152]}, + "model.visual.blocks.7.attn.proj.weight": {"dtype":"BF16","file":"model-00001-of-00018.safetensors","shape":[1152,1152]}, + "model.visual.blocks.7.attn.qkv.bias": {"dtype":"BF16","file":"model-00001-of-00018.safetensors","shape":[3456]}, + "model.visual.blocks.7.attn.qkv.weight": {"dtype":"BF16","file":"model-00001-of-00018.safetensors","shape":[3456,1152]}, + "model.visual.blocks.7.mlp.linear_fc1.bias": {"dtype":"BF16","file":"model-00001-of-00018.safetensors","shape":[4304]}, + "model.visual.blocks.7.mlp.linear_fc1.weight": {"dtype":"BF16","file":"model-00001-of-00018.safetensors","shape":[4304,1152]}, + "model.visual.blocks.7.mlp.linear_fc2.bias": {"dtype":"BF16","file":"model-00001-of-00018.safetensors","shape":[1152]}, + "model.visual.blocks.7.mlp.linear_fc2.weight": {"dtype":"BF16","file":"model-00001-of-00018.safetensors","shape":[1152,4304]}, + "model.visual.blocks.7.norm1.bias": {"dtype":"BF16","file":"model-00001-of-00018.safetensors","shape":[1152]}, + "model.visual.blocks.7.norm1.weight": {"dtype":"BF16","file":"model-00001-of-00018.safetensors","shape":[1152]}, + "model.visual.blocks.7.norm2.bias": {"dtype":"BF16","file":"model-00001-of-00018.safetensors","shape":[1152]}, + "model.visual.blocks.7.norm2.weight": {"dtype":"BF16","file":"model-00001-of-00018.safetensors","shape":[1152]}, + "model.visual.blocks.8.attn.proj.bias": {"dtype":"BF16","file":"model-00001-of-00018.safetensors","shape":[1152]}, + "model.visual.blocks.8.attn.proj.weight": {"dtype":"BF16","file":"model-00001-of-00018.safetensors","shape":[1152,1152]}, + "model.visual.blocks.8.attn.qkv.bias": {"dtype":"BF16","file":"model-00001-of-00018.safetensors","shape":[3456]}, + "model.visual.blocks.8.attn.qkv.weight": {"dtype":"BF16","file":"model-00001-of-00018.safetensors","shape":[3456,1152]}, + "model.visual.blocks.8.mlp.linear_fc1.bias": {"dtype":"BF16","file":"model-00001-of-00018.safetensors","shape":[4304]}, + "model.visual.blocks.8.mlp.linear_fc1.weight": {"dtype":"BF16","file":"model-00001-of-00018.safetensors","shape":[4304,1152]}, + "model.visual.blocks.8.mlp.linear_fc2.bias": {"dtype":"BF16","file":"model-00001-of-00018.safetensors","shape":[1152]}, + "model.visual.blocks.8.mlp.linear_fc2.weight": {"dtype":"BF16","file":"model-00001-of-00018.safetensors","shape":[1152,4304]}, + "model.visual.blocks.8.norm1.bias": {"dtype":"BF16","file":"model-00001-of-00018.safetensors","shape":[1152]}, + "model.visual.blocks.8.norm1.weight": {"dtype":"BF16","file":"model-00001-of-00018.safetensors","shape":[1152]}, + "model.visual.blocks.8.norm2.bias": {"dtype":"BF16","file":"model-00001-of-00018.safetensors","shape":[1152]}, + "model.visual.blocks.8.norm2.weight": {"dtype":"BF16","file":"model-00001-of-00018.safetensors","shape":[1152]}, + "model.visual.blocks.9.attn.proj.bias": {"dtype":"BF16","file":"model-00001-of-00018.safetensors","shape":[1152]}, + "model.visual.blocks.9.attn.proj.weight": {"dtype":"BF16","file":"model-00001-of-00018.safetensors","shape":[1152,1152]}, + "model.visual.blocks.9.attn.qkv.bias": {"dtype":"BF16","file":"model-00001-of-00018.safetensors","shape":[3456]}, + "model.visual.blocks.9.attn.qkv.weight": {"dtype":"BF16","file":"model-00001-of-00018.safetensors","shape":[3456,1152]}, + "model.visual.blocks.9.mlp.linear_fc1.bias": {"dtype":"BF16","file":"model-00001-of-00018.safetensors","shape":[4304]}, + "model.visual.blocks.9.mlp.linear_fc1.weight": {"dtype":"BF16","file":"model-00001-of-00018.safetensors","shape":[4304,1152]}, + "model.visual.blocks.9.mlp.linear_fc2.bias": {"dtype":"BF16","file":"model-00001-of-00018.safetensors","shape":[1152]}, + "model.visual.blocks.9.mlp.linear_fc2.weight": {"dtype":"BF16","file":"model-00001-of-00018.safetensors","shape":[1152,4304]}, + "model.visual.blocks.9.norm1.bias": {"dtype":"BF16","file":"model-00001-of-00018.safetensors","shape":[1152]}, + "model.visual.blocks.9.norm1.weight": {"dtype":"BF16","file":"model-00001-of-00018.safetensors","shape":[1152]}, + "model.visual.blocks.9.norm2.bias": {"dtype":"BF16","file":"model-00001-of-00018.safetensors","shape":[1152]}, + "model.visual.blocks.9.norm2.weight": {"dtype":"BF16","file":"model-00001-of-00018.safetensors","shape":[1152]}, + "model.visual.merger.linear_fc1.bias": {"dtype":"BF16","file":"model-00001-of-00018.safetensors","shape":[4608]}, + "model.visual.merger.linear_fc1.weight": {"dtype":"BF16","file":"model-00001-of-00018.safetensors","shape":[4608,4608]}, + "model.visual.merger.linear_fc2.bias": {"dtype":"BF16","file":"model-00001-of-00018.safetensors","shape":[5120]}, + "model.visual.merger.linear_fc2.weight": {"dtype":"BF16","file":"model-00001-of-00018.safetensors","shape":[5120,4608]}, + "model.visual.merger.norm.bias": {"dtype":"BF16","file":"model-00001-of-00018.safetensors","shape":[1152]}, + "model.visual.merger.norm.weight": {"dtype":"BF16","file":"model-00001-of-00018.safetensors","shape":[1152]}, + "model.visual.patch_embed.proj.bias": {"dtype":"BF16","file":"model-00001-of-00018.safetensors","shape":[1152]}, + "model.visual.patch_embed.proj.weight": {"dtype":"BF16","file":"model-00001-of-00018.safetensors","shape":[1152,3,2,16,16]}, + "model.visual.pos_embed.weight": {"dtype":"BF16","file":"model-00001-of-00018.safetensors","shape":[2304,1152]}, + "mtp.fc.weight": {"dtype":"BF16","file":"model-00018-of-00018.safetensors","shape":[5120,10240]}, + "mtp.layers.0.input_layernorm.weight": {"dtype":"BF16","file":"model-00018-of-00018.safetensors","shape":[5120]}, + "mtp.layers.0.mlp.down_proj.weight": {"dtype":"BF16","file":"model-00018-of-00018.safetensors","shape":[5120,17408]}, + "mtp.layers.0.mlp.gate_proj.weight": {"dtype":"BF16","file":"model-00018-of-00018.safetensors","shape":[17408,5120]}, + "mtp.layers.0.mlp.up_proj.weight": {"dtype":"BF16","file":"model-00018-of-00018.safetensors","shape":[17408,5120]}, + "mtp.layers.0.post_attention_layernorm.weight": {"dtype":"BF16","file":"model-00018-of-00018.safetensors","shape":[5120]}, + "mtp.layers.0.self_attn.k_norm.weight": {"dtype":"BF16","file":"model-00018-of-00018.safetensors","shape":[256]}, + "mtp.layers.0.self_attn.k_proj.weight": {"dtype":"BF16","file":"model-00018-of-00018.safetensors","shape":[1024,5120]}, + "mtp.layers.0.self_attn.o_proj.weight": {"dtype":"BF16","file":"model-00018-of-00018.safetensors","shape":[5120,6144]}, + "mtp.layers.0.self_attn.q_norm.weight": {"dtype":"BF16","file":"model-00018-of-00018.safetensors","shape":[256]}, + "mtp.layers.0.self_attn.q_proj.weight": {"dtype":"BF16","file":"model-00018-of-00018.safetensors","shape":[12288,5120]}, + "mtp.layers.0.self_attn.v_proj.weight": {"dtype":"BF16","file":"model-00018-of-00018.safetensors","shape":[1024,5120]}, + "mtp.norm.weight": {"dtype":"BF16","file":"model-00018-of-00018.safetensors","shape":[5120]}, + "mtp.pre_fc_norm_embedding.weight": {"dtype":"BF16","file":"model-00018-of-00018.safetensors","shape":[5120]}, + "mtp.pre_fc_norm_hidden.weight": {"dtype":"BF16","file":"model-00018-of-00018.safetensors","shape":[5120]} + } + }, + "provenance": { + "base_model_id": "Qwen/Qwen3.8-27B", + "base_revision": "1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0", + "checkpoint_revision": "76e3c209e8441fa658221c7ba2725bad2f811176", + "source_revision": "6822fe0fd53c5e6670af6ba99fb2c857a661e532" + }, + "source": "Pinned Hugging Face repository tree LFS SHA256 values and byte-range safetensors headers; metadata SHA256 calculated from pinned downloads.", + "upstream_source_sha256": { + "LICENSE": "c71d239df91726fc519c6eb72d318ec65820627232b2f796219e87dcf35d0ab4", + "README.md": "d4dde35eaaa83719a3f8a6bac10e073daa47a8a18e90b1e5c7de0f3a897a2e22", + "jemm/__init__.py": "0e83f00b91ed5dc18dab9c5e839d661164bbb3b94b0488ebca63fb73cca3d093", + "jemm/contract.py": "74e8a43c2ffef90dac74f555c6c0d40e5e22302352e94308a4b221122ca6979f", + "jemm/model.py": "6c85487ab7a26741a7c4c70ab29f41ed23968657136fe4d840a1be2c3cf3c937", + "jemm/serve.py": "e1c4c61a70d00d2c2c1e04ed83077d2f0684cdcfd4413dc1c80047aeabf8c2c2", + "jemm/systemone.py": "18195fb0da87a9ae168aa1098ca63e3edc3402a809a417c82176caabdc11227a", + "pyproject.toml": "ee0035ab2de83c0281a558593f32737f70e2d578b3cb961dc2a6a46d84686dc5" + } +} diff --git a/src/models/jemm/native/Cargo.toml b/src/models/jemm/native/Cargo.toml new file mode 100644 index 00000000..11d9c841 --- /dev/null +++ b/src/models/jemm/native/Cargo.toml @@ -0,0 +1,18 @@ +[package] +name = "omni-jemm-native" +version = "0.1.0" +edition = "2024" +publish = false +[dependencies] +sha2 = "0.10.9" +anyhow = "1.0.100" +axum = "0.8.8" +base64 = "0.22.1" +half = "2.7.1" +image = { version = "0.25", default-features = false, features = ["png", "jpeg", "webp"] } +safetensors = "0.8.0" +omni-qwen3-5-native = { path = "../../qwen3_5/native" } +omni-runtime = { path = "../../../runtime" } +serde_json = { version = "1.0.149", features = ["float_roundtrip", "preserve_order"] } +tokenizers = { version = "=0.22.2", default-features = false, features = ["onig"] } +tokio = { version = "1.49.0", features = ["macros", "net", "rt-multi-thread", "sync"] } diff --git a/src/models/jemm/native/LICENSE.jemm b/src/models/jemm/native/LICENSE.jemm new file mode 100644 index 00000000..261eeb9e --- /dev/null +++ b/src/models/jemm/native/LICENSE.jemm @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/src/models/jemm/native/LICENSE.unicode b/src/models/jemm/native/LICENSE.unicode new file mode 100644 index 00000000..56da5891 --- /dev/null +++ b/src/models/jemm/native/LICENSE.unicode @@ -0,0 +1,39 @@ +UNICODE LICENSE V3 + +COPYRIGHT AND PERMISSION NOTICE + +Copyright © 1991-2026 Unicode, Inc. + +NOTICE TO USER: Carefully read the following legal agreement. BY +DOWNLOADING, INSTALLING, COPYING OR OTHERWISE USING DATA FILES, AND/OR +SOFTWARE, YOU UNEQUIVOCALLY ACCEPT, AND AGREE TO BE BOUND BY, ALL OF THE +TERMS AND CONDITIONS OF THIS AGREEMENT. IF YOU DO NOT AGREE, DO NOT +DOWNLOAD, INSTALL, COPY, DISTRIBUTE OR USE THE DATA FILES OR SOFTWARE. + +Permission is hereby granted, free of charge, to any person obtaining a +copy of data files and any associated documentation (the "Data Files") or +software and any associated documentation (the "Software") to deal in the +Data Files or Software without restriction, including without limitation +the rights to use, copy, modify, merge, publish, distribute, and/or sell +copies of the Data Files or Software, and to permit persons to whom the +Data Files or Software are furnished to do so, provided that either (a) +this copyright and permission notice appear with all copies of the Data +Files or Software, or (b) this copyright and permission notice appear in +associated Documentation. + +THE DATA FILES AND SOFTWARE ARE PROVIDED "AS IS", WITHOUT WARRANTY OF ANY +KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT OF +THIRD PARTY RIGHTS. + +IN NO EVENT SHALL THE COPYRIGHT HOLDER OR HOLDERS INCLUDED IN THIS NOTICE +BE LIABLE FOR ANY CLAIM, OR ANY SPECIAL INDIRECT OR CONSEQUENTIAL DAMAGES, +OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, +WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, +ARISING OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THE DATA +FILES OR SOFTWARE. + +Except as contained in this notice, the name of a copyright holder shall +not be used in advertising or otherwise to promote the sale, use or other +dealings in these Data Files or Software without prior written +authorization of the copyright holder. diff --git a/src/models/jemm/native/src/artifacts.rs b/src/models/jemm/native/src/artifacts.rs new file mode 100644 index 00000000..b3abe60d --- /dev/null +++ b/src/models/jemm/native/src/artifacts.rs @@ -0,0 +1,97 @@ +//! Streaming integrity verification before native device initialization. +use anyhow::{Context, Result, ensure}; +use serde_json::Value; +use sha2::{Digest, Sha256}; +use std::{ + fs::File, + io::Read, + path::{Component, Path}, +}; +fn relative(name: &str) -> bool { + !name.is_empty() + && Path::new(name) + .components() + .all(|p| matches!(p, Component::Normal(_))) +} +pub fn verify_export(dir: &Path, manifest: &Value) -> Result<()> { + let hashes = manifest["export_sha256"] + .as_object() + .context("export_sha256 must contain all native artifact hashes")?; + ensure!(!hashes.is_empty(), "export_sha256 must not be empty"); + for (name, expected) in hashes { + ensure!( + relative(name), + "export hash paths must be relative normal paths" + ); + let hash = expected + .as_str() + .context("export SHA256 must be a string")?; + ensure!( + hash.len() == 64 + && hash + .bytes() + .all(|b| b.is_ascii_hexdigit() && !b.is_ascii_uppercase()), + "invalid export SHA256 for {name}" + ); + ensure!( + !["jemm_export.json", "export_progress.json"].contains(&name.as_str()), + "export hash map must not include manifest/progress markers" + ); + } + for name in [ + "config.json", + "tokenizer.json", + "preprocessor_config.json", + "model.safetensors.index.json", + "vision.safetensors", + "jemm_lm_head.safetensors", + ] { + ensure!( + hashes.contains_key(name), + "export_sha256 missing required artifact {name}" + ); + } + let index: Value = + serde_json::from_slice(&std::fs::read(dir.join("model.safetensors.index.json"))?)?; + let map = index["weight_map"] + .as_object() + .context("language index weight_map")?; + ensure!( + !map.is_empty(), + "language index weight_map must not be empty" + ); + for file in map.values() { + let file = file.as_str().context("language shard path")?; + ensure!( + relative(file) && hashes.contains_key(file), + "export_sha256 missing language shard {file}" + ); + } + let canonical_dir = dir.canonicalize()?; + let mut buffer = vec![0u8; 1 << 20]; + for (name, expected) in hashes { + let path = dir.join(name); + let metadata = + std::fs::symlink_metadata(&path).with_context(|| format!("artifact {name}"))?; + ensure!( + metadata.is_file() + && !metadata.file_type().is_symlink() + && path.canonicalize()?.starts_with(&canonical_dir), + "artifact must be a regular file within export: {name}" + ); + let mut file = File::open(&path)?; + let mut digest = Sha256::new(); + loop { + let count = file.read(&mut buffer)?; + if count == 0 { + break; + } + digest.update(&buffer[..count]); + } + ensure!( + format!("{:x}", digest.finalize()) == expected.as_str().expect("validated hash"), + "export SHA256 mismatch: {name}" + ); + } + Ok(()) +} diff --git a/src/models/jemm/native/src/contract.rs b/src/models/jemm/native/src/contract.rs new file mode 100644 index 00000000..a63147d0 --- /dev/null +++ b/src/models/jemm/native/src/contract.rs @@ -0,0 +1,411 @@ +//! JEMM's ordered single-question prompts and calibrated decision responses. +//! Adapted from ypcypc/JEMM @ 6822fe0 (Apache-2.0). +use anyhow::{Context, Result, bail, ensure}; +use omni_qwen3_5_native::json; +use serde_json::{Map, Value, json}; + +pub const MAX_BODY_BYTES: usize = 16_000_000; +pub const MAX_QUESTIONS: usize = 64; +pub const MAX_COMPILED_PROMPT_BYTES: usize = 16 * 1024 * 1024; +pub const LABELS: &str = "ABCDEFGHIJKLMNOPQRSTUVWXYZ012345"; +pub const SYSTEM: &str = "Choose the best available candidate for the question using only the supplied state. Return exactly one candidate label."; +pub const BASE_REVISION: &str = "1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0"; +pub const CHECKPOINT_REVISION: &str = "76e3c209e8441fa658221c7ba2725bad2f811176"; +pub const TEMPERATURE: f64 = 1.3480874159655591; +pub const MM_TEMPERATURE: f64 = 1.3954832341582943; +pub const THRESHOLD: f64 = 0.9872681877423998; +pub const SOURCE_REVISION: &str = "6822fe0fd53c5e6670af6ba99fb2c857a661e532"; +#[derive(Debug)] +pub struct Question { + pub id: String, + pub kind: String, + pub keys: Vec, + pub prompt: String, +} +pub fn flat(s: &str) -> String { + s.split(|c: char| c.is_whitespace() || matches!(c, '\u{1c}'..='\u{1f}')) + .filter(|s| !s.is_empty()) + .collect::>() + .join(" ") +} +/// Validate integer lexemes before serde can coerce oversized integers to f64. +/// Integer -0 is normalized to Python's integer 0; floating -0.0 stays negative. +pub fn normalize_json_integers(raw: &[u8]) -> Result> { + let mut out = Vec::with_capacity(raw.len()); + let (mut cursor, mut inside, mut escaped) = (0usize, false, false); + while cursor < raw.len() { + let byte = raw[cursor]; + if inside { + out.push(byte); + cursor += 1; + if escaped { + escaped = false; + } else if byte == b'\\' { + escaped = true; + } else if byte == b'"' { + inside = false; + } + continue; + } + if byte == b'"' { + inside = true; + out.push(byte); + cursor += 1; + continue; + } + if byte == b'-' || byte.is_ascii_digit() { + let start = cursor; + while cursor < raw.len() + && !matches!( + raw[cursor], + b' ' | b'\t' | b'\r' | b'\n' | b',' | b']' | b'}' | b':' + ) + { + cursor += 1; + } + let token = &raw[start..cursor]; + let digits = token.strip_prefix(b"-").unwrap_or(token); + if !digits.is_empty() && digits.iter().all(u8::is_ascii_digit) { + let text = std::str::from_utf8(token)?; + ensure!( + if token[0] == b'-' { + text.parse::().is_ok() + } else { + text.parse::().is_ok() + }, + "integer outside i64/u64 range is unsupported" + ); + if token == b"-0" { + out.push(b'0'); + } else { + out.extend(token); + } + } else { + out.extend(token); + } + continue; + } + out.push(byte); + cursor += 1; + } + Ok(out) +} +pub fn parse(raw: &[u8]) -> Result> { + ensure!( + raw.len() <= MAX_BODY_BYTES, + "request exceeds 16000000-byte body limit" + ); + json::parse(&normalize_json_integers(raw)?).map_err(anyhow::Error::msg) +} +/// Preserve Python float formatting and insertion order, with compact separators. +pub fn compact(v: &Value) -> String { + let spaced = json::dumps(v); + let mut out = String::new(); + let (mut inside, mut escape) = (false, false); + for c in spaced.chars() { + if inside { + out.push(c); + if escape { + escape = false; + } else if c == '\\' { + escape = true; + } else if c == '"' { + inside = false; + } + } else if c == '"' { + inside = true; + out.push(c); + } else if c != ' ' { + out.push(c); + } + } + out +} +fn py_repr(v: &Value) -> String { + match v { + Value::String(s) => { + let quote = if s.contains('\'') && !s.contains('"') { + '"' + } else { + '\'' + }; + let mut out = String::from(quote); + for c in s.chars() { + match c { + '\\' => out.push_str("\\\\"), + '\n' => out.push_str("\\n"), + '\r' => out.push_str("\\r"), + '\t' => out.push_str("\\t"), + c if c == quote => { + out.push('\\'); + out.push(c); + } + c if !crate::unicode::is_printable(c) => { + let n = c as u32; + out.push_str(&if n <= 255 { + format!("\\x{n:02x}") + } else if n <= 65535 { + format!("\\u{n:04x}") + } else { + format!("\\U{n:08x}") + }); + } + c => out.push(c), + } + } + out.push(quote); + out + } + Value::Array(a) => format!("[{}]", a.iter().map(py_repr).collect::>().join(", ")), + Value::Object(m) => format!( + "{{{}}}", + m.iter() + .map(|(k, v)| format!("{}: {}", py_repr(&Value::String(k.clone())), py_repr(v))) + .collect::>() + .join(", ") + ), + _ => py_str(v), + } +} +pub fn py_str(v: &Value) -> String { + match v { + Value::Null => "None".into(), + Value::Bool(true) => "True".into(), + Value::Bool(false) => "False".into(), + Value::String(s) => s.clone(), + Value::Number(_) => json::dumps(v), + _ => py_repr(v), + } +} +fn description(spec: &Value) -> String { + if spec.is_object() { + py_str(spec.get("description").unwrap_or(&json!(""))) + } else if spec.is_null() { + String::new() + } else { + py_str(spec) + } +} +fn render_tool(text: &str) -> Result { + // Inspect syntax and the name key without interpreting numeric values. + // RawValue preserves 1e400 even under workspace arbitrary_precision. + let Ok(raw) = serde_json::from_str::>(text) else { + return Ok(flat(text)); + }; + let Ok(tool) = serde_json::from_str::< + std::collections::HashMap>, + >(raw.get()) else { + return Ok(flat(text)); + }; + if !tool.contains_key("name") { + return Ok(flat(text)); + } + let tool = parse(text.as_bytes())?; + let mut result = flat(&py_str(&tool["name"])); + let summary = flat(&py_str(tool.get("description").unwrap_or(&json!("")))); + if !summary.is_empty() { + result.push_str(" — "); + result.push_str(&summary); + } + if let Some(params) = tool.get("parameters").and_then(Value::as_object) { + let props = params + .get("properties") + .and_then(Value::as_object) + .unwrap_or(params); + let required = params.get("required").and_then(Value::as_array); + let mut names = Vec::new(); + for (key, spec) in props { + if ["properties", "required", "type"].contains(&key.as_str()) && !spec.is_object() { + continue; + } + let mut piece = key.clone(); + if spec.is_object() { + if required.is_some_and(|a| a.iter().any(|v| v.as_str() == Some(key))) { + piece.push('*'); + } + if let Some(default) = spec + .get("default") + .filter(|v| !v.is_null() && v.as_str() != Some("")) + { + piece.push('='); + piece.extend(flat(&py_str(default)).chars().take(24)); + } + } + names.push(piece); + } + if !names.is_empty() { + result.push_str(" | params: "); + result.push_str(&names.join(", ")); + } + } + Ok(result) +} +pub fn compile(raw: &[u8]) -> Result> { + let request = parse(raw)?; + ensure!( + matches!(request.get("model"), None | Some(Value::Null)) + || request["model"] + .as_str() + .is_some_and(|m| ["JEMM", "jemm", "MaestroYan/JEMM"].contains(&m)), + "requested model is not loaded" + ); + let questions = request + .get("questions") + .and_then(Value::as_object) + .context("no questions")?; + ensure!( + !questions.is_empty() && questions.len() <= MAX_QUESTIONS, + "questions must contain 1 to 64 entries" + ); + let state = match request.get("state") { + None => std::borrow::Cow::Borrowed(""), + Some(Value::String(s)) => std::borrow::Cow::Borrowed(s.as_str()), + Some(v) => std::borrow::Cow::Owned(compact(v)), + }; + let state = state.trim_matches(|c: char| c.is_whitespace() || matches!(c, '\u{1c}'..='\u{1f}')); + let mut out = Vec::with_capacity(questions.len()); + let mut prompt_bytes = 0; + for (id, q) in questions { + ensure!(q.is_object(), "question {id:?} is not an object"); + let kind = match q.get("type") { + None => "choice", + Some(v) => v.as_str().context("question type must be a string")?, + }; + let criteria = q.get("criteria").unwrap_or(&Value::Null); + let mut candidates: Vec<(String, String, bool)> = Vec::new(); + match kind { + "choice" => { + let map = criteria + .as_object() + .context("choice question needs a criteria map with at least two options")?; + ensure!( + (2..=32).contains(&map.len()), + "UNSUPPORTED: candidate count" + ); + for (key, spec) in map { + let tool = spec + .get("action") + .and_then(Value::as_object) + .is_some_and(|a| a.contains_key("tool_name")); + candidates.push((key.clone(), description(spec), tool)); + } + } + "score" => { + let levels = criteria + .as_array() + .context("score question needs a list of at least two levels")?; + ensure!( + (2..=32).contains(&levels.len()), + "UNSUPPORTED: candidate count" + ); + for (i, l) in levels.iter().enumerate() { + let d = description(l); + candidates.push(( + i.to_string(), + if d.is_empty() { + format!("level {i}") + } else { + d + }, + false, + )); + } + } + "noul" => { + for (key, desc, fallback) in [("yes", "true", "yes"), ("no", "false", "no")] { + let d = description(criteria.get(desc).unwrap_or(&Value::Null)); + candidates.push(( + key.into(), + if d.is_empty() { fallback.into() } else { d }, + false, + )); + } + } + _ => bail!("unsupported question type: {kind:?}"), + } + ensure!( + (2..=32).contains(&candidates.len()), + "UNSUPPORTED: candidate count" + ); + let lines = candidates + .iter() + .enumerate() + .map(|(i, (_, d, tool))| { + Ok(format!( + "{}) {}", + LABELS.as_bytes()[i] as char, + if *tool { render_tool(d)? } else { flat(d) } + )) + }) + .collect::>>()? + .join("\n"); + let instructions = flat(&py_str(q.get("instructions").unwrap_or(&json!("")))); + const OVERHEAD: usize = + "State:\n\n\nQuestion: \n\nCandidates:\n\n\nAnswer with exactly one candidate label." + .len(); + let bytes = [state.len(), instructions.len(), lines.len(), OVERHEAD] + .into_iter() + .try_fold(0usize, |sum, n| { + sum.checked_add(n) + .context("compiled prompt length overflow") + })?; + reserve_prompt_bytes(&mut prompt_bytes, bytes, MAX_COMPILED_PROMPT_BYTES)?; + let prompt = format!( + "State:\n{}\n\nQuestion: {}\n\nCandidates:\n{lines}\n\nAnswer with exactly one candidate label.", + state, instructions + ); + out.push(Question { + id: id.clone(), + kind: kind.into(), + keys: candidates.into_iter().map(|c| c.0).collect(), + prompt, + }); + } + Ok(out) +} +pub fn answer(q: &Question, logits: &[f32], temperature: f64) -> Result { + ensure!( + temperature.is_finite() && temperature > 0., + "invalid temperature" + ); + ensure!( + logits.len() == q.keys.len() && logits.iter().all(|v| v.is_finite()), + "invalid logits" + ); + let max = logits.iter().copied().fold(f32::NEG_INFINITY, f32::max) as f64; + let mut p = logits + .iter() + .map(|&v| ((v as f64 - max) / temperature).exp()) + .collect::>(); + let sum = p.iter().sum::(); + p.iter_mut().for_each(|v| *v /= sum); + let mode = (1..p.len()).fold(0, |m, i| if p[i] > p[m] { i } else { m }); + let probs: Map = q + .keys + .iter() + .cloned() + .zip(p.iter().map(|v| json!(v))) + .collect(); + let confidence = p[mode]; + Ok(match q.kind.as_str() { + "noul" => json!({"type":"noul","noul":p[0],"probabilities":probs,"confidence":confidence}), + "score" => { + json!({"type":"score","expected_value":p.iter().enumerate().map(|(i,p)|i as f64*p).sum::(),"probabilities":probs,"confidence":confidence}) + } + _ => { + json!({"type":"choice","choice":q.keys[mode],"probabilities":probs,"confidence":confidence}) + } + }) +} +/// Reserve aggregate compiled prompt storage before allocating the next prompt. +pub fn reserve_prompt_bytes(total: &mut usize, additional: usize, limit: usize) -> Result<()> { + let reserved = total + .checked_add(additional) + .context("compiled prompt length overflow")?; + ensure!( + reserved <= limit, + "aggregate compiled prompts exceed byte limit {limit}" + ); + *total = reserved; + Ok(()) +} diff --git a/src/models/jemm/native/src/executor.rs b/src/models/jemm/native/src/executor.rs new file mode 100644 index 00000000..5534f33c --- /dev/null +++ b/src/models/jemm/native/src/executor.rs @@ -0,0 +1,287 @@ +//! One prefill per question and BF16 projection of the untied LM-head label rows. +use crate::processing::Input; +use anyhow::{Result, ensure}; +use half::bf16; +use omni_qwen3_5_native::{ + cuda::{self, DeviceBuffer, Stream}, + inputs::MultimodalInput, + model::{Config, Model}, + vision::VisionModel, +}; +use omni_runtime::SerialScheduler; +use safetensors::{Dtype, SafeTensors}; +use serde_json::Value; +use std::{ + ffi::c_void, + panic::{AssertUnwindSafe, catch_unwind}, + path::Path, + sync::{ + Arc, Mutex, + atomic::{AtomicBool, Ordering}, + }, +}; +/// Validate selected LM-head rows, then pad the output dimension for CUDA GEMM. +pub fn head_bytes(data: &[u8], hidden: usize) -> Result> { + let tensors = SafeTensors::deserialize(data)?; + ensure!( + tensors.names().len() == 1, + "expected only selected LM-head weight" + ); + let weight = tensors.tensor("weight")?; + ensure!( + weight.dtype() == Dtype::BF16 && weight.shape() == [32, hidden], + "expected selected BF16 lm_head rows [32,hidden]" + ); + ensure!( + weight + .data() + .as_chunks::<2>() + .0 + .iter() + .all(|b| bf16::from_le_bytes([b[0], b[1]]).is_finite()), + "non-finite LM head" + ); + let mut bytes = vec![0u8; 256 * hidden * 2]; + bytes[..weight.data().len()].copy_from_slice(weight.data()); + Ok(bytes) +} +struct Gemm(*mut c_void); +// SAFETY: each handle is exclusively used under the loaded-state mutex and scheduler. +unsafe impl Send for Gemm {} +impl Drop for Gemm { + fn drop(&mut self) { + // SAFETY: this handle came from cs1_gemm_create and is owned exclusively. + unsafe { (cuda::api().cs1_gemm_destroy)(self.0) } + } +} +struct OwnedStream(Stream); +impl Drop for OwnedStream { + fn drop(&mut self) { + let _ = cuda::synchronize(self.0); + // SAFETY: the owned stream has been synchronized before destruction. + unsafe { + (cuda::api().cs1_stream_destroy)(self.0); + } + } +} +struct LabelHead { + stream: OwnedStream, + gemm: Gemm, + weight: DeviceBuffer, + input: DeviceBuffer, + output: DeviceBuffer, + hidden: usize, +} +impl LabelHead { + fn load(bytes: &[u8], hidden: usize) -> Result { + let stream = OwnedStream(cuda::new_stream()?); + let weight = DeviceBuffer::new(bytes.len())?; + // SAFETY: the weight allocation has exactly bytes.len() bytes. + unsafe { + cuda::upload(weight.at(0), bytes, stream.0)?; + } + // SAFETY: backend initialization returns an owning handle or null. + let gemm = unsafe { (cuda::api().cs1_gemm_create)(32 << 20) }; + ensure!(!gemm.is_null(), "LM head cuBLASLt setup failed"); + Ok(Self { + stream, + gemm: Gemm(gemm), + weight, + input: DeviceBuffer::new(hidden * 2)?, + output: DeviceBuffer::new(256 * 2)?, + hidden, + }) + } + fn score(&mut self, last: &[f32], count: usize) -> Result> { + ensure!( + last.len() == self.hidden && (2..=32).contains(&count), + "invalid hidden state or label count" + ); + let bytes = last + .iter() + .flat_map(|&v| bf16::from_f32(v).to_le_bytes()) + .collect::>(); + // SAFETY: input/weight/output allocations match GEMM dimensions; readback + // waits for queued operations on this exclusively owned stream. + unsafe { + cuda::upload(self.input.at(0), &bytes, self.stream.0)?; + cuda::check( + (cuda::api().cs1_gemm)( + self.gemm.0, + self.input.at(0), + self.weight.at(0), + self.output.at(0), + 1, + 256, + self.hidden as i32, + 256, + self.stream.0, + ), + "BF16 label LM-head GEMM", + )?; + let mut out = vec![0u8; 256 * 2]; + cuda::download(&mut out, self.output.at(0), self.stream.0)?; + let logits = out + .as_chunks::<2>() + .0 + .iter() + .take(count) + .map(|b| bf16::from_le_bytes([b[0], b[1]]).to_f32()) + .collect::>(); + ensure!( + logits.iter().all(|v| v.is_finite()), + "non-finite LM-head logits" + ); + Ok(logits) + } + } + fn synchronize(&self) -> Result<()> { + cuda::synchronize(self.stream.0) + } +} +impl Drop for LabelHead { + fn drop(&mut self) { + let _ = self.synchronize(); + } +} +struct Loaded { + language: Model, + vision: VisionModel, + head: LabelHead, +} +impl Loaded { + fn synchronize(&self) -> Result<()> { + let a = self.language.synchronize(); + let b = self.vision.synchronize(); + let c = self.head.synchronize(); + a.and(b).and(c) + } + fn execute(&mut self, inputs: &[Input]) -> Result>> { + let mut embeddings = Vec::new(); + if let Some(first) = inputs.first() { + for image in first.images.iter() { + embeddings.extend(self.vision.forward(image)?); + } + } + let mut rows = Vec::with_capacity(inputs.len()); + for input in inputs { + let last = if input.images.is_empty() { + self.language.forward(&input.ids)? + } else { + self.language.forward_multimodal(&MultimodalInput { + token_ids: &input.ids, + image_token_indices: &input.image_indices, + image_embeddings: &embeddings, + position_ids: [ + &input.positions[0], + &input.positions[1], + &input.positions[2], + ], + })? + }; + rows.push(self.head.score(&last, input.n_candidates)?); + } + self.synchronize()?; + Ok(rows) + } +} +pub struct Executor { + state: Arc>>, + ready: Arc, +} +impl Executor { + pub async fn load(dir: &Path, library: &Path, manifest: &Value) -> Result { + crate::artifacts::verify_export(dir, manifest)?; + let cfg = Config::load(dir)?; + ensure!( + ( + cfg.hidden, + cfg.intermediate, + cfg.full_attention.len(), + cfg.heads, + cfg.kv_heads, + cfg.lin_k_heads, + cfg.lin_v_heads + ) == (5120, 17408, 64, 24, 4, 16, 48), + "expected Qwen3.8-27B backbone" + ); + ensure!( + manifest["label_head_file"] == "jemm_lm_head.safetensors", + "expected selected LM-head file" + ); + let bytes = head_bytes( + &std::fs::read(dir.join("jemm_lm_head.safetensors"))?, + cfg.hidden, + )?; + let (dir, library) = (dir.to_path_buf(), library.to_path_buf()); + let state = tokio::task::spawn_blocking(move || -> Result { + let language = Model::load(&dir, &library)?; + let vision = VisionModel::load(&dir, &library)?; + let head = LabelHead::load(&bytes, cfg.hidden)?; + Ok(Loaded { + language, + vision, + head, + }) + }) + .await??; + Ok(Self { + state: Arc::new(Mutex::new(Some(state))), + ready: Arc::new(AtomicBool::new(true)), + }) + } + pub fn available(&self) -> bool { + self.ready.load(Ordering::Acquire) + } + /// Admission covers all image, language and head work for one entire request. + pub async fn execute( + &self, + scheduler: &SerialScheduler, + inputs: Vec, + ) -> Result>> { + let state = self.state.clone(); + let ready = self.ready.clone(); + scheduler + .run(move || { + let mut state = match state.lock() { + Ok(state) => state, + Err(poison) => { + ready.store(false, Ordering::Release); + let mut state = poison.into_inner(); + if let Some(loaded) = state.as_ref() { + let _ = loaded.synchronize(); + } + state.take(); + return Err(anyhow::anyhow!("poisoned model state")); + } + }; + guarded( + &mut state, + |loaded| loaded.execute(&inputs), + Loaded::synchronize, + || ready.store(false, Ordering::Release), + ) + }) + .await + } +} + +fn guarded( + state: &mut Option, + work: impl FnOnce(&mut T) -> Result, + synchronize: impl FnOnce(&T) -> Result<()>, + mark_unavailable: impl FnOnce(), +) -> Result { + let Some(loaded) = state.as_mut() else { + mark_unavailable(); + return Err(anyhow::anyhow!("model is unavailable")); + }; + let result = catch_unwind(AssertUnwindSafe(|| work(loaded))) + .unwrap_or_else(|_| Err(anyhow::anyhow!("model execution panicked"))); + if result.is_err() { + mark_unavailable(); + let _ = synchronize(loaded); + state.take(); + } + result +} diff --git a/src/models/jemm/native/src/lib.rs b/src/models/jemm/native/src/lib.rs new file mode 100644 index 00000000..813f1cd8 --- /dev/null +++ b/src/models/jemm/native/src/lib.rs @@ -0,0 +1,6 @@ +pub mod artifacts; +pub mod contract; +pub mod executor; +pub mod processing; +pub mod server; +mod unicode; diff --git a/src/models/jemm/native/src/main.rs b/src/models/jemm/native/src/main.rs new file mode 100644 index 00000000..83503b05 --- /dev/null +++ b/src/models/jemm/native/src/main.rs @@ -0,0 +1,4 @@ +#[tokio::main] +async fn main() -> anyhow::Result<()> { + omni_jemm_native::server::run().await +} diff --git a/src/models/jemm/native/src/processing.rs b/src/models/jemm/native/src/processing.rs new file mode 100644 index 00000000..7b122ce1 --- /dev/null +++ b/src/models/jemm/native/src/processing.rs @@ -0,0 +1,326 @@ +//! Complete-request validation, image transforms, tokenization and response reconstruction. +use crate::contract::{self, Question}; +use anyhow::{Context, Result, ensure}; +use base64::Engine as _; +use image::{ImageFormat, ImageReader, RgbImage}; +use omni_qwen3_5_native::image_preprocess::{ + ImageLimits, ProcessedImage, preprocess_rgb8_with_limits, +}; +use serde_json::{Map, Value, json}; +use std::{path::Path, sync::Arc, time::Instant}; +use tokenizers::Tokenizer; +pub const MAX_IMAGE_PIXELS: usize = 3072 * 32 * 32; +pub fn decode_images(items: &Value) -> Result> { + if items.is_null() { + return Ok(Vec::new()); + } + let items = items + .as_array() + .context("images must be a list of at most 4 base64 strings")?; + ensure!( + items.len() <= 4, + "images must be a list of at most 4 base64 strings" + ); + items + .iter() + .enumerate() + .map(|(index, item)| { + let mut text = item + .as_str() + .with_context(|| format!("image {index} is not a base64 string"))?; + if text.starts_with("data:") { + let (head, data) = text + .split_once(',') + .context("image is not a base64 data URL")?; + ensure!(head.ends_with(";base64"), "image is not a base64 data URL"); + text = data; + } + let cleaned: String = text + .chars() + .filter(|c| !c.is_whitespace() && !matches!(c, '\u{1c}'..='\u{1f}')) + .collect(); + let bytes = base64::engine::general_purpose::STANDARD + .decode(cleaned) + .context("image could not be decoded")?; + let reader = ImageReader::new(std::io::Cursor::new(&bytes)).with_guessed_format()?; + ensure!( + matches!( + reader.format(), + Some(ImageFormat::Png | ImageFormat::Jpeg | ImageFormat::WebP) + ), + "image {index} is not PNG, JPEG or WebP" + ); + let (width, height) = reader.into_dimensions()?; + ensure!( + (width as usize) + .checked_mul(height as usize) + .is_some_and(|n| n <= MAX_IMAGE_PIXELS), + "image {index} exceeds {MAX_IMAGE_PIXELS} pixels" + ); + Ok(ImageReader::new(std::io::Cursor::new(bytes)) + .with_guessed_format()? + .decode()? + .into_rgb8()) + }) + .collect() +} +/// Qwen's image grids consume merged H/W positions; text resumes after the maximum. +pub fn image_positions( + ids: &[u32], + image_token: u32, + grids: &[[usize; 3]], +) -> Result<[Vec; 3]> { + let mut axes: [Vec; 3] = std::array::from_fn(|_| Vec::with_capacity(ids.len())); + let (mut cursor, mut base) = (0usize, 0i64); + for &[t, h, w] in grids { + ensure!( + t == 1 && h > 0 && w > 0 && h % 2 == 0 && w % 2 == 0, + "invalid image grid" + ); + let start = ids[cursor..] + .iter() + .position(|&id| id == image_token) + .map(|i| cursor + i) + .context("missing image placeholders")?; + for _ in cursor..start { + for axis in &mut axes { + axis.push(base); + } + base += 1; + } + let (h, w) = (h / 2, w / 2); + let count = h.checked_mul(w).context("image grid overflow")?; + ensure!( + start.checked_add(count).is_some_and(|end| end <= ids.len()) + && ids[start..start + count] + .iter() + .all(|&id| id == image_token), + "image grid placeholder mismatch" + ); + for row in 0..h { + for col in 0..w { + axes[0].push(base); + axes[1].push(base + row as i64); + axes[2].push(base + col as i64); + } + } + base += h.max(w) as i64; + cursor = start + count; + } + ensure!( + ids[cursor..].iter().all(|&id| id != image_token), + "unexpected image placeholders" + ); + for _ in cursor..ids.len() { + for axis in &mut axes { + axis.push(base); + } + base += 1; + } + Ok(axes) +} +pub struct Input { + pub ids: Vec, + pub n_candidates: usize, + pub images: Arc>, + pub image_indices: Vec, + pub positions: [Vec; 3], +} +pub struct PreparedRequest { + pub inputs: Vec, + pub context: ResponseContext, +} +pub struct ResponseContext { + questions: Vec, + input_tokens: usize, + temperature: f64, + start: Instant, +} +pub struct Processor { + tokenizer: Tokenizer, + prefix: String, + suffix: String, + temperature: f64, + mm_temperature: f64, + image_token: u32, + limits: ImageLimits, +} +impl Processor { + pub fn load(dir: &Path, manifest: &Value) -> Result { + ensure!( + manifest["format"] == "jemm-native/1" + && manifest["model_id"] == "JEMM" + && manifest["base_model_id"] == "Qwen/Qwen3.8-27B" + && manifest["base_revision"] == contract::BASE_REVISION + && manifest["checkpoint_revision"] == contract::CHECKPOINT_REVISION + && manifest["source_revision"] == contract::SOURCE_REVISION, + "expected pinned JEMM native export" + ); + ensure!( + manifest["max_tokens"] == 8192 && manifest["max_mm_tokens"] == 3072, + "expected JEMM token budgets" + ); + let mut tokenizer = + Tokenizer::from_file(dir.join("tokenizer.json")).map_err(anyhow::Error::msg)?; + tokenizer + .with_truncation(None) + .map_err(anyhow::Error::msg)?; + tokenizer.with_padding(None); + let label_ids = manifest["label_token_ids"] + .as_array() + .context("label_token_ids")?; + ensure!( + label_ids.len() == 32 + && label_ids + .iter() + .filter_map(Value::as_u64) + .collect::>() + .len() + == 32, + "expected 32 distinct label IDs" + ); + for (label, id) in contract::LABELS.chars().zip(label_ids) { + let encoded = tokenizer + .encode(label.to_string(), false) + .map_err(anyhow::Error::msg)?; + ensure!( + encoded.get_ids().len() == 1 && Some(encoded.get_ids()[0] as u64) == id.as_u64(), + "label must be exactly one token matching export" + ); + } + let prefix = manifest["chat_prefix"] + .as_str() + .context("chat_prefix")? + .to_owned(); + let suffix = manifest["chat_suffix"] + .as_str() + .context("chat_suffix")? + .to_owned(); + ensure!( + prefix.contains(contract::SYSTEM) && suffix.contains("\n\n"), + "expected JEMM system and disabled-thinking template" + ); + let temperature = manifest["temperature"].as_f64().context("temperature")?; + let mm_temperature = manifest["mm_temperature"] + .as_f64() + .context("mm_temperature")?; + ensure!( + temperature == contract::TEMPERATURE + && mm_temperature == contract::MM_TEMPERATURE + && manifest["threshold"].as_f64() == Some(contract::THRESHOLD), + "expected pinned JEMM calibration" + ); + let config: Value = serde_json::from_slice(&std::fs::read(dir.join("config.json"))?)?; + let image_token = config["image_token_id"] + .as_u64() + .and_then(|x| u32::try_from(x).ok()) + .context("image_token_id")?; + let preprocessor: Value = + serde_json::from_slice(&std::fs::read(dir.join("preprocessor_config.json"))?)?; + ensure!( + preprocessor["size"]["shortest_edge"] == 65536 + && preprocessor["size"]["longest_edge"] == 16777216, + "expected pinned image processor size" + ); + Ok(Self { + tokenizer, + prefix, + suffix, + temperature, + mm_temperature, + image_token, + limits: ImageLimits { + max_source_side: 32768, + max_source_pixels: MAX_IMAGE_PIXELS, + min_pixels: 65536, + max_pixels: 16777216, + max_patches: 12288, + }, + }) + } + pub fn prepare(&self, raw: &[u8]) -> Result { + let start = Instant::now(); + let request = contract::parse(raw)?; + let questions = contract::compile(raw)?; + let images = decode_images(request.get("images").unwrap_or(&Value::Null))? + .into_iter() + .map(|rgb| { + preprocess_rgb8_with_limits( + rgb.width() as usize, + rgb.height() as usize, + rgb.as_raw(), + &self.limits, + ) + }) + .collect::>>()?; + let images = Arc::new(images); + let grids = images.iter().map(|i| i.image_grid_thw).collect::>(); + let mut inputs = Vec::new(); + let mut input_tokens = 0; + for q in &questions { + let image_prefix = images + .iter() + .map(|image| { + format!( + "<|vision_start|>{}<|vision_end|>", + "<|image_pad|>".repeat(image.image_tokens()) + ) + }) + .collect::(); + let chat = format!("{}{image_prefix}{}{}", self.prefix, q.prompt, self.suffix); + let encoded = self + .tokenizer + .encode(chat, false) + .map_err(anyhow::Error::msg)?; + let ids = encoded.get_ids().to_vec(); + let budget = if images.is_empty() { 8192 } else { 3072 }; + ensure!( + !ids.is_empty() && ids.len() <= budget, + "UNSUPPORTED: input exceeds token budget" + ); + let image_indices = ids + .iter() + .enumerate() + .filter_map(|(i, &v)| (!images.is_empty() && v == self.image_token).then_some(i)) + .collect::>(); + let positions = if images.is_empty() { + std::array::from_fn(|_| (0..ids.len()).map(|i| i as i64).collect()) + } else { + image_positions(&ids, self.image_token, &grids)? + }; + input_tokens += ids.len(); + inputs.push(Input { + ids, + n_candidates: q.keys.len(), + images: images.clone(), + image_indices, + positions, + }); + } + Ok(PreparedRequest { + inputs, + context: ResponseContext { + questions, + input_tokens, + temperature: if images.is_empty() { + self.temperature + } else { + self.mm_temperature + }, + start, + }, + }) + } +} +impl ResponseContext { + pub fn finish(self, rows: Vec>) -> Result { + ensure!(rows.len() == self.questions.len(), "invalid logit rows"); + let mut answers = Map::new(); + for (q, row) in self.questions.iter().zip(rows) { + answers.insert(q.id.clone(), contract::answer(q, &row, self.temperature)?); + } + Ok( + json!({"model":"JEMM","answers":answers,"usage":{"input_tokens":self.input_tokens,"output_tokens":0,"latency_ms":self.start.elapsed().as_secs_f64()*1000.}}), + ) + } +} diff --git a/src/models/jemm/native/src/server.rs b/src/models/jemm/native/src/server.rs new file mode 100644 index 00000000..4c33e19b --- /dev/null +++ b/src/models/jemm/native/src/server.rs @@ -0,0 +1,131 @@ +//! JEMM HTTP transport and model readiness. +use crate::{executor::Executor, processing::Processor}; +use anyhow::{Context, Result, ensure}; +use axum::{ + Json, Router, + body::Bytes, + extract::{DefaultBodyLimit, State, rejection::BytesRejection}, + http::{HeaderMap, StatusCode}, + response::{IntoResponse, Response}, + routing::{get, post}, +}; +use omni_qwen3_5_native::cuda; +use omni_runtime::SerialScheduler; +use serde_json::{Value, json}; +use std::{path::Path, sync::Arc}; +const WARMUP:&[u8]=br#"{"state":"Update installed.","questions":{"q":{"instructions":"Choose the action.","criteria":{"close":"Close dialog","wait":"Wait"}}}}"#; +pub struct Engine { + pub processor: Processor, + pub executor: Executor, + pub scheduler: SerialScheduler, +} +fn manifest(dir: &Path) -> Result { + Ok(serde_json::from_slice( + &std::fs::read(dir.join("jemm_export.json")) + .context("export JEMM before launching the native worker")?, + )?) +} +impl Engine { + pub async fn load(dir: &Path, library: &Path) -> Result { + let export = manifest(dir)?; + let processor = Processor::load(dir, &export)?; + let executor = Executor::load(dir, library, &export).await?; + let engine = Self { + processor, + executor, + scheduler: SerialScheduler::default(), + }; + let prepared = engine.processor.prepare(WARMUP)?; + let rows = engine + .executor + .execute(&engine.scheduler, prepared.inputs) + .await?; + prepared.context.finish(rows)?; + ensure!(engine.executor.available(), "warmup retired the model"); + Ok(engine) + } +} +fn error(status: StatusCode, message: impl ToString) -> Response { + (status, Json(json!({"error":message.to_string()}))).into_response() +} +pub async fn decide(engine: &Engine, raw: &[u8]) -> Response { + if !engine.executor.available() { + return error(StatusCode::SERVICE_UNAVAILABLE, "model is unavailable"); + } + let prepared = match engine.processor.prepare(raw) { + Ok(p) => p, + Err(e) => return error(StatusCode::UNPROCESSABLE_ENTITY, e), + }; + match engine + .executor + .execute(&engine.scheduler, prepared.inputs) + .await + .and_then(|rows| prepared.context.finish(rows)) + { + Ok(body) => Json(body).into_response(), + Err(e) => { + eprintln!("inference failed: {e:#}"); + error(StatusCode::SERVICE_UNAVAILABLE, "model inference failed") + } + } +} +async fn systemone( + State(engine): State>, + headers: HeaderMap, + body: Result, +) -> Response { + let content_type = headers + .get("content-type") + .and_then(|v| v.to_str().ok()) + .unwrap_or("") + .split(';') + .next() + .unwrap_or("") + .trim(); + if !content_type.eq_ignore_ascii_case("application/json") { + return error( + StatusCode::UNSUPPORTED_MEDIA_TYPE, + "Content-Type must be application/json", + ); + } + match body { + Ok(raw) => decide(&engine, &raw).await, + Err(e) => error(e.status(), e.body_text()), + } +} +pub fn router(engine: Arc) -> Router { + Router::new() + .route( + "/health", + get(|State(engine): State>| async move { + if engine.executor.available() { + Json(json!({"status":"ready","model":"JEMM"})).into_response() + } else { + error(StatusCode::SERVICE_UNAVAILABLE, "model is unavailable") + } + }), + ) + .route("/v1/systemone", post(systemone)) + .layer(DefaultBodyLimit::max(crate::contract::MAX_BODY_BYTES)) + .with_state(engine) +} +pub async fn run() -> Result<()> { + let dir = std::env::var_os("JEMM_MODEL").context("set JEMM_MODEL to the native export")?; + let dir = Path::new(&dir); + ensure!( + std::env::args_os().len() == 1, + "usage: omni-jemm-native (configure using JEMM_MODEL and JEMM_CUDA_LIB)" + ); + let library = std::env::var_os("JEMM_CUDA_LIB") + .map(Into::into) + .map_or_else(cuda::default_library, Ok)?; + let engine = Arc::new(Engine::load(dir, &library).await?); + let host = std::env::var("JEMM_HOST").unwrap_or_else(|_| "127.0.0.1".into()); + let port: u16 = std::env::var("JEMM_PORT") + .map_or(Ok(8000), |v| v.parse()) + .context("JEMM_PORT")?; + let listener = tokio::net::TcpListener::bind((host.as_str(), port)).await?; + eprintln!("JEMM ready on {host}:{port}"); + axum::serve(listener, router(engine)).await?; + Ok(()) +} diff --git a/src/models/jemm/native/src/unicode.rs b/src/models/jemm/native/src/unicode.rs new file mode 100644 index 00000000..096d15b4 --- /dev/null +++ b/src/models/jemm/native/src/unicode.rs @@ -0,0 +1,21 @@ +//! Python 3.12 printability from Unicode 15.0 general categories. +//! Source: https://www.unicode.org/Public/15.0.0/ucd/UnicodeData.txt +//! Source SHA256: 806e9aed65037197f1ec85e12be6e8cd870fc5608b4de0fffd990f689f376a73. +//! Table stores alternating printable range boundaries as little-endian u32. +//! Data license: ../LICENSE.unicode. +const BOUNDARIES: &[u8] = include_bytes!("unicode15_printable.bin"); +pub fn is_printable(c: char) -> bool { + let mut low = 0usize; + let mut high = BOUNDARIES.len() / 4; + while low < high { + let middle = (low + high) / 2; + let bytes = &BOUNDARIES[middle * 4..middle * 4 + 4]; + let point = u32::from_le_bytes([bytes[0], bytes[1], bytes[2], bytes[3]]); + if point <= c as u32 { + low = middle + 1; + } else { + high = middle; + } + } + low % 2 == 1 +} diff --git a/src/models/jemm/native/src/unicode15_printable.bin b/src/models/jemm/native/src/unicode15_printable.bin new file mode 100644 index 0000000000000000000000000000000000000000..75f0b26afb772a1d0f03493f6b181fd6d02b828b GIT binary patch literal 5688 zcmWmI33OD|835orAd4Wo3o4t%1zdxoU=m5HvII086QARmW@iqTu#_=J{WFF-_#^bD@g4I;=I&ZO-b!?=X zkJ!Ri>e#_fcJU><*+V0dRv{cmGg{M*4s@b7=hBCZ>C2_`qd&Ria|Hvqiop~yjAE|i zdPXyjQpPip`+1ld%w#_0EZ}igP{C>{d4o4u%Q`ku&1SYxOPXzD*v%gHahTBB`*Ix3 zXhSobf6=>IhS1W8OR`t7)miWGLlh@ zVH~Bzn7~BJxSx5<=P?$rf(qW?P1drGjcnp0HdD)|)bRyhl3_P{$nrD$32pq)mJW1e zAcH7kC{czpfr-RJAuA&aTxb=I+-4OCIhX11`EPx*{Gc93BYS@v;&NZSzdC}1oP z^LJ)2lYcOWxxC0ryv#B-P{l?zQB4h7NmI{GGK6*^455h0+{sfcVm%x97qxs&Xs=JI zso`JLlBTGmcu`E0k&I$AC6p3l7PFa8g4I;=7HfEyD&FILHc>+zJLu5KI&m_c8N)cH z5oa+;4swX!2q(ICvN*DYB!>y7=!fp~qmbdmn9LOJOlyNU{ULnP1(tOS?x||k5H^viV8q=9gg2g0RMT)J2 z)72EBOefBMj?%x2oMR~0GK&Q7u$~&y93gZSx9(n>ZuDRfLntQ7C`MDl7{)S=QpWQT zao*%DHnNHBe8x_`Aj>`uaF9mY_XwdEIgF=_ySRsP5-cIfd!+f9aE3iYf^*IkcZLz; zp7TPuk0)8k{`2JsXJ6o1dT{|4lEWnK;7+EqkR;1k&TFK2n>D;k71h+Rl{9ti;7c-; zUMMas=4tj5E)o;693_%t?);vGEF$z#f5@Q^4k%4PH?j{=H_GMX4k zs`!Sz{7Pu;_qd!9jN)#lvVcV_VFf9|3Hl|@>s0Xz4fJYbUC1TYHWH>2r-GHNB1I*~ z=+(~ilu*WOQv8z)zY^NZPp;=arn8<6d_bD-$kIUQ5DAxZ8AFJ2KXEdA#Q{RcNa#o> zt|ZPQJjyI)GoNyvOG{eOnttSxM?M26Ul+DihVS)xTojObe2)Zy~MftEaO}x-h>=$(bw6)2*TCk zImmUA_};3W@x-{5DZEP+n@RH{Sq>6z6E~uiFov7BnW;=8LE)50c$#pB9OVx@N;yd~ z{D&<22xXCQ=n>aPo#%61=UFeZuQ~HLz)_CTNJvFO&(-z^mk{L!Ze#=#m_!*zD?Pt1 z5^kf6X~daDg2g;TlCU)rE+mJ+MEMW}D;y+jQxECQ-8{%%!pGu572Cw z+s%;?Ea6#_yueakVi_xVl~t_e9oDg)57|ibPvs`9X~RjJOjo**OCALjGLS(GW+>Nj zJuiPI=lN${B-Apu-rAAmc~XQg)f&#?JVM6UwB%Gy=XdnvEY9XUa+pFn3Bp%uIynrW zkf9V4CGmA6yharbG!p(T4|n@a<{Nd0Z~2}d_=zlygm1-$YU&8zMZz^iDWQ}YcYYrU zZxZ>zJP1EJk7-5^&Y&k}au(;%i{4y9U;1-7x#W@0m0ZPOhET*%hH(wo66HE>U^q82 zl2MGNgfWa|9HopW#vi$v3H*s$xQ*Mnn|qkb15D!~rW0p2b9jd5d6Ac>U?uCwvX78; zj+4U^BzcY1RPi2ZwsDZ~vs`5lP4`7Yd-5ryh$z<+VN1WL_ z!4|SK5JHoXdAW)IdzysYx0{4Qx~*vvdhkMRlaS)8z3v~<-ryRVs@YA|>!u;q)0e)! HsLAp_0A+7F literal 0 HcmV?d00001 From 82cbe7928249a55e29abffe658abc25d70f9f458 Mon Sep 17 00:00:00 2001 From: levius <2114377220@qq.com> Date: Thu, 8 Oct 2026 15:52:32 +0800 Subject: [PATCH 02/21] jemm: load verified calibration and return typed errors --- recipe/jemm/export.py | 16 ++++++---- src/models/jemm/native/src/artifacts.rs | 1 + src/models/jemm/native/src/contract.rs | 3 -- src/models/jemm/native/src/processing.rs | 37 +++++++++++++++++------- src/models/jemm/native/src/server.rs | 8 ++++- 5 files changed, 45 insertions(+), 20 deletions(-) diff --git a/recipe/jemm/export.py b/recipe/jemm/export.py index 54cc23a1..61d8e94c 100644 --- a/recipe/jemm/export.py +++ b/recipe/jemm/export.py @@ -11,6 +11,7 @@ import hashlib import importlib.metadata import json +import math import os import platform import shutil @@ -30,8 +31,6 @@ LABELS = "ABCDEFGHIJKLMNOPQRSTUVWXYZ012345" SYSTEM = ("Choose the best available candidate for the question using only the supplied state. " "Return exactly one candidate label.") -CALIBRATION = {"temperature": 1.3480874159655591, "mm_temperature": 1.3954832341582943, - "threshold": 0.9872681877423998} INVENTORY_PATH = Path(__file__).with_name("pinned_inventory.json") @@ -191,9 +190,14 @@ def export_checkpoint(base, adapter, out, provenance, *, tokenizer=None, invento raise ValueError("untied head shape disagrees with text configuration") adapter_config = json.loads((adapter / "adapter_config.json").read_text()) pairs, scale = adapter_pairs(inventory["base"]["tensors"], inventory["adapter"]["tensors"], adapter_config) - calibration = json.loads((adapter / "decision_config.json").read_text()) - if any(calibration.get(key) != value for key, value in CALIBRATION.items()): - raise ValueError("decision calibration differs from pinned JEMM") + decision = json.loads((adapter / "decision_config.json").read_text()) + calibration = {} + for key in ("temperature", "mm_temperature", "threshold"): + value = decision.get(key) if isinstance(decision, dict) else None + if type(value) not in (int, float) or not math.isfinite(value) or ( + value <= 0 if key != "threshold" else not 0 <= value <= 1): + raise ValueError(f"invalid decision calibration: {key}") + calibration[key] = value if tokenizer is None: from transformers import AutoTokenizer tokenizer = AutoTokenizer.from_pretrained(str(base), local_files_only=True) @@ -305,7 +309,7 @@ def export_checkpoint(base, adapter, out, provenance, *, tokenizer=None, invento exports = {p.name: sha256(p) for p in sorted(out.iterdir()) if p.is_file() and not p.name.endswith(".tmp") and p.name != "export_progress.json"} if "jemm_lm_head.safetensors" not in exports or "vision.safetensors" not in exports: raise ValueError("export lacks selected head or vision tensors") - manifest = {"format": "jemm-native/1", "model_id": "JEMM", **PINS, **CALIBRATION, + manifest = {"format": "jemm-native/1", "model_id": "JEMM", **PINS, **calibration, "max_tokens": 8192, "max_mm_tokens": 3072, "chat_prefix": chat_prefix, "chat_suffix": chat_suffix, "enable_thinking": False, "preserve_thinking": False, "labels": list(LABELS), "label_token_ids": label_ids, "label_head_file": "jemm_lm_head.safetensors", "label_head_tensor": "weight", "label_head_dtype": "BF16", diff --git a/src/models/jemm/native/src/artifacts.rs b/src/models/jemm/native/src/artifacts.rs index b3abe60d..345efcd3 100644 --- a/src/models/jemm/native/src/artifacts.rs +++ b/src/models/jemm/native/src/artifacts.rs @@ -42,6 +42,7 @@ pub fn verify_export(dir: &Path, manifest: &Value) -> Result<()> { "config.json", "tokenizer.json", "preprocessor_config.json", + "decision_config.json", "model.safetensors.index.json", "vision.safetensors", "jemm_lm_head.safetensors", diff --git a/src/models/jemm/native/src/contract.rs b/src/models/jemm/native/src/contract.rs index a63147d0..ec26403d 100644 --- a/src/models/jemm/native/src/contract.rs +++ b/src/models/jemm/native/src/contract.rs @@ -11,9 +11,6 @@ pub const LABELS: &str = "ABCDEFGHIJKLMNOPQRSTUVWXYZ012345"; pub const SYSTEM: &str = "Choose the best available candidate for the question using only the supplied state. Return exactly one candidate label."; pub const BASE_REVISION: &str = "1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0"; pub const CHECKPOINT_REVISION: &str = "76e3c209e8441fa658221c7ba2725bad2f811176"; -pub const TEMPERATURE: f64 = 1.3480874159655591; -pub const MM_TEMPERATURE: f64 = 1.3954832341582943; -pub const THRESHOLD: f64 = 0.9872681877423998; pub const SOURCE_REVISION: &str = "6822fe0fd53c5e6670af6ba99fb2c857a661e532"; #[derive(Debug)] pub struct Question { diff --git a/src/models/jemm/native/src/processing.rs b/src/models/jemm/native/src/processing.rs index 7b122ce1..55827624 100644 --- a/src/models/jemm/native/src/processing.rs +++ b/src/models/jemm/native/src/processing.rs @@ -200,16 +200,33 @@ impl Processor { prefix.contains(contract::SYSTEM) && suffix.contains("\n\n"), "expected JEMM system and disabled-thinking template" ); - let temperature = manifest["temperature"].as_f64().context("temperature")?; - let mm_temperature = manifest["mm_temperature"] - .as_f64() - .context("mm_temperature")?; - ensure!( - temperature == contract::TEMPERATURE - && mm_temperature == contract::MM_TEMPERATURE - && manifest["threshold"].as_f64() == Some(contract::THRESHOLD), - "expected pinned JEMM calibration" - ); + let calibration: Value = serde_json::from_slice( + &std::fs::read(dir.join("decision_config.json")).context("decision_config.json")?, + )?; + let mut values = [0.0; 3]; + for (index, key) in ["temperature", "mm_temperature", "threshold"] + .iter() + .enumerate() + { + let value = calibration[key] + .as_f64() + .with_context(|| format!("calibration {key}"))?; + ensure!( + value.is_finite() + && if index < 2 { + value > 0.0 + } else { + (0.0..=1.0).contains(&value) + }, + "invalid calibration {key}" + ); + ensure!( + manifest[key].as_f64() == Some(value), + "manifest disagrees with decision_config.json: {key}" + ); + values[index] = value; + } + let [temperature, mm_temperature, _threshold] = values; let config: Value = serde_json::from_slice(&std::fs::read(dir.join("config.json"))?)?; let image_token = config["image_token_id"] .as_u64() diff --git a/src/models/jemm/native/src/server.rs b/src/models/jemm/native/src/server.rs index 4c33e19b..6273be3b 100644 --- a/src/models/jemm/native/src/server.rs +++ b/src/models/jemm/native/src/server.rs @@ -46,7 +46,13 @@ impl Engine { } } fn error(status: StatusCode, message: impl ToString) -> Response { - (status, Json(json!({"error":message.to_string()}))).into_response() + let kind = match status { + StatusCode::UNPROCESSABLE_ENTITY | StatusCode::PAYLOAD_TOO_LARGE => "ValueError", + StatusCode::UNSUPPORTED_MEDIA_TYPE => "TypeError", + _ => "RuntimeError", + }; + let detail: String = message.to_string().chars().take(200).collect(); + (status, Json(json!({"error":kind,"detail":detail}))).into_response() } pub async fn decide(engine: &Engine, raw: &[u8]) -> Response { if !engine.executor.available() { From 589a96c94535e8f0bb3f675571e1e9de2b773101 Mon Sep 17 00:00:00 2001 From: levius <2114377220@qq.com> Date: Thu, 8 Oct 2026 16:18:41 +0800 Subject: [PATCH 03/21] qwen: add shared 27B vision and multimodal CUDA Graph replay --- src/models/qwen3_5/native/src/model.rs | 79 ++++- src/models/qwen3_5/native/src/vision/model.rs | 273 +++++++++++++----- 2 files changed, 263 insertions(+), 89 deletions(-) diff --git a/src/models/qwen3_5/native/src/model.rs b/src/models/qwen3_5/native/src/model.rs index 6d37cd2f..b243f229 100644 --- a/src/models/qwen3_5/native/src/model.rs +++ b/src/models/qwen3_5/native/src/model.rs @@ -563,9 +563,10 @@ pub struct Model { gemm: *mut c_void, /// Buffers for the largest packed token count so far; grows as needed. scratch: Option, - /// Opt-in replay with at most 64 captures keyed by ordered sequence lengths. + /// Opt-in replay with at most 64 captures per mode, keyed by sequence lengths. graph_enabled: bool, graphs: VecDeque<(Vec, cuda::Graph)>, + mm_graphs: VecDeque<(Vec, cuda::Graph)>, } // SAFETY: the raw pointers are device addresses and a cuBLASLt handle owned by the @@ -575,6 +576,7 @@ unsafe impl Send for Model {} impl Drop for Model { fn drop(&mut self) { self.graphs.clear(); + self.mm_graphs.clear(); // SAFETY: created by cs1_gemm_create and not destroyed before. unsafe { (cuda::api().cs1_gemm_destroy)(self.gemm) }; } @@ -656,6 +658,7 @@ impl Model { scratch: None, graph_enabled: std::env::var("CUA_S1_GRAPH").as_deref() == Ok("1"), graphs: VecDeque::new(), + mm_graphs: VecDeque::new(), }; Ok(model) } @@ -709,6 +712,7 @@ impl Model { cuda::set_device(0)?; if self.scratch.as_ref().is_none_or(|s| t > s.cap) { self.graphs.clear(); + self.mm_graphs.clear(); self.scratch = None; self.scratch = Some(Scratch::new( &self.cfg, @@ -757,28 +761,21 @@ impl Model { if self.graph_enabled { if let Some((_, graph)) = self.graphs.iter().find(|(shape, _)| *shape == lengths) { graph.launch(self.stream)?; + if std::env::var("CUA_S1_GRAPH_TRACE").as_deref() == Ok("1") { + eprintln!("CUA_S1_GRAPH mode=text event=replay tokens={t}"); + } } else { // Warm plans and keep the eager result: run() advances the residual // in place, so replaying on this cache miss would advance it twice. self.run(s, &lengths, false)?; cuda::synchronize(self.stream)?; - match cuda::Graph::capture(self.stream, || self.run(s, &lengths, false)) { - Ok(graph) => { - if self.graphs.len() == 64 { - self.graphs.pop_front(); - } - self.graphs.push_back((lengths.clone(), graph)); - } - Err(error) => { - eprintln!("CUDA Graph capture failed; using eager execution: {error:#}"); - self.graph_enabled = false; - self.graphs.clear(); - } - } + let captured = cuda::Graph::capture(self.stream, || self.run(s, &lengths, false)); + self.cache_graph(&lengths, false, captured); } } else { self.run(s, &lengths, false)?; } + let s = self.scratch.as_ref().unwrap(); let mut end = 0; lengths .iter() @@ -833,8 +830,58 @@ impl Model { } begin = end; } - self.run(s, &[t], true)?; - self.last_hidden(s, t) + if self.graph_enabled { + if let Some((_, graph)) = self + .mm_graphs + .iter() + .find(|(shape, _)| shape.as_slice() == [t]) + { + graph.launch(self.stream)?; + if std::env::var("CUA_S1_GRAPH_TRACE").as_deref() == Ok("1") { + eprintln!("CUA_S1_GRAPH mode=multimodal event=replay tokens={t}"); + } + } else { + // Uploads and embedding are outside capture. Keep the warmed eager + // result; launching now would advance the residual a second time. + self.run(s, &[t], true)?; + cuda::synchronize(self.stream)?; + let captured = cuda::Graph::capture(self.stream, || self.run(s, &[t], true)); + self.cache_graph(&[t], true, captured); + } + } else { + self.run(s, &[t], true)?; + } + self.last_hidden(self.scratch.as_ref().unwrap(), t) + } + + /// Both modes retain the eager miss result; capture records without executing. + fn cache_graph(&mut self, lengths: &[usize], multimodal: bool, captured: Result) { + let mode = if multimodal { "multimodal" } else { "text" }; + match captured { + Ok(graph) => { + let cache = if multimodal { + &mut self.mm_graphs + } else { + &mut self.graphs + }; + if cache.len() == 64 { + cache.pop_front(); + } + cache.push_back((lengths.to_vec(), graph)); + if std::env::var("CUA_S1_GRAPH_TRACE").as_deref() == Ok("1") { + let t: usize = lengths.iter().sum(); + eprintln!("CUA_S1_GRAPH mode={mode} event=capture tokens={t}"); + } + } + Err(error) => { + eprintln!( + "CUDA Graph capture failed in {mode} mode; using eager execution: {error:#}" + ); + self.graph_enabled = false; + self.graphs.clear(); + self.mm_graphs.clear(); + } + } } fn upload_positions(&self, s: &Scratch, positions: [&[i64]; 3]) -> Result<()> { diff --git a/src/models/qwen3_5/native/src/vision/model.rs b/src/models/qwen3_5/native/src/vision/model.rs index c2821b61..c4f71638 100644 --- a/src/models/qwen3_5/native/src/vision/model.rs +++ b/src/models/qwen3_5/native/src/vision/model.rs @@ -7,7 +7,11 @@ use crate::{ use anyhow::{Result, ensure}; use half::bf16; use safetensors::{Dtype, tensor::TensorView}; -use std::{collections::BTreeMap, ffi::c_void, path::Path}; +use std::{ + collections::{BTreeMap, VecDeque}, + ffi::c_void, + path::Path, +}; type Trace<'a> = Option<&'a mut dyn FnMut(&str, &[bf16]) -> Result<()>>; @@ -30,10 +34,13 @@ impl Drop for Gemm { /// One request at a time; returns row-major [image_tokens, config.out_hidden_size]. pub struct VisionModel { config: VisionConfig, + // Captures and their buffers retire before weights, GEMM workspace and stream. + scratch: VecDeque, + graph_enabled: bool, base: BTreeMap, lora: BTreeMap, - stream: OwnedStream, gemm: Gemm, + stream: OwnedStream, } impl VisionModel { pub fn load(dir: impl AsRef, library: &Path) -> Result { @@ -119,6 +126,8 @@ impl VisionModel { ensure!(!gemm.0.is_null(), "cannot create vision cuBLAS handle"); let mut model = Self { config, + scratch: VecDeque::new(), + graph_enabled: std::env::var("CUA_S1_VISION_GRAPH").as_deref() == Ok("1"), base: BTreeMap::new(), lora: BTreeMap::new(), stream, @@ -308,11 +317,15 @@ impl VisionModel { } fn run(&mut self, image: &ProcessedImage, mut callback: Trace<'_>) -> Result> { cuda::set_device(0)?; - let geo = VisionGeometry::new(image.image_grid_thw, &self.config)?; - let h = self.config.hidden_size; - let intermediate = self.config.intermediate_size; - let merged = h * 4; - let output = self.config.out_hidden_size; + let cached = self + .scratch + .iter() + .position(|s| s.grid == image.image_grid_thw); + let geometry = if cached.is_none() { + Some(VisionGeometry::new(image.image_grid_thw, &self.config)?) + } else { + None + }; let n = image.image_grid_thw[1] * image.image_grid_thw[2]; ensure!( image.pixel_values.len() == n * 1536, @@ -332,50 +345,82 @@ impl VisionModel { .iter() .flat_map(|v| bf16::from_f32(*v).to_bits().to_le_bytes()) .collect(); - let pixels = self.upload(&pixels)?; - let indices = self.upload( - &geo.indices - .iter() - .flat_map(|v| v.to_le_bytes()) - .collect::>(), - )?; - let weights = self.upload( - &geo.weights - .iter() - .flat_map(|v| v.to_le_bytes()) - .collect::>(), - )?; - let co = self.upload( - &geo.cos - .iter() - .flat_map(|v| v.to_le_bytes()) - .collect::>(), - )?; - let si = self.upload( - &geo.sin - .iter() - .flat_map(|v| v.to_le_bytes()) - .collect::>(), - )?; - let w = Work::new( - if self.lora.is_empty() { 0 } else { n }, - intermediate.max(merged).max(output), - )?; - let attention_workspace = DeviceBuffer::new(if h == 1024 { - 0 + let index = if let Some(index) = cached { + index } else { - n * self.config.num_heads * 80 * 4 * 2 - })?; - let x = DeviceBuffer::new(n * h * 2)?; - let norm = DeviceBuffer::new(n * h * 2)?; - let qkv = DeviceBuffer::new(n * h * 3 * 2)?; - let q = DeviceBuffer::new(n * h * 2)?; - let k = DeviceBuffer::new(n * h * 2)?; - let attn = DeviceBuffer::new(n * h * 2)?; - let delta = DeviceBuffer::new(n * h * 2)?; - let mlp = DeviceBuffer::new(n * intermediate.max(merged) * 2)?; - self.linear("patch_embed.proj", &pixels, &x, n, h, 1536, &w)?; - self.trace(&mut callback, "patch_embed", &x, n * h)?; + self.synchronize()?; + // FIFO is bounded to four exact grids; graph drops before its buffers. + if self.scratch.len() == 4 { + self.scratch.pop_front(); + } + let scratch = Scratch::new(self, image.image_grid_thw, geometry.unwrap())?; + self.scratch.push_back(scratch); + self.scratch.len() - 1 + }; + let scratch = &self.scratch[index]; + // SAFETY: the current image has the validated exact resident buffer size. + unsafe { + cuda::upload(scratch.pixels.at(0), &pixels, self.stream.0)?; + } + if callback.is_none() + && self.graph_enabled + && let Some(graph) = &scratch.graph + { + graph.launch(self.stream.0)?; + graph_trace("replayed", scratch.grid); + return self.read(&scratch.out, n / 4 * self.config.out_hidden_size); + } + self.enqueue(scratch, &mut callback)?; + let result = self.read(&scratch.out, n / 4 * self.config.out_hidden_size)?; + if let Some(callback) = callback.as_mut() { + callback("merger.output", &result)?; + } else if self.graph_enabled { + // Readback completed warmup. Capture records without executing; keep + // the eager result even when recording/instantiation fails. + let captured = cuda::Graph::capture(self.stream.0, || self.enqueue(scratch, &mut None)); + match captured { + Ok(graph) => { + self.scratch[index].graph = Some(graph); + graph_trace("captured", image.image_grid_thw); + } + Err(error) => { + eprintln!("Vision CUDA Graph capture failed; using eager execution: {error:#}"); + self.graph_enabled = false; + for scratch in &mut self.scratch { + scratch.graph = None; + } + } + } + } + Ok(result) + } + fn enqueue(&self, scratch: &Scratch, callback: &mut Trace<'_>) -> Result<()> { + let n = scratch.grid[1] * scratch.grid[2]; + let h = self.config.hidden_size; + let intermediate = self.config.intermediate_size; + let merged = h * 4; + let output = self.config.out_hidden_size; + let Scratch { + pixels, + indices, + weights, + co, + si, + w, + attention_workspace, + x, + norm, + qkv, + q, + k, + attn, + delta, + mlp, + out, + .. + } = scratch; + self.linear("patch_embed.proj", pixels, x, n, h, 1536, w)?; + self.trace(callback, "patch_embed", x, n * h)?; unsafe { if h == 1024 { check( @@ -404,11 +449,11 @@ impl VisionModel { )?; } } - self.trace(&mut callback, "position", &x, n * h)?; + self.trace(callback, "position", x, n * h)?; for i in 0..self.config.depth { let p = format!("blocks.{i}"); - self.norm(&format!("{p}.norm1"), &x, &norm, n)?; - self.linear(&format!("{p}.attn.qkv"), &norm, &qkv, n, h * 3, h, &w)?; + self.norm(&format!("{p}.norm1"), x, norm, n)?; + self.linear(&format!("{p}.attn.qkv"), norm, qkv, n, h * 3, h, w)?; unsafe { if h == 1024 { check( @@ -466,22 +511,22 @@ impl VisionModel { )?; } } - self.linear(&format!("{p}.attn.proj"), &attn, &delta, n, h, h, &w)?; + self.linear(&format!("{p}.attn.proj"), attn, delta, n, h, h, w)?; unsafe { check( (api().cs1_vision_add)(x.at(0), delta.at(0), n * h, self.stream.0), "vision attention residual", )?; } - self.norm(&format!("{p}.norm2"), &x, &norm, n)?; + self.norm(&format!("{p}.norm2"), x, norm, n)?; self.linear( &format!("{p}.mlp.linear_fc1"), - &norm, - &mlp, + norm, + mlp, n, intermediate, h, - &w, + w, )?; unsafe { check( @@ -491,12 +536,12 @@ impl VisionModel { } self.linear( &format!("{p}.mlp.linear_fc2"), - &mlp, - &delta, + mlp, + delta, n, h, intermediate, - &w, + w, )?; unsafe { check( @@ -504,28 +549,110 @@ impl VisionModel { "vision MLP residual", )?; } - self.trace(&mut callback, &p, &x, n * h)?; + self.trace(callback, &p, x, n * h)?; } - self.norm("merger.norm", &x, &norm, n)?; - self.trace(&mut callback, "merger.norm", &norm, n * h)?; + self.norm("merger.norm", x, norm, n)?; + self.trace(callback, "merger.norm", norm, n * h)?; // Consecutive groups of four patches already have the required 2x2 merge order. - self.linear("merger.linear_fc1", &norm, &mlp, n / 4, merged, merged, &w)?; - self.trace(&mut callback, "merger.linear_fc1", &mlp, n * h)?; + self.linear("merger.linear_fc1", norm, mlp, n / 4, merged, merged, w)?; + self.trace(callback, "merger.linear_fc1", mlp, n * h)?; unsafe { check( (api().cs1_vision_gelu)(mlp.at(0), n * h, 1, self.stream.0), "vision exact GELU", )?; } - let out = DeviceBuffer::new(n / 4 * output * 2)?; - self.linear("merger.linear_fc2", &mlp, &out, n / 4, output, merged, &w)?; - let result = self.read(&out, n / 4 * output)?; - if let Some(callback) = callback.as_mut() { - callback("merger.output", &result)?; - } - Ok(result) + self.linear("merger.linear_fc2", mlp, out, n / 4, output, merged, w)?; + Ok(()) + } +} +impl Drop for VisionModel { + fn drop(&mut self) { + // Even failure paths retain every referenced address through synchronization. + let _ = self.synchronize(); + } +} +fn graph_trace(action: &str, grid: [usize; 3]) { + if std::env::var("CUA_S1_GRAPH_TRACE").as_deref() == Ok("1") { + eprintln!("Vision CUDA Graph {action} grid={grid:?}"); } } +struct Scratch { + graph: Option, + grid: [usize; 3], + pixels: DeviceBuffer, + indices: DeviceBuffer, + weights: DeviceBuffer, + co: DeviceBuffer, + si: DeviceBuffer, + w: Work, + attention_workspace: DeviceBuffer, + x: DeviceBuffer, + norm: DeviceBuffer, + qkv: DeviceBuffer, + q: DeviceBuffer, + k: DeviceBuffer, + attn: DeviceBuffer, + delta: DeviceBuffer, + mlp: DeviceBuffer, + out: DeviceBuffer, +} +impl Scratch { + fn new(model: &VisionModel, grid: [usize; 3], geo: VisionGeometry) -> Result { + let n = grid[1] * grid[2]; + let h = model.config.hidden_size; + let width = model + .config + .intermediate_size + .max(h * 4) + .max(model.config.out_hidden_size); + Ok(Self { + graph: None, + grid, + pixels: DeviceBuffer::new(n * 1536 * 2)?, + indices: model.upload( + &geo.indices + .iter() + .flat_map(|v| v.to_le_bytes()) + .collect::>(), + )?, + weights: model.upload( + &geo.weights + .iter() + .flat_map(|v| v.to_le_bytes()) + .collect::>(), + )?, + co: model.upload( + &geo.cos + .iter() + .flat_map(|v| v.to_le_bytes()) + .collect::>(), + )?, + si: model.upload( + &geo.sin + .iter() + .flat_map(|v| v.to_le_bytes()) + .collect::>(), + )?, + w: Work::new(if model.lora.is_empty() { 0 } else { n }, width)?, + attention_workspace: DeviceBuffer::new(if h == 1024 { + 0 + } else { + n * model.config.num_heads * 80 * 4 * 2 + })?, + x: DeviceBuffer::new(n * h * 2)?, + norm: DeviceBuffer::new(n * h * 2)?, + qkv: DeviceBuffer::new(n * h * 3 * 2)?, + q: DeviceBuffer::new(n * h * 2)?, + k: DeviceBuffer::new(n * h * 2)?, + attn: DeviceBuffer::new(n * h * 2)?, + delta: DeviceBuffer::new(n * h * 2)?, + mlp: DeviceBuffer::new(n * width * 2)?, + out: DeviceBuffer::new(n / 4 * model.config.out_hidden_size * 2)?, + }) + } +} + struct Work { float_input: DeviceBuffer, rank: DeviceBuffer, From a3a861cd5423ccf3e0281af3b30105a8133779ae Mon Sep 17 00:00:00 2001 From: levius <2114377220@qq.com> Date: Fri, 9 Oct 2026 08:43:16 +0800 Subject: [PATCH 04/21] fix(jemm): warm vision before readiness and document parity limits --- README.md | 1 + docs/supported-models.md | 1 + mkdocs.yml | 1 + recipe/README.md | 2 + recipe/jemm/README.md | 97 ++++++++++++++ src/models/jemm/README.md | 105 +++++++++++++++ src/models/jemm/native/Cargo.toml | 20 +++ src/models/jemm/native/src/executor.rs | 8 ++ src/models/jemm/native/src/server.rs | 49 ++++--- tests/jemm/artifacts.rs | 65 +++++++++ tests/jemm/cleanup.rs | 83 ++++++++++++ tests/jemm/contract.rs | 174 +++++++++++++++++++++++++ tests/jemm/cuda.rs | 28 ++++ tests/jemm/head.rs | 19 +++ tests/jemm/processing.rs | 26 ++++ tests/jemm/server.rs | 40 ++++++ tests/jemm/tokenization.rs | 99 ++++++++++++++ 17 files changed, 793 insertions(+), 25 deletions(-) create mode 100644 recipe/jemm/README.md create mode 100644 src/models/jemm/README.md create mode 100644 tests/jemm/artifacts.rs create mode 100644 tests/jemm/cleanup.rs create mode 100644 tests/jemm/contract.rs create mode 100644 tests/jemm/cuda.rs create mode 100644 tests/jemm/head.rs create mode 100644 tests/jemm/processing.rs create mode 100644 tests/jemm/server.rs create mode 100644 tests/jemm/tokenization.rs diff --git a/README.md b/README.md index 62c30ed0..61bb4414 100644 --- a/README.md +++ b/README.md @@ -166,6 +166,7 @@ Python screenshot worker, and CLM has a stub-encoder contract recipe: | LAYA | [External worker](recipe/laya/README.md); [Python worker on Apple Silicon (MPS) and CPU](recipe/laya/apple-silicon.md); [CPU checkpoint reader](src/models/laya/README.md); [native Rust/CUDA worker on Hopper](recipe/laya/native/README.md) | | Cua-S1 4B 0.2 (`text` adapter) | [Python worker](recipe/cua_s1/text.md); [native worker](recipe/cua_s1/native.md), CUDA, run on sm_89 | | Cua-S1 4B 0.2 (`multimodal` adapter) | [Python CUDA worker](src/frontend/cua_s1.py); one PNG/JPEG screenshot, `choice`; native screenshot execution remains in progress | +| JEMM (Qwen3.8-27B) | [Native Rust/CUDA worker](recipe/jemm/README.md); text/single-image parity on A800; [known multi-image limitation](src/models/jemm/README.md#validation-and-known-multi-image-limitation) | | Open-Jev-27B-v1.1 | [Native Rust/CUDA worker](recipe/open_jev/native.md); eager independent text candidates; [H200 validation](recipe/open_jev/validation.md) | | Open-Jev-9B | The same [native Rust/CUDA worker](recipe/open_jev/native.md); eager independent text candidates; [reference comparison on sm_89](recipe/open_jev/validation-9b.md) | | CLM-v0.1-8B | [External worker with a CPU stub encoder](recipe/clm/README.md); contract checks only, real Qwen3-8B decisions unverified by this recipe | diff --git a/docs/supported-models.md b/docs/supported-models.md index 9aba8d39..d17ea7bb 100644 --- a/docs/supported-models.md +++ b/docs/supported-models.md @@ -16,6 +16,7 @@ Models that are being added are also tracked in issues labeled [new model](https | Cua-S1 4B 0.2, `multimodal` adapter | Reference worker on Transformers and PEFT, [`src/frontend/cua_s1.py`](../src/frontend/cua_s1.py); no recipe yet | Not supported | Validated ([#17](https://github.com/ThinkFlowLab/system1-omni/pull/17), [#18](https://github.com/ThinkFlowLab/system1-omni/pull/18)) | Not supported | The state is one PNG or JPEG image; upstream's `weights.lock.json` next to the base weights | | Open-Jev-27B-v1.1 | [Native Rust/CUDA worker](../recipe/open_jev/native.md) on the shared Qwen3.5/3.8 executor | Not supported | Validated on H200 (sm_90) for the [74 single-candidate workload](../recipe/open_jev/validation.md) | Not supported | Compute capability 8.0 or newer, CUDA toolkit to build, exported merged weights and trained head | | Open-Jev-9B | The same [native Rust/CUDA worker](../recipe/open_jev/native.md) | Not supported | Validated on compute capability 8.9 against the reference for [253 requests](../recipe/open_jev/validation-9b.md) | Not supported | Compute capability 8.0 or newer, CUDA toolkit to build, exported merged weights and trained head | +| JEMM / Qwen3.8-27B | [Native Rust/CUDA worker](../recipe/jemm/README.md) | Host checks only; no CPU inference | A800 (`sm_80`) text/single-image corpus; [two-/four-image parity limitation](../src/models/jemm/README.md#validation-and-known-multi-image-limitation) | Not supported | Pinned merged export, selected untied LM head, additive v2 vision symbols | | CLM-v0.1-8B | [External `clm-serve` recipe](../recipe/clm/README.md) with a CPU stub embeddings server | **Stub-encoder contract checks only** ([#23](https://github.com/ThinkFlowLab/system1-omni/pull/23)); not real Qwen3-8B decisions | Real encoder unverified by the merged recipe | Unverified | Python, upstream CLM and head checkpoint; a real encoder requires a separate embeddings server | - **Validated:** covered by the recipe on `main` or by the checks in the linked merged pull request. diff --git a/mkdocs.yml b/mkdocs.yml index e276a583..94ec7e11 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -68,5 +68,6 @@ nav: - Open-Jev native text worker: recipe/open_jev/native.md - Open-Jev-27B H200 validation: recipe/open_jev/validation.md - Open-Jev-9B validation: recipe/open_jev/validation-9b.md + - JEMM native worker: recipe/jemm/README.md - CLM stub-encoder contract: recipe/clm/README.md - Contributing: CONTRIBUTING.md diff --git a/recipe/README.md b/recipe/README.md index 8909eb94..43816a72 100644 --- a/recipe/README.md +++ b/recipe/README.md @@ -17,6 +17,8 @@ For a first real decision, follow the [complete CPU walkthrough](../docs/getting backbone and trained decision head of Open-Jev-27B-v1.1 or Open-Jev-9B, then serve with Rust and CUDA. [Open-Jev-9B validation](open_jev/validation-9b.md) compares the 9B worker with the reference. +- [JEMM native worker](jemm/README.md): pinned text/image decisions and export; + known multi-image numerical limitation is documented in the model contract. - [CLM behind the frontend](clm/README.md): run CLM's own server behind the frontend on CPU with a stub encoder, and what the response comparison has to allow for. diff --git a/recipe/jemm/README.md b/recipe/jemm/README.md new file mode 100644 index 00000000..186fb029 --- /dev/null +++ b/recipe/jemm/README.md @@ -0,0 +1,97 @@ +# JEMM setup and validation + +Run from the repository root on Linux with a CUDA GPU (compute capability 8.0+), +nvcc/cuBLASLt and Rust. Historical validation used one A800 80 GB. The raw base +and adapter need about 55.6 GB disk and the export about 52 GB more. Export is an +offline CPU operation using torch, safetensors and Transformers; the running +Rust worker needs no Python serving process. Keep inputs/exports immutable. + +## Download and export + +In a Python environment with `huggingface-hub`, torch, safetensors and Transformers: + +```sh +export JEMM_DATA=/data/jemm +hf download Qwen/Qwen3.8-27B \ + --revision 1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0 \ + --include '*.safetensors' '*.json' '*.jinja' '*.txt' 'README.md' 'LICENSE' \ + --local-dir "$JEMM_DATA/base" +hf download MaestroYan/JEMM \ + --revision 76e3c209e8441fa658221c7ba2725bad2f811176 \ + --include 'adapter_config.json' 'adapter_model.safetensors' 'decision_config.json' 'README.md' 'LICENSE' \ + --local-dir "$JEMM_DATA/adapter" +cat > "$JEMM_DATA/provenance.json" <<'JSON' +{ + "base_model_id": "Qwen/Qwen3.8-27B", + "base_revision": "1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0", + "checkpoint_revision": "76e3c209e8441fa658221c7ba2725bad2f811176", + "source_revision": "6822fe0fd53c5e6670af6ba99fb2c857a661e532" +} +JSON +python recipe/jemm/export.py --base "$JEMM_DATA/base" \ + --adapter "$JEMM_DATA/adapter" --out "$JEMM_DATA/native" \ + --provenance "$JEMM_DATA/provenance.json" --threads 4 +``` + +The checked-in pinned inventory verifies every required input file and tensor. +Interrupted exports may resume only with matching inputs, producer and recorded +runtime. Do not substitute similarly named checkpoints or newer revisions. + +## Build, launch and request + +```sh +src/backends/cuda/qwen3_5/build.sh target/release 80 +cargo build --release --locked -p omni-jemm-native -p omni-jev +JEMM_MODEL="$JEMM_DATA/native" \ +JEMM_CUDA_LIB="$PWD/target/release/libqwen3_5_cuda.so" \ + target/release/omni-jemm-native +``` + +Replace `80` with the GPU's compute capability. Rebuild the CUDA library and +consumers together; JEMM's 27B vision path requires the additive v2 symbols. +The default listener is `127.0.0.1:8000`, set by `JEMM_HOST`/`JEMM_PORT`. +It binds after text and image warmup complete. In another terminal: + +```sh +curl --fail http://127.0.0.1:8000/health +OMNI_JEV_BIND=127.0.0.1:8080 OMNI_JEV_BACKEND_URL=http://127.0.0.1:8000 \ + target/release/omni-jev +curl --fail http://127.0.0.1:8080/v1/systemone \ + -H 'Content-Type: application/json' \ + --data-binary '{"model":"JEMM","state":"Update installed.","questions":{"action":{"instructions":"Choose the next action.","criteria":{"close":"Close dialog","wait":"Wait"}}}}' +``` + +Optional `images` contains up to four base64 PNG/JPEG/WebP strings; images precede +text and are shared across questions. **Two-/four-image reference parity exceeds +the predeclared probability gate in the supplemental corpus.** See the +[model contract and numerical limitation](../../src/models/jemm/README.md#validation-and-known-multi-image-limitation) +before relying on multi-image outputs. This is a documented limitation, not a +claim that those inputs now pass. Native CPU/Metal execution is unsupported. + +## Checks and reference + +```sh +cargo fmt --all --check +cargo clippy --workspace --locked --all-targets -- -D warnings +cargo test --workspace --locked +cargo build --workspace --release --locked +# Requires an available CUDA device and real library: +JEMM_CUDA_LIB="$PWD/target/release/libqwen3_5_cuda.so" \ + cargo test --release --locked -p omni-jemm-native bf16_label_head -- \ + --ignored --test-threads=1 +``` + +CPU checks compile the worker and exercise host contracts. They cannot prove the +real image warmup, multi-image parity or device failure recovery. Hardware tests +require a separately reserved device; normal CI compiles and ignores the CUDA case. + +For the official reference, clone ypcypc/JEMM and detach at +`6822fe0fd53c5e6670af6ba99fb2c857a661e532`, then use `python -m jemm.serve +--adapter "$JEMM_DATA/adapter" --base "$JEMM_DATA/base" --device cuda:0` +with the source on PYTHONPATH and its dependencies installed. Its default port +is 8790. Freeze requests before comparing native/reference outputs and retain +tokens, grids, pixels, raw responses and source/library/model hashes. Predeclare +probability 0.02, Score 0.1 and winner margin 0.05 gates; stop at a failed gate. +The [historical evidence release](https://github.com/Levius-Fubuki/system1-omni/releases/tag/jemm-a800-20261008) +retains runners and raw results at their original source revisions. It does not +validate new code or general model accuracy. diff --git a/src/models/jemm/README.md b/src/models/jemm/README.md new file mode 100644 index 00000000..ee18a612 --- /dev/null +++ b/src/models/jemm/README.md @@ -0,0 +1,105 @@ +# JEMM native worker + +The Rust/CUDA worker serves ordered `choice`, `score` and `noul` questions over +text and up to four PNG/JPEG/WebP images. See the [recipe](../../../recipe/jemm/README.md) +for download, export, launch and validation. CPU inference and native Metal are +unsupported. Multi-image reference parity has a known limitation below. + +## Pinned artifacts + +| Artifact | Revision | +| --- | --- | +| ypcypc/JEMM reference | `6822fe0fd53c5e6670af6ba99fb2c857a661e532` | +| MaestroYan/JEMM adapter | `76e3c209e8441fa658221c7ba2725bad2f811176` | +| Qwen/Qwen3.8-27B base/tokenizer/processor | `1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0` | + +The exporter verifies pinned file SHA-256s and tensor inventory, merges language +LoRA in FP32 into cloned BF16 weights and extracts 32 rows from the untied LM +head. `jemm_export.json` records every exported file's SHA-256. Startup verifies +complete file/shard coverage before CUDA initialization; keep files immutable +while serving. This head requires the pinned temperatures 1.3480874159655591 +(text), 1.3954832341582943 (images) and threshold 0.9872681877423998. Follow-up +#122 loads validated calibration from the checksum-covered decision config; +its calibration policy is a separate change. The threshold is provenance and +does not gate SystemOne responses. + +## Request and response contract + +`POST /v1/systemone` takes `state`, a nonempty ordered `questions` object and +optional base64 `images` (bare strings or data URLs). Missing state is empty; +structured state uses compact insertion-ordered JSON. Instructions/descriptions +use Python-style coercion and whitespace flattening. Candidate order selects +labels `A`–`Z`, then `0`–`5`; Choice and Score accept 2–32 candidates. + +- Choice returns the first argmax label, probabilities and max-probability confidence. +- Noul orders yes/no and returns `noul = p(yes)`, probabilities and confidence. +- Score returns zero-based level probabilities, confidence and `expected_value`. + +Every question gets one complete prompt; vision features are shared only within +that request. Usage sums complete prompt lengths, including image tokens, for +every question. `output_tokens` is zero; `latency_ms` is included in usage. +Text/image prompt limits are 8192/3072 tokens with no truncation. Each source +image is limited to 3,145,728 pixels. Native preparation additionally bounds +compiled prompt text to 16 MiB and question count to 64, before device work. + +The **64-question limit is a native resource bound**, not an expansion of the +official HTTP bundle limit. At the pinned revision, `DecisionModel.respond` +calls `_decide` and `systemone.single_requests`, which iterate independent +questions without the eight-question cap. `MAX_BUNDLE_QUESTIONS = 8` applies +only to `contract.bundle_text`/`encode_bundle`, which this endpoint does not use. +The native bound deliberately rejects larger independent HTTP requests. + +JSON duplicate keys, nonfinite values, integers outside i64/u64, mismatched +model names and excessive nesting are rejected. Integer `-0` normalizes to 0; +floating `-0.0` remains a float. Native Python repr has a known short-escape gap +for control characters inside container-shaped instruction text. + +The HTTP body limit is 16,000,000 bytes (413 above it); invalid preparation +returns 422. JSON bodies are parsed regardless of Content-Type, like the pinned +reference. Errors have `error` (ValueError/TypeError/RuntimeError) and `detail` +bounded to 200 Unicode characters. Native retirement returns 503 rather than +the reference's 500, and some validation exception classifications/messages +differ. `/health` returns `{"status":"READY","model":"JEMM"}` only after +both a text and a synthetic-image warmup have completed. Startup errors prevent +listener binding; inference errors/panics retire the model and make health 503. + +## Ownership and lifetimes + +`contract.rs` renders prompts and reconstructs calibrated answers; +`processing.rs` validates the entire request, decodes/transforms images, tokenizes +unpadded rows and constructs three-axis positions. Prepared inputs own token +IDs, candidate counts, image positions/indices and shared image patches. +Response context owns question identities, input usage, calibration and timing. + +`executor.rs` owns shared Qwen language/vision models, selected BF16 head weights +and CUDA state. Vision runs once per request, language once per question, and +head GEMM rounds to BF16 before FP32 host softmax. One `SerialScheduler` admission +covers the entire request. Both streams synchronize before resource release; +caught execution errors/panics clear atomic readiness and retire loaded state. +There is no cross-request batching or prefix cache. + +## Validation and known multi-image limitation + +The historical A800 BF16 corpus (13 requests / 16 questions, text and single-image +inputs) passed exact tokens/positions/grids/pixels and predeclared probability +0.02 / Score 0.1 / winner-margin 0.05 gates. Max probability drift was 0.0015338802. +These results are scoped to the source revisions and corpus in +[the evidence release](https://github.com/Levius-Fubuki/system1-omni/releases/tag/jemm-a800-20261008). + +**Two- and four-image reference parity is not established.** The supplemental +2/3/4-PNG-image feasibility corpus exceeded the unchanged 0.02 probability gate +for two and four images, with max drift **0.042138323189940485**; three images +passed. All nine question records had exact official tokens, positions, grids, +candidate counts and FP32 pixels. #118 and #122 produced identical answers and +usage for these requests. This narrows investigation to execution/merge numerics, +but does not identify the cause. Candidate repeated rounds stopped after the +failed feasibility gate; failures were retained. The worker accepts these input +classes, but applications requiring verified multi-image fidelity must validate +their own workloads or use the pinned reference. No tolerance was relaxed. + +Root `tests/jemm/` covers prompt order/coercion, candidate limits, the 64-question +resource boundary, response math, image decoding/positions, tokenizer and token +budgets, export integrity, BF16 head padding, error-envelope bounds, warmup image +decoding and synchronized retirement. The real CUDA head test is ignored in CPU +CI. CPU tests do not prove full-model parity or execute the startup image forward. +No fresh GPU campaign or performance claim accompanies the review fixes. diff --git a/src/models/jemm/native/Cargo.toml b/src/models/jemm/native/Cargo.toml index 11d9c841..8735128e 100644 --- a/src/models/jemm/native/Cargo.toml +++ b/src/models/jemm/native/Cargo.toml @@ -16,3 +16,23 @@ omni-runtime = { path = "../../../runtime" } serde_json = { version = "1.0.149", features = ["float_roundtrip", "preserve_order"] } tokenizers = { version = "=0.22.2", default-features = false, features = ["onig"] } tokio = { version = "1.49.0", features = ["macros", "net", "rt-multi-thread", "sync"] } + +[[test]] +name = "contract" +path = "../../../../tests/jemm/contract.rs" + +[[test]] +name = "processing" +path = "../../../../tests/jemm/processing.rs" + +[[test]] +name = "head" +path = "../../../../tests/jemm/head.rs" + +[[test]] +name = "artifacts" +path = "../../../../tests/jemm/artifacts.rs" + +[[test]] +name = "tokenization" +path = "../../../../tests/jemm/tokenization.rs" diff --git a/src/models/jemm/native/src/executor.rs b/src/models/jemm/native/src/executor.rs index 5534f33c..e90c6c30 100644 --- a/src/models/jemm/native/src/executor.rs +++ b/src/models/jemm/native/src/executor.rs @@ -285,3 +285,11 @@ fn guarded( } result } + +#[cfg(test)] +#[path = "../../../../../tests/jemm/cleanup.rs"] +mod cleanup_tests; + +#[cfg(test)] +#[path = "../../../../../tests/jemm/cuda.rs"] +mod cuda_tests; diff --git a/src/models/jemm/native/src/server.rs b/src/models/jemm/native/src/server.rs index 4c33e19b..db9c8998 100644 --- a/src/models/jemm/native/src/server.rs +++ b/src/models/jemm/native/src/server.rs @@ -5,7 +5,7 @@ use axum::{ Json, Router, body::Bytes, extract::{DefaultBodyLimit, State, rejection::BytesRejection}, - http::{HeaderMap, StatusCode}, + http::StatusCode, response::{IntoResponse, Response}, routing::{get, post}, }; @@ -13,7 +13,8 @@ use omni_qwen3_5_native::cuda; use omni_runtime::SerialScheduler; use serde_json::{Value, json}; use std::{path::Path, sync::Arc}; -const WARMUP:&[u8]=br#"{"state":"Update installed.","questions":{"q":{"instructions":"Choose the action.","criteria":{"close":"Close dialog","wait":"Wait"}}}}"#; +const TEXT_WARMUP:&[u8]=br#"{"state":"Update installed.","questions":{"q":{"instructions":"Choose the action.","criteria":{"close":"Close dialog","wait":"Wait"}}}}"#; +const WARMUP: &[u8] = br#"{"state":"Update installed.","questions":{"q":{"instructions":"Choose the action.","criteria":{"close":"Close dialog","wait":"Wait"}}},"images":["data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAACAAAAAgCAIAAAD8GO2jAAAAKklEQVR4nGPgUbKgKWIYtWDUglELRi0YtWDUglELRi0YtWDUglELhooFAG0HmBCThFO/AAAAAElFTkSuQmCC"]}"#; pub struct Engine { pub processor: Processor, pub executor: Executor, @@ -35,18 +36,27 @@ impl Engine { executor, scheduler: SerialScheduler::default(), }; - let prepared = engine.processor.prepare(WARMUP)?; - let rows = engine - .executor - .execute(&engine.scheduler, prepared.inputs) - .await?; - prepared.context.finish(rows)?; + // Validate both device paths before binding the listener or reporting ready. + for raw in [TEXT_WARMUP, WARMUP] { + let prepared = engine.processor.prepare(raw)?; + let rows = engine + .executor + .execute(&engine.scheduler, prepared.inputs) + .await?; + prepared.context.finish(rows)?; + } ensure!(engine.executor.available(), "warmup retired the model"); Ok(engine) } } fn error(status: StatusCode, message: impl ToString) -> Response { - (status, Json(json!({"error":message.to_string()}))).into_response() + let kind = match status { + StatusCode::UNPROCESSABLE_ENTITY | StatusCode::PAYLOAD_TOO_LARGE => "ValueError", + StatusCode::UNSUPPORTED_MEDIA_TYPE => "TypeError", + _ => "RuntimeError", + }; + let detail: String = message.to_string().chars().take(200).collect(); + (status, Json(json!({"error":kind,"detail":detail}))).into_response() } pub async fn decide(engine: &Engine, raw: &[u8]) -> Response { if !engine.executor.available() { @@ -71,23 +81,8 @@ pub async fn decide(engine: &Engine, raw: &[u8]) -> Response { } async fn systemone( State(engine): State>, - headers: HeaderMap, body: Result, ) -> Response { - let content_type = headers - .get("content-type") - .and_then(|v| v.to_str().ok()) - .unwrap_or("") - .split(';') - .next() - .unwrap_or("") - .trim(); - if !content_type.eq_ignore_ascii_case("application/json") { - return error( - StatusCode::UNSUPPORTED_MEDIA_TYPE, - "Content-Type must be application/json", - ); - } match body { Ok(raw) => decide(&engine, &raw).await, Err(e) => error(e.status(), e.body_text()), @@ -99,7 +94,7 @@ pub fn router(engine: Arc) -> Router { "/health", get(|State(engine): State>| async move { if engine.executor.available() { - Json(json!({"status":"ready","model":"JEMM"})).into_response() + Json(json!({"status":"READY","model":"JEMM"})).into_response() } else { error(StatusCode::SERVICE_UNAVAILABLE, "model is unavailable") } @@ -129,3 +124,7 @@ pub async fn run() -> Result<()> { axum::serve(listener, router(engine)).await?; Ok(()) } + +#[cfg(test)] +#[path = "../../../../../tests/jemm/server.rs"] +mod tests; diff --git a/tests/jemm/artifacts.rs b/tests/jemm/artifacts.rs new file mode 100644 index 00000000..58fac3a6 --- /dev/null +++ b/tests/jemm/artifacts.rs @@ -0,0 +1,65 @@ +use omni_jemm_native::artifacts::verify_export; +use serde_json::json; +#[test] +fn export_hash_map_is_mandatory_and_rejects_traversal() { + let dir = std::env::temp_dir(); + assert!( + verify_export(&dir, &json!({})) + .unwrap_err() + .to_string() + .contains("export_sha256") + ); + assert!( + verify_export(&dir, &json!({"export_sha256":{"../escape":"0".repeat(64)}})) + .unwrap_err() + .to_string() + .contains("relative") + ); +} +#[test] +fn hashes_every_exported_file_and_requires_all_language_shards() { + use sha2::{Digest, Sha256}; + let dir = std::env::temp_dir().join(format!("jemm-artifacts-{}", std::process::id())); + std::fs::create_dir_all(&dir).unwrap(); + let index = br#"{"weight_map":{"embed_tokens.weight":"model-1.safetensors"}}"#; + let mut hashes = serde_json::Map::new(); + for name in [ + "config.json", + "tokenizer.json", + "preprocessor_config.json", + "model.safetensors.index.json", + "vision.safetensors", + "jemm_lm_head.safetensors", + "model-1.safetensors", + "decision_config.json", + ] { + let bytes = if name == "model.safetensors.index.json" { + index.as_slice() + } else { + b"original" + }; + std::fs::write(dir.join(name), bytes).unwrap(); + hashes.insert(name.into(), json!(format!("{:x}", Sha256::digest(bytes)))); + } + let manifest = json!({"export_sha256":hashes}); + verify_export(&dir, &manifest).unwrap(); + std::fs::write(dir.join("decision_config.json"), b"tampered").unwrap(); + assert!( + verify_export(&dir, &manifest) + .unwrap_err() + .to_string() + .contains("mismatch: decision_config.json") + ); + let mut incomplete = manifest; + incomplete["export_sha256"] + .as_object_mut() + .unwrap() + .remove("model-1.safetensors"); + assert!( + verify_export(&dir, &incomplete) + .unwrap_err() + .to_string() + .contains("missing language shard") + ); + std::fs::remove_dir_all(dir).unwrap(); +} diff --git a/tests/jemm/cleanup.rs b/tests/jemm/cleanup.rs new file mode 100644 index 00000000..b0b34f25 --- /dev/null +++ b/tests/jemm/cleanup.rs @@ -0,0 +1,83 @@ +use super::{AtomicBool, guarded}; +use std::sync::{ + Arc, + atomic::{AtomicUsize, Ordering}, +}; +struct Sentinel(Arc); +impl Drop for Sentinel { + fn drop(&mut self) { + self.0.fetch_add( + if self.0.load(Ordering::SeqCst) == 1 { + 1 + } else { + 100 + }, + Ordering::SeqCst, + ); + } +} +#[test] +fn execution_error_synchronizes_and_retires_loaded_state() { + let event = Arc::new(AtomicUsize::new(0)); + let mut state = Some(Sentinel(event.clone())); + let ready = AtomicBool::new(true); + let result = guarded( + &mut state, + |_| Err::<(), _>(anyhow::anyhow!("forward failed")), + |s| { + assert!( + !ready.load(Ordering::SeqCst), + "unavailable before synchronization" + ); + s.0.store(1, Ordering::SeqCst); + Ok(()) + }, + || { + ready.store(false, Ordering::SeqCst); + }, + ); + assert!(result.is_err()); + assert!(state.is_none()); + assert_eq!(event.load(Ordering::SeqCst), 2); +} +#[test] +fn execution_panic_synchronizes_and_retires_loaded_state() { + let event = Arc::new(AtomicUsize::new(0)); + let mut state = Some(Sentinel(event.clone())); + let ready = AtomicBool::new(true); + let result = guarded( + &mut state, + |_| -> anyhow::Result<()> { panic!("forward panic") }, + |s| { + assert!( + !ready.load(Ordering::SeqCst), + "unavailable before synchronization" + ); + s.0.store(1, Ordering::SeqCst); + Ok(()) + }, + || { + ready.store(false, Ordering::SeqCst); + }, + ); + assert!(result.is_err()); + assert!(state.is_none()); + assert_eq!(event.load(Ordering::SeqCst), 2); +} +#[test] +fn health_availability_does_not_wait_for_execution_mutex() { + use super::{AtomicBool, Executor, Mutex}; + use std::sync::mpsc; + let executor = Executor { + state: Arc::new(Mutex::new(None)), + ready: Arc::new(AtomicBool::new(true)), + }; + let state = executor.state.clone(); + let held = state.lock().unwrap(); + let (tx, rx) = mpsc::channel(); + let thread = std::thread::spawn(move || tx.send(executor.available()).unwrap()); + let response = rx.recv_timeout(std::time::Duration::from_secs(1)); + drop(held); + thread.join().unwrap(); + assert!(response.unwrap()); +} diff --git a/tests/jemm/contract.rs b/tests/jemm/contract.rs new file mode 100644 index 00000000..2286ccf0 --- /dev/null +++ b/tests/jemm/contract.rs @@ -0,0 +1,174 @@ +use omni_jemm_native::contract::{answer, compile}; +#[test] +fn prompt_preserves_order_and_compact_state() { + let qs=compile(br#"{"state":{"z":1,"a":true},"questions":{"q":{"instructions":" choose\n one ","criteria":{"last":"last option","first":{"description":"first"}}}}}"#).unwrap(); + assert_eq!(qs[0].keys, ["last", "first"]); + assert_eq!( + qs[0].prompt, + "State:\n{\"z\":1,\"a\":true}\n\nQuestion: choose one\n\nCandidates:\nA) last option\nB) first\n\nAnswer with exactly one candidate label." + ); +} +#[test] +fn noul_uses_yes_first_and_score_expected_value() { + let qs = compile( + br#"{"questions":{"n":{"type":"noul"},"s":{"type":"score","criteria":[null,"mid","hi"]}}}"#, + ) + .unwrap(); + let n = answer(&qs[0], &[0., 0.], 1.3480874159655591).unwrap(); + assert_eq!(n["noul"], 0.5); + assert_eq!(n["confidence"], 0.5); + let s = answer(&qs[1], &[0., 0., 0.], 1.).unwrap(); + assert!((s["expected_value"].as_f64().unwrap() - 1.).abs() < 1e-12); + assert!(s.get("score").is_none()); +} +#[test] +fn renders_tool_parameters_and_python_coercion() { + let qs=compile(br#"{"questions":{"q":{"instructions":null,"criteria":{"t":{"description":"{\"name\":\"run\",\"description\":\" Do it \",\"parameters\":{\"properties\":{\"x\":{\"default\":true},\"y\":{}},\"required\":[\"y\"]}}","action":{"tool_name":"run"}},"n":false}}}}"#).unwrap(); + assert!(qs[0].prompt.contains("Question: None")); + assert!( + qs[0] + .prompt + .contains("A) run — Do it | params: x=True, y*\nB) False") + ); +} +#[test] +fn rejects_candidate_limits_bad_kinds_and_nonfinite_scores() { + for body in [ + br#"{"questions":{"q":{"criteria":{"a":"one"}}}}"#.as_slice(), + br#"{"questions":{"q":{"type":"unknown"}}}"#, + ] { + assert!(compile(body).is_err()); + } + let q = compile(br#"{"questions":{"q":{"criteria":{"a":"one","b":"two"}}}}"#).unwrap(); + assert!(answer(&q[0], &[f32::NAN, 0.], 1.).is_err()); + let a = answer(&q[0], &[1., 1.], 1.).unwrap(); + assert_eq!(a["choice"], "a"); +} +#[test] +fn rejects_non_string_type_and_supports_all_32_labels() { + assert!( + compile(br#"{"questions":{"q":{"type":null,"criteria":{"a":"one","b":"two"}}}}"#).is_err() + ); + let mut criteria = serde_json::Map::new(); + for i in 0..32 { + criteria.insert(i.to_string(), serde_json::json!(format!("candidate{i}"))); + } + let raw = serde_json::json!({"questions":{"q":{"criteria":criteria}}}); + let qs = compile(&serde_json::to_vec(&raw).unwrap()).unwrap(); + assert!( + qs[0] + .prompt + .ends_with("5) candidate31\n\nAnswer with exactly one candidate label.") + ); + let mut raw = raw; + raw["questions"]["q"]["criteria"]["extra"] = serde_json::json!("33"); + assert!(compile(&serde_json::to_vec(&raw).unwrap()).is_err()); +} +#[test] +fn python_container_repr_preserves_combining_marks_and_escapes_private_use() { + use omni_jemm_native::contract::py_str; + assert_eq!( + py_str(&serde_json::json!(["e\u{0300}", "\u{e000}", true, null])), + "['e\u{0300}', '\\ue000', True, None]" + ); +} +#[test] +fn python_unicode15_repr_handles_nko_arabic_controls_and_unassigned() { + use omni_jemm_native::contract::py_str; + assert_eq!( + py_str(&serde_json::json!([ + "\u{07fd}", + "\u{08d3}\u{08db}", + "\u{0001}", + "\u{e000}", + "\u{0378}" + ])), + "['\u{07fd}', '\u{08d3}\u{08db}', '\\x01', '\\ue000', '\\u0378']" + ); +} +#[test] +fn integer_negative_zero_matches_python_and_big_integers_are_rejected() { + let q = compile(br#"{"state":-0,"questions":{"q":{"criteria":{"a":"a","b":"b"}}}}"#).unwrap(); + assert!(q[0].prompt.starts_with("State:\n0\n")); + let error = compile( + br#"{"state":18446744073709551616,"questions":{"q":{"criteria":{"a":"a","b":"b"}}}}"#, + ) + .unwrap_err(); + assert!(error.to_string().contains("integer outside i64/u64")); + assert!( + compile( + br#"{"state":-9223372036854775809,"questions":{"q":{"criteria":{"a":"a","b":"b"}}}}"# + ) + .is_err() + ); +} +#[test] +fn aggregate_prompt_budget_reserves_before_allocation_and_caps_questions() { + use omni_jemm_native::contract::reserve_prompt_bytes; + let mut total = 0; + reserve_prompt_bytes(&mut total, 4, 8).unwrap(); + assert!(reserve_prompt_bytes(&mut total, 5, 8).is_err()); + assert_eq!(total, 4); + let questions = (0..65) + .map(|i| { + ( + i.to_string(), + serde_json::json!({"criteria":{"a":"a","b":"b"}}), + ) + }) + .collect::>(); + assert!( + compile(&serde_json::to_vec(&serde_json::json!({"questions":questions})).unwrap()).is_err() + ); +} +#[test] +fn recognized_tool_json_rejects_nonfinite_defaults_under_workspace_features() { + let body = serde_json::json!({"questions":{"q":{"criteria":{"a":{"description":"{\"name\":\"run\",\"parameters\":{\"properties\":{\"x\":{\"default\":1e400}}}}","action":{"tool_name":"run"}},"b":"fallback"}}}}); + assert!(compile(&serde_json::to_vec(&body).unwrap()).is_err()); +} +#[test] +fn tool_defaults_preserve_integer_zero_bounds_and_fallbacks() { + fn tool_body(description: &str) -> Vec { + serde_json::to_vec(&serde_json::json!({"questions":{"q":{"criteria":{"a":{"description":description,"action":{"tool_name":"run"}},"b":"fallback"}}}})).unwrap() + } + for (default, expected) in [ + ("-0", "0"), + ("18446744073709551615", "18446744073709551615"), + ] { + let text = r#"{"name":"run","parameters":{"properties":{"x":{"default":DEFAULT}}}}"# + .replace("DEFAULT", default); + let q = compile(&tool_body(&text)).unwrap(); + assert!( + q[0].prompt + .contains(&format!("A) run | params: x={expected}")) + ); + } + assert!( + compile(&tool_body( + "{\"name\":\"run\",\"parameters\":{\"x\":{\"default\":18446744073709551616}}}" + )) + .is_err() + ); + for text in ["{broken json", "{\"default\":1e400}"] { + let q = compile(&tool_body(text)).unwrap(); + assert!(q[0].prompt.contains(&format!("A) {text}"))); + } +} + +#[test] +fn systemone_accepts_nine_questions_and_enforces_native_resource_cap() { + use serde_json::{Map, json}; + for count in [1, 8, 9, 64, 65] { + let questions: Map = (0..count) + .map(|i| (format!("q{i}"), json!({"type":"noul"}))) + .collect(); + let result = compile(&serde_json::to_vec(&json!({"questions":questions})).unwrap()); + if count <= 64 { + let compiled = result.unwrap(); + assert_eq!(compiled.len(), count); + assert_eq!(compiled.last().unwrap().id, format!("q{}", count - 1)); + } else { + assert!(result.unwrap_err().to_string().contains("1 to 64")); + } + } +} diff --git a/tests/jemm/cuda.rs b/tests/jemm/cuda.rs new file mode 100644 index 00000000..0dfae4e0 --- /dev/null +++ b/tests/jemm/cuda.rs @@ -0,0 +1,28 @@ +use super::LabelHead; +use half::bf16; +use omni_qwen3_5_native::cuda; +#[test] +#[ignore = "requires JEMM_CUDA_LIB and an available CUDA device"] +fn bf16_label_head_gemm_matches_exact_constant_rows() { + let library = std::env::var_os("JEMM_CUDA_LIB").expect("set JEMM_CUDA_LIB"); + cuda::load(std::path::Path::new(&library)).unwrap(); + cuda::set_device(0).unwrap(); + let hidden = 5120; + let mut bytes = vec![0u8; 256 * hidden * 2]; + for row in 0..32 { + for col in 0..hidden { + let offset = (row * hidden + col) * 2; + bytes[offset..offset + 2] + .copy_from_slice(&bf16::from_f32(row as f32 / 32.).to_le_bytes()); + } + } + let mut head = LabelHead::load(&bytes, hidden).unwrap(); + let actual = head.score(&vec![1.; hidden], 32).unwrap(); + for (row, score) in actual.iter().enumerate() { + assert_eq!( + *score, + bf16::from_f32(row as f32 * 160.).to_f32(), + "label row {row}" + ); + } +} diff --git a/tests/jemm/head.rs b/tests/jemm/head.rs new file mode 100644 index 00000000..1ed08278 --- /dev/null +++ b/tests/jemm/head.rs @@ -0,0 +1,19 @@ +use omni_jemm_native::executor::head_bytes; +use safetensors::{ + Dtype, + tensor::{TensorView, serialize}, +}; +#[test] +fn pads_selected_bf16_lm_head_rows_to_256_without_substitution() { + let raw = (0..64) + .flat_map(|i| half::bf16::from_f32(i as f32).to_le_bytes()) + .collect::>(); + let tensor = TensorView::new(Dtype::BF16, vec![32, 2], &raw).unwrap(); + let data = serialize([("weight", tensor)], None).unwrap(); + let padded = head_bytes(&data, 2).unwrap(); + assert_eq!(padded.len(), 256 * 2 * 2); + assert_eq!(&padded[..raw.len()], raw.as_slice()); + assert!(padded[raw.len()..].iter().all(|&x| x == 0)); + let tensor = TensorView::new(Dtype::BF16, vec![64], &raw).unwrap(); + assert!(head_bytes(&serialize([("weight", tensor)], None).unwrap(), 2).is_err()); +} diff --git a/tests/jemm/processing.rs b/tests/jemm/processing.rs new file mode 100644 index 00000000..3036938f --- /dev/null +++ b/tests/jemm/processing.rs @@ -0,0 +1,26 @@ +use omni_jemm_native::processing::{decode_images, image_positions}; +use serde_json::json; +#[test] +fn image_decoder_accepts_rgb_png_data_urls_and_whitespace() { + use base64::Engine; + let mut bytes = std::io::Cursor::new(Vec::new()); + image::RgbImage::from_pixel(32, 32, image::Rgb([12, 34, 56])) + .write_to(&mut bytes, image::ImageFormat::Png) + .unwrap(); + let data = base64::engine::general_purpose::STANDARD.encode(bytes.into_inner()); + let images = decode_images(&json!([format!("data:image/png;base64,\n\u{1c}{data}")])).unwrap(); + assert_eq!(images[0].get_pixel(0, 0).0, [12, 34, 56]); + assert!(decode_images(&json!(["bad"])).is_err()); + assert!(decode_images(&json!(["a", "b", "c", "d", "e"])).is_err()); + assert!(decode_images(&json!(null)).unwrap().is_empty()); +} +#[test] +fn multimodal_positions_restart_text_after_grid_maximum() { + // 2x3 merged rows, then two text tokens, then a 1x2 image. + let ids = [1, 99, 99, 99, 99, 99, 99, 2, 3, 99, 99, 4]; + let p = image_positions(&ids, 99, &[[1, 4, 6], [1, 2, 4]]).unwrap(); + assert_eq!(p[0], [0, 1, 1, 1, 1, 1, 1, 4, 5, 6, 6, 8]); + assert_eq!(p[1], [0, 1, 1, 1, 2, 2, 2, 4, 5, 6, 6, 8]); + assert_eq!(p[2], [0, 1, 2, 3, 1, 2, 3, 4, 5, 6, 7, 8]); + assert!(image_positions(&ids, 99, &[[1, 4, 4]]).is_err()); +} diff --git a/tests/jemm/server.rs b/tests/jemm/server.rs new file mode 100644 index 00000000..d101cb84 --- /dev/null +++ b/tests/jemm/server.rs @@ -0,0 +1,40 @@ +use super::*; + +#[test] +fn readiness_warmup_includes_a_decodable_minimal_image() { + let request: Value = serde_json::from_slice(WARMUP).unwrap(); + let images = crate::processing::decode_images(&request["images"]).unwrap(); + assert_eq!( + images.len(), + 1, + "readiness must exercise vision as well as language" + ); + assert_eq!(images[0].dimensions(), (32, 32)); + assert_eq!(crate::contract::compile(WARMUP).unwrap().len(), 1); + let text: Value = serde_json::from_slice(TEXT_WARMUP).unwrap(); + assert!( + crate::processing::decode_images(&text["images"]) + .unwrap() + .is_empty() + ); + assert_eq!(crate::contract::compile(TEXT_WARMUP).unwrap().len(), 1); +} + +#[tokio::test] +async fn error_envelope_is_typed_and_bounds_unicode_detail() { + for (status, class) in [ + (StatusCode::UNPROCESSABLE_ENTITY, "ValueError"), + (StatusCode::PAYLOAD_TOO_LARGE, "ValueError"), + (StatusCode::UNSUPPORTED_MEDIA_TYPE, "TypeError"), + (StatusCode::SERVICE_UNAVAILABLE, "RuntimeError"), + ] { + let response = error(status, "界".repeat(250)); + assert_eq!(response.status(), status); + let bytes = axum::body::to_bytes(response.into_body(), 4096) + .await + .unwrap(); + let body: Value = serde_json::from_slice(&bytes).unwrap(); + assert_eq!(body["error"], class); + assert_eq!(body["detail"].as_str().unwrap().chars().count(), 200); + } +} diff --git a/tests/jemm/tokenization.rs b/tests/jemm/tokenization.rs new file mode 100644 index 00000000..736d2995 --- /dev/null +++ b/tests/jemm/tokenization.rs @@ -0,0 +1,99 @@ +use omni_jemm_native::{contract, processing::Processor}; +use serde_json::{Value, json}; +use std::sync::atomic::{AtomicUsize, Ordering}; +use tokenizers::{ + AddedToken, Tokenizer, models::wordlevel::WordLevel, + pre_tokenizers::whitespace::WhitespaceSplit, +}; +static ID: AtomicUsize = AtomicUsize::new(0); +struct Fixture(std::path::PathBuf); +impl Drop for Fixture { + fn drop(&mut self) { + let _ = std::fs::remove_dir_all(&self.0); + } +} +fn fixture() -> (Fixture, Value) { + let dir = std::env::temp_dir().join(format!( + "jemm-tokenizer-{}-{}", + std::process::id(), + ID.fetch_add(1, Ordering::Relaxed) + )); + std::fs::create_dir_all(&dir).unwrap(); + let mut vocab = contract::LABELS + .chars() + .enumerate() + .map(|(i, c)| (c.to_string(), i as u32 + 1)) + .collect::>(); + vocab.push(("[UNK]".into(), 0)); + vocab.push(("<|image_pad|>".into(), 33)); + let mut tok = Tokenizer::new( + WordLevel::builder() + .vocab(vocab.into_iter().collect()) + .unk_token("[UNK]".into()) + .build() + .unwrap(), + ); + tok.with_pre_tokenizer(Some(WhitespaceSplit)); + tok.add_special_tokens(&[AddedToken::from("<|image_pad|>", true)]); + tok.save(dir.join("tokenizer.json"), false).unwrap(); + std::fs::write(dir.join("config.json"), br#"{"image_token_id":33}"#).unwrap(); + std::fs::write( + dir.join("preprocessor_config.json"), + br#"{"size":{"shortest_edge":65536,"longest_edge":16777216}}"#, + ) + .unwrap(); + let manifest = json!({"format":"jemm-native/1","model_id":"JEMM","base_model_id":"Qwen/Qwen3.8-27B","base_revision":contract::BASE_REVISION,"checkpoint_revision":contract::CHECKPOINT_REVISION,"source_revision":contract::SOURCE_REVISION,"max_tokens":8192,"max_mm_tokens":3072,"label_token_ids":(1..=32).collect::>(),"chat_prefix":format!("<|im_start|>system\n{}<|im_end|>\n<|im_start|>user\n",contract::SYSTEM),"chat_suffix":"<|im_end|>\n<|im_start|>assistant\n\n\n\n\n","temperature":1.3480874159655591,"mm_temperature":1.3954832341582943,"threshold":0.9872681877423998}); + (Fixture(dir), manifest) +} +#[test] +fn text_special_token_ids_are_preserved_and_usage_counts_one_prompt() { + let (f, m) = fixture(); + let p = Processor::load(&f.0, &m).unwrap(); + let prepared = p + .prepare(br#"{"questions":{"q":{"criteria":{"a":"<|image_pad|>","b":"normal"}}}}"#) + .unwrap(); + assert!(prepared.inputs[0].ids.contains(&33)); + let count = prepared.inputs[0].ids.len(); + let response = prepared.context.finish(vec![vec![0., 0.]]).unwrap(); + assert_eq!(response["usage"]["input_tokens"], count); + assert_eq!(response["usage"]["output_tokens"], 0); +} +#[test] +fn rejects_mismatched_label_ids_and_oversized_prompt_before_execution() { + let (f, mut m) = fixture(); + m["label_token_ids"][0] = json!(9); + assert!(Processor::load(&f.0, &m).is_err()); + m["label_token_ids"][0] = json!(1); + let p = Processor::load(&f.0, &m).unwrap(); + let body = + json!({"questions":{"q":{"instructions":"x ".repeat(8192),"criteria":{"a":"a","b":"b"}}}}); + assert!(p.prepare(&serde_json::to_vec(&body).unwrap()).is_err()); +} +#[test] +fn exported_tokenizer_truncation_and_padding_cannot_change_request_tokens() { + let (f, m) = fixture(); + let mut t = Tokenizer::from_file(f.0.join("tokenizer.json")).unwrap(); + t.with_truncation(Some(tokenizers::TruncationParams { + max_length: 3, + ..Default::default() + })) + .unwrap(); + t.save(f.0.join("tokenizer.json"), false).unwrap(); + let processor = Processor::load(&f.0, &m).unwrap(); + let prepared = processor + .prepare(br#"{"questions":{"q":{"criteria":{"a":"first option","b":"second option"}}}}"#) + .unwrap(); + assert!(prepared.inputs[0].ids.len() > 3); +} +#[test] +fn rejects_custom_calibration_for_pinned_adapter() { + let (f, mut m) = fixture(); + m["mm_temperature"] = json!(1.); + assert!( + Processor::load(&f.0, &m) + .err() + .unwrap() + .to_string() + .contains("pinned JEMM calibration") + ); +} From ec86ced0fe922a28c1d6710a369b04ba014b3c77 Mon Sep 17 00:00:00 2001 From: levius <2114377220@qq.com> Date: Fri, 9 Oct 2026 08:46:21 +0800 Subject: [PATCH 05/21] test(jemm): retain file-driven calibration regressions --- src/models/jemm/README.md | 9 +++--- tests/jemm/tokenization.rs | 56 ++++++++++++++++++++++++++++++++++++-- 2 files changed, 59 insertions(+), 6 deletions(-) diff --git a/src/models/jemm/README.md b/src/models/jemm/README.md index ee18a612..b1083c63 100644 --- a/src/models/jemm/README.md +++ b/src/models/jemm/README.md @@ -17,10 +17,11 @@ The exporter verifies pinned file SHA-256s and tensor inventory, merges language LoRA in FP32 into cloned BF16 weights and extracts 32 rows from the untied LM head. `jemm_export.json` records every exported file's SHA-256. Startup verifies complete file/shard coverage before CUDA initialization; keep files immutable -while serving. This head requires the pinned temperatures 1.3480874159655591 -(text), 1.3954832341582943 (images) and threshold 0.9872681877423998. Follow-up -#122 loads validated calibration from the checksum-covered decision config; -its calibration policy is a separate change. The threshold is provenance and +while serving. Calibration loads from checksum-covered `decision_config.json`; temperatures +must be finite and positive, threshold finite and within [0,1], and all values +must agree with the export manifest. The pinned artifact values are +1.3480874159655591 (text), 1.3954832341582943 (images) and 0.9872681877423998 +(threshold). The threshold is provenance and does not gate SystemOne responses. ## Request and response contract diff --git a/tests/jemm/tokenization.rs b/tests/jemm/tokenization.rs index 736d2995..11042532 100644 --- a/tests/jemm/tokenization.rs +++ b/tests/jemm/tokenization.rs @@ -43,6 +43,13 @@ fn fixture() -> (Fixture, Value) { ) .unwrap(); let manifest = json!({"format":"jemm-native/1","model_id":"JEMM","base_model_id":"Qwen/Qwen3.8-27B","base_revision":contract::BASE_REVISION,"checkpoint_revision":contract::CHECKPOINT_REVISION,"source_revision":contract::SOURCE_REVISION,"max_tokens":8192,"max_mm_tokens":3072,"label_token_ids":(1..=32).collect::>(),"chat_prefix":format!("<|im_start|>system\n{}<|im_end|>\n<|im_start|>user\n",contract::SYSTEM),"chat_suffix":"<|im_end|>\n<|im_start|>assistant\n\n\n\n\n","temperature":1.3480874159655591,"mm_temperature":1.3954832341582943,"threshold":0.9872681877423998}); + std::fs::write( + dir.join("decision_config.json"), + serde_json::to_vec(&json!({"temperature":manifest["temperature"], + "mm_temperature":manifest["mm_temperature"], "threshold":manifest["threshold"]})) + .unwrap(), + ) + .unwrap(); (Fixture(dir), manifest) } #[test] @@ -86,7 +93,7 @@ fn exported_tokenizer_truncation_and_padding_cannot_change_request_tokens() { assert!(prepared.inputs[0].ids.len() > 3); } #[test] -fn rejects_custom_calibration_for_pinned_adapter() { +fn rejects_manifest_calibration_disagreement() { let (f, mut m) = fixture(); m["mm_temperature"] = json!(1.); assert!( @@ -94,6 +101,51 @@ fn rejects_custom_calibration_for_pinned_adapter() { .err() .unwrap() .to_string() - .contains("pinned JEMM calibration") + .contains("manifest disagrees with decision_config.json") + ); +} + +#[test] +fn file_driven_calibration_must_be_finite_in_range_and_agree_with_manifest() { + for (field, invalid) in [ + ("temperature", json!(0.0)), + ("mm_temperature", json!(-1.0)), + ("threshold", json!(1.1)), + ] { + let (f, mut m) = fixture(); + m[field] = invalid; + std::fs::write( + f.0.join("decision_config.json"), + serde_json::to_vec(&json!({"temperature":m["temperature"], + "mm_temperature":m["mm_temperature"], "threshold":m["threshold"]})) + .unwrap(), + ) + .unwrap(); + assert!( + Processor::load(&f.0, &m) + .err() + .unwrap() + .to_string() + .contains("invalid calibration") + ); + } + let (f, mut m) = fixture(); + m["temperature"] = json!(2.0); + m["mm_temperature"] = json!(3.0); + m["threshold"] = json!(0.5); + std::fs::write( + f.0.join("decision_config.json"), + serde_json::to_vec(&json!({"temperature":2.0,"mm_temperature":3.0,"threshold":0.5})) + .unwrap(), + ) + .unwrap(); + let p = Processor::load(&f.0, &m).unwrap(); + let prepared = p + .prepare(br#"{"questions":{"q":{"type":"noul"}}}"#) + .unwrap(); + let response = prepared.context.finish(vec![vec![2.0, 0.0]]).unwrap(); + assert!( + (response["answers"]["q"]["noul"].as_f64().unwrap() - 1.0 / (1.0 + (-1.0f64).exp())).abs() + < 1e-12 ); } From 89746b8fc891c4aac219a7421f48ac33e110f9e0 Mon Sep 17 00:00:00 2001 From: levius <2114377220@qq.com> Date: Fri, 9 Oct 2026 08:52:44 +0800 Subject: [PATCH 06/21] docs(jemm): retain graph replay switches and cache limits --- src/models/jemm/README.md | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/src/models/jemm/README.md b/src/models/jemm/README.md index b1083c63..7979f292 100644 --- a/src/models/jemm/README.md +++ b/src/models/jemm/README.md @@ -79,6 +79,22 @@ covers the entire request. Both streams synchronize before resource release; caught execution errors/panics clear atomic readiness and retire loaded state. There is no cross-request batching or prefix cache. +## Optional CUDA Graph replay + +`CUA_S1_GRAPH=1` enables language replay, with separate 64-entry FIFO caches +for text ordered lengths and multimodal `[tokens]`. `CUA_S1_VISION_GRAPH=1` +independently enables vision replay; its four-slot FIFO uses exact `[T,H,W]` +grids. Both switches default off. Current IDs/features/positions/pixels are +uploaded before replay. An eager miss completes before recording, preserving its +answer. Capture failure logs unconditionally, disables that graph path and keeps +the eager result. Eviction synchronizes before graph/buffer destruction. + +Vision scratch persists for up to four grids even with graphs disabled, changing +resident activation memory from per-call allocation. This bounds grid slots, +not weights or GEMM plan memory. Startup's image warmup can now capture its own +shape when enabled; it does not prewarm every user grid. These mechanisms do not +establish lower latency on every workload or fix the multi-image numerical gap. + ## Validation and known multi-image limitation The historical A800 BF16 corpus (13 requests / 16 questions, text and single-image From 9d4436ef33f57f5a6f353bdda2c7a61d61dd541a Mon Sep 17 00:00:00 2001 From: levius <2114377220@qq.com> Date: Sat, 10 Oct 2026 22:34:45 +0800 Subject: [PATCH 07/21] Apply separate FP32 language LoRA before native projection consumers --- src/models/qwen3_5/native/src/lib.rs | 2 + src/models/qwen3_5/native/src/lora.rs | 502 +++++++++++++++++++++++++ src/models/qwen3_5/native/src/model.rs | 142 ++++++- tests/qwen3_5/lora.rs | 430 +++++++++++++++++++++ tests/qwen3_5/lora_fake.c | 40 ++ 5 files changed, 1115 insertions(+), 1 deletion(-) create mode 100644 src/models/qwen3_5/native/src/lora.rs create mode 100644 tests/qwen3_5/lora.rs create mode 100644 tests/qwen3_5/lora_fake.c diff --git a/src/models/qwen3_5/native/src/lib.rs b/src/models/qwen3_5/native/src/lib.rs index d63f746a..80880570 100644 --- a/src/models/qwen3_5/native/src/lib.rs +++ b/src/models/qwen3_5/native/src/lib.rs @@ -8,3 +8,5 @@ pub mod inputs; pub mod image_preprocess; pub mod vision; + +mod lora; diff --git a/src/models/qwen3_5/native/src/lora.rs b/src/models/qwen3_5/native/src/lora.rs new file mode 100644 index 00000000..44ff43c1 --- /dev/null +++ b/src/models/qwen3_5/native/src/lora.rs @@ -0,0 +1,502 @@ +//! Separate, immutable FP32 language LoRA weights and per-projection execution. +use crate::{ + cuda::{self, DeviceBuffer, Stream, check}, + model::Config, +}; +use anyhow::{Context, Result, ensure}; +use std::{collections::BTreeMap, ffi::c_void, ops::Range, path::Path}; + +const PREFIX: &str = "base_model.model.model.language_model."; +const MAX_RANK: usize = 128; +const MAX_HEADER: usize = 8 << 20; + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) enum Group { + LinearInput, + LinearOutput, + AttentionInput, + AttentionOutput, + GateUp, + Down, +} +impl Group { + fn index(self) -> usize { + self as usize + } +} +#[derive(Clone, Debug)] +struct Target { + name: String, + layer: usize, + group: Group, + rows: usize, + cols: usize, + offset: usize, + stride: usize, +} +struct Pair { + target: Target, + rank: usize, + a: Range, + b: Range, +} +type Components = [Option<(Range, usize)>; 2]; + +struct Parsed { + pairs: Vec, +} + +fn targets(cfg: &Config) -> Result> { + let mul = |a: usize, b: usize| a.checked_mul(b).context("LoRA dimension overflow"); + let add = |a: usize, b: usize| a.checked_add(b).context("LoRA dimension overflow"); + ensure!( + !cfg.full_attention.is_empty() && cfg.full_attention.len() <= 1024, + "LoRA layer count" + ); + let (h, i) = (cfg.hidden, cfg.intermediate); + let (kd, vd) = ( + mul(cfg.lin_k_heads, cfg.lin_k_dim)?, + mul(cfg.lin_v_heads, cfg.lin_v_dim)?, + ); + let q = mul(mul(cfg.heads, cfg.head_dim)?, 2)?; + let kv = mul(cfg.kv_heads, cfg.head_dim)?; + let conv = add(mul(2, kd)?, vd)?; + let lin_stride = add(add(conv, vd)?, mul(2, cfg.lin_v_heads)?)?; + let attn_stride = add(q, mul(2, kv)?)?; + let mut targets = Vec::new(); + for (layer, &full) in cfg.full_attention.iter().enumerate() { + let mut push = |name: &str, + group: Group, + rows: usize, + cols: usize, + offset: usize, + stride: usize| + -> Result<()> { + ensure!( + rows > 0 && cols > 0 && stride <= i32::MAX as usize && cols <= i32::MAX as usize, + "LoRA dimensions" + ); + ensure!( + offset.checked_add(rows).is_some_and(|end| end <= stride), + "LoRA output slice" + ); + targets.push(Target { + name: format!("layers.{layer}.{name}"), + layer, + group, + rows, + cols, + offset, + stride, + }); + Ok(()) + }; + if full { + push( + "self_attn.q_proj", + Group::AttentionInput, + q, + h, + 0, + attn_stride, + )?; + push( + "self_attn.k_proj", + Group::AttentionInput, + kv, + h, + q, + attn_stride, + )?; + push( + "self_attn.v_proj", + Group::AttentionInput, + kv, + h, + add(q, kv)?, + attn_stride, + )?; + push("self_attn.o_proj", Group::AttentionOutput, h, q / 2, 0, h)?; + } else { + push( + "linear_attn.in_proj_qkv", + Group::LinearInput, + conv, + h, + 0, + lin_stride, + )?; + push( + "linear_attn.in_proj_z", + Group::LinearInput, + vd, + h, + conv, + lin_stride, + )?; + push( + "linear_attn.in_proj_b", + Group::LinearInput, + cfg.lin_v_heads, + h, + add(conv, vd)?, + lin_stride, + )?; + push( + "linear_attn.in_proj_a", + Group::LinearInput, + cfg.lin_v_heads, + h, + add(add(conv, vd)?, cfg.lin_v_heads)?, + lin_stride, + )?; + push("linear_attn.out_proj", Group::LinearOutput, h, vd, 0, h)?; + } + push("mlp.gate_proj", Group::GateUp, i, h, 0, mul(2, i)?)?; + push("mlp.up_proj", Group::GateUp, i, h, i, mul(2, i)?)?; + push("mlp.down_proj", Group::Down, h, i, 0, h)?; + } + Ok(targets) +} + +fn parse(cfg: &Config, bytes: &[u8], scale: f32) -> Result { + ensure!( + scale.is_finite() && scale > 0., + "LoRA scale must be finite and positive" + ); + let allowed: BTreeMap<_, _> = targets(cfg)? + .into_iter() + .map(|t| (t.name.clone(), t)) + .collect(); + let header_len = usize::try_from(u64::from_le_bytes( + bytes.get(..8).context("LoRA header length")?.try_into()?, + ))?; + ensure!(header_len <= MAX_HEADER, "LoRA header too large"); + let start = header_len.checked_add(8).context("LoRA header overflow")?; + let header = crate::json::parse(bytes.get(8..start).context("truncated LoRA header")?) + .map_err(anyhow::Error::msg)?; + ensure!( + header.len() <= allowed.len() * 2 + 1, + "too many LoRA tensors" + ); + let mut components: BTreeMap = BTreeMap::new(); + for (name, info) in header { + if name == "__metadata__" { + continue; + } + let native = name + .strip_prefix(PREFIX) + .context("unsupported LoRA tensor prefix")?; + let (target, letter) = if let Some(target) = native.strip_suffix(".lora_A.weight") { + (target, 0) + } else { + ( + native + .strip_suffix(".lora_B.weight") + .context("unsupported LoRA tensor suffix")?, + 1, + ) + }; + let layout = allowed + .get(target) + .with_context(|| format!("unsupported LoRA target {target}"))?; + ensure!(info["dtype"] == "F32", "LoRA tensor {name} must be FP32"); + let shape: [usize; 2] = serde_json::from_value(info["shape"].clone()) + .context("LoRA tensor must be a matrix")?; + let rank = if letter == 0 { shape[0] } else { shape[1] }; + ensure!( + (1..=MAX_RANK).contains(&rank), + "LoRA rank must be 1..={MAX_RANK}" + ); + ensure!( + shape + == if letter == 0 { + [rank, layout.cols] + } else { + [layout.rows, rank] + }, + "LoRA shape mismatch for {name}" + ); + let [begin, end]: [usize; 2] = + serde_json::from_value(info["data_offsets"].clone()).context("LoRA tensor offsets")?; + let size = shape[0] + .checked_mul(shape[1]) + .and_then(|n| n.checked_mul(4)) + .context("LoRA tensor size overflow")?; + ensure!( + end.checked_sub(begin) == Some(size), + "LoRA tensor byte count" + ); + let range = start.checked_add(begin).context("LoRA offset overflow")? + ..start.checked_add(end).context("LoRA offset overflow")?; + let data = bytes + .get(range.clone()) + .context("LoRA tensor exceeds source")?; + ensure!( + data.as_chunks::<4>() + .0 + .iter() + .all(|&v| f32::from_le_bytes(v).is_finite()), + "nonfinite LoRA tensor {name}" + ); + let halves = components.entry(target.into()).or_default(); + ensure!( + halves[letter].replace((range, rank)).is_none(), + "duplicate LoRA component" + ); + } + ensure!(!components.is_empty(), "empty LoRA adapter"); + // All sizes/offsets are bounded above before safetensors performs its payload checks. + safetensors::SafeTensors::deserialize(bytes).context("invalid LoRA safetensors")?; + let pairs = components + .into_iter() + .map(|(name, [a, b])| { + let (a, rank) = a.with_context(|| format!("unpaired LoRA target {name}: missing A"))?; + let (b, b_rank) = + b.with_context(|| format!("unpaired LoRA target {name}: missing B"))?; + ensure!(rank == b_rank, "LoRA A/B rank mismatch for {name}"); + Ok(Pair { + target: allowed[&name].clone(), + rank, + a, + b, + }) + }) + .collect::>()?; + Ok(Parsed { pairs }) +} + +pub(crate) struct Checkpoint { + map: memmap2::Mmap, + parsed: Parsed, + scale: f32, +} +impl Checkpoint { + pub(crate) fn load(cfg: &Config, path: &Path, scale: f32) -> Result { + ensure!( + scale.is_finite() && scale > 0., + "LoRA scale must be finite and positive" + ); + let file = std::fs::File::open(path) + .with_context(|| format!("LoRA adapter {}", path.display()))?; + let max_bytes = targets(cfg)?.iter().try_fold(MAX_HEADER + 8, |n, t| { + t.rows + .checked_add(t.cols) + .and_then(|v| v.checked_mul(MAX_RANK * 4)) + .and_then(|v| n.checked_add(v)) + .context("LoRA maximum size overflow") + })?; + ensure!( + file.metadata()?.len() <= u64::try_from(max_bytes)?, + "LoRA source exceeds configured tensor bound" + ); + // SAFETY: the source checkpoint must remain immutable while validation/upload runs. + let map = unsafe { memmap2::Mmap::map(&file)? }; + let parsed = parse(cfg, &map, scale)?; + Ok(Self { map, parsed, scale }) + } + pub(crate) fn upload(self, layers: usize, stream: Stream) -> Result { + let mut result = Lora { + layers: (0..layers) + .map(|_| std::array::from_fn(|_| Vec::new())) + .collect(), + scale: self.scale, + layout: WorkLayout::for_pairs(1, &self.parsed.pairs)?.unwrap(), + }; + for pair in self.parsed.pairs { + let a = DeviceBuffer::new(pair.a.len())?; + let b = DeviceBuffer::new(pair.b.len())?; + // SAFETY: source and device buffers have the validated exact FP32 byte counts. + unsafe { + cuda::upload(a.at(0), &self.map[pair.a], stream)?; + cuda::upload(b.at(0), &self.map[pair.b], stream)?; + } + result.layers[pair.target.layer][pair.target.group.index()].push(GpuPair { + target: pair.target, + rank: pair.rank, + a, + b, + }); + } + Ok(result) + } +} + +#[derive(Clone, Copy)] +struct WorkLayout { + input: usize, + rank: usize, + delta: usize, + gather: usize, +} +impl WorkLayout { + fn for_pairs(rows: usize, pairs: &[Pair]) -> Result> { + if pairs.is_empty() { + return Ok(None); + } + let bytes = |width: usize, element: usize| { + rows.checked_mul(width) + .and_then(|n| n.checked_mul(element)) + .context("LoRA work size overflow") + }; + let cols = pairs.iter().map(|p| p.target.cols).max().unwrap(); + let outputs = pairs.iter().map(|p| p.target.rows).max().unwrap(); + let rank = pairs.iter().map(|p| p.rank).max().unwrap(); + Ok(Some(Self { + input: bytes(cols, 4)?, + rank: bytes(rank, 4)?, + delta: bytes(outputs, 4)?, + gather: bytes(outputs, 2)?, + })) + } + fn scaled(self, rows: usize) -> Result { + let scale = |n: usize| n.checked_mul(rows).context("LoRA work size overflow"); + Ok(Self { + input: scale(self.input)?, + rank: scale(self.rank)?, + delta: scale(self.delta)?, + gather: scale(self.gather)?, + }) + } +} +struct GpuPair { + target: Target, + rank: usize, + a: DeviceBuffer, + b: DeviceBuffer, +} +pub(crate) struct Lora { + layers: Vec<[Vec; 6]>, + scale: f32, + layout: WorkLayout, +} +pub(crate) struct Work { + input: DeviceBuffer, + rank: DeviceBuffer, + delta: DeviceBuffer, + gather: DeviceBuffer, + rows: usize, +} +impl Lora { + pub(crate) fn work(&self, rows: usize) -> Result { + let layout = self.layout.scaled(rows)?; + Ok(Work { + input: DeviceBuffer::new(layout.input)?, + rank: DeviceBuffer::new(layout.rank)?, + delta: DeviceBuffer::new(layout.delta)?, + gather: DeviceBuffer::new(layout.gather)?, + rows, + }) + } + #[allow(clippy::too_many_arguments)] // Validated projection layout plus exclusively owned device work. + pub(crate) unsafe fn apply( + &self, + layer: usize, + group: Group, + input: *const c_void, + output: *mut c_void, + lengths: &[usize], + work: &Work, + gemm: *mut c_void, + stream: Stream, + ) -> Result<()> { + let pairs = &self.layers[layer][group.index()]; + if pairs.is_empty() { + return Ok(()); + } + let (cols, stride) = (pairs[0].target.cols, pairs[0].target.stride); + let mut row = 0; + for &rows in lengths { + ensure!(rows > 0 && rows <= work.rows, "LoRA work row bound"); + // Each prompt keeps its own FP32 GEMM M, including packed base projections. + unsafe { + check( + (cuda::api().cs1_vision_to_float)( + input.wrapping_byte_add(row * cols * 2), + work.input.at(0).cast(), + rows * cols, + stream, + ), + "language LoRA input", + )?; + for pair in pairs { + let n = pair.target.rows; + check( + (cuda::api().cs1_gemm_f32)( + gemm, + work.input.at(0).cast(), + pair.a.at(0).cast(), + work.rank.at(0).cast(), + rows as i32, + pair.rank as i32, + cols as i32, + stream, + ), + "language LoRA A", + )?; + check( + (cuda::api().cs1_gemm_f32)( + gemm, + work.rank.at(0).cast(), + pair.b.at(0).cast(), + work.delta.at(0).cast(), + rows as i32, + n as i32, + pair.rank as i32, + stream, + ), + "language LoRA B", + )?; + let out = output.wrapping_byte_add((row * stride + pair.target.offset) * 2); + if n == stride { + check( + (cuda::api().cs1_vision_lora_add)( + out, + work.delta.at(0).cast(), + rows * n, + self.scale, + stream, + ), + "language LoRA add", + )?; + } else { + cuda::copy2d( + work.gather.at(0), + n * 2, + out, + stride * 2, + n * 2, + rows, + stream, + )?; + check( + (cuda::api().cs1_vision_lora_add)( + work.gather.at(0), + work.delta.at(0).cast(), + rows * n, + self.scale, + stream, + ), + "language LoRA add", + )?; + cuda::copy2d( + out, + stride * 2, + work.gather.at(0), + n * 2, + n * 2, + rows, + stream, + )?; + } + } + } + row += rows; + } + Ok(()) + } +} + +#[cfg(test)] +#[path = "../../../../../tests/qwen3_5/lora.rs"] +mod tests; diff --git a/src/models/qwen3_5/native/src/model.rs b/src/models/qwen3_5/native/src/model.rs index 304c3083..6a353fbb 100644 --- a/src/models/qwen3_5/native/src/model.rs +++ b/src/models/qwen3_5/native/src/model.rs @@ -19,6 +19,7 @@ use serde_json::Value as Json; use crate::cuda::{self, DeviceBuffer, Stream, check}; use crate::inputs::{MultimodalInput, rotary_tables}; +use crate::lora::{self, Group}; const ALIGN: usize = 256; const BF16: usize = 2; @@ -439,10 +440,11 @@ struct Scratch { sin: usize, custom_cos: usize, custom_sin: usize, + lora: Option, } impl Scratch { - fn new(cfg: &Config, cap: usize, stream: Stream) -> Result { + fn new(cfg: &Config, cap: usize, stream: Stream, lora: Option<&lora::Lora>) -> Result { let (h, kd, vd, hv) = (cfg.hidden, cfg.key_dim(), cfg.value_dim(), cfg.lin_v_heads); let (hq, hk, hd) = (cfg.heads, cfg.kv_heads, cfg.head_dim); let w = Widths::of(cfg); @@ -546,6 +548,9 @@ impl Scratch { sin, custom_cos, custom_sin, + lora: lora + .map(|adapter| adapter.work(cap.min(cfg.max_positions))) + .transpose()?, }) } @@ -594,6 +599,7 @@ pub struct Model { embed: Tensor, final_norm: Tensor, layers: Vec, + lora: Option, stream: Stream, gemm: *mut c_void, /// Buffers for the largest packed token count so far; grows as needed. @@ -601,15 +607,32 @@ pub struct Model { /// Opt-in replay with at most 64 captures keyed by ordered sequence lengths. graph_enabled: bool, graphs: VecDeque<(Vec, cuda::Graph)>, + // The opt-in adapter path owns its stream; it retires after every device allocation. + _lora_stream: Option, } // SAFETY: the raw pointers are device addresses and a cuBLASLt handle owned by the // model; the engine runs one forward pass at a time behind a mutex. unsafe impl Send for Model {} +struct LoraStream(Stream); +impl Drop for LoraStream { + fn drop(&mut self) { + // SAFETY: the adapter Model owns this stream, and has retired all allocations. + unsafe { (cuda::api().cs1_stream_destroy)(self.0) }; + } +} + impl Drop for Model { fn drop(&mut self) { + if self.lora.is_some() { + let _ = cuda::set_device(0); + let _ = cuda::synchronize(self.stream); + } self.graphs.clear(); + if self.lora.is_some() { + self.scratch = None; + } // SAFETY: created by cs1_gemm_create and not destroyed before. unsafe { (cuda::api().cs1_gemm_destroy)(self.gemm) }; } @@ -619,10 +642,37 @@ impl Model { /// Load the CUDA library and the weights. pub fn load(dir: &Path, library: &Path) -> Result { let cfg = Config::load(dir)?; + Self::load_checkpoint(dir, library, cfg, None) + } + + /// Load an immutable FP32 PEFT language adapter without merging it into BF16 weights. + /// `adapter_path` names one safetensors file; partial paired targets and ranks 1..=128 + /// are supported. Validate the pinned full inventory separately when required. + pub fn load_with_lora( + dir: &Path, + library: &Path, + adapter_path: &Path, + scale: f32, + ) -> Result { + let cfg = Config::load(dir)?; + let adapter = lora::Checkpoint::load(&cfg, adapter_path, scale)?; + Self::load_checkpoint(dir, library, cfg, Some(adapter)) + } + + fn load_checkpoint( + dir: &Path, + library: &Path, + cfg: Config, + adapter: Option, + ) -> Result { cuda::load(library)?; cuda::set_device(0)?; let stream = cuda::new_stream()?; + let stream_owner = adapter.as_ref().map(|_| LoraStream(stream)); let weights = Weights::load(dir, stream)?; + let lora = adapter + .map(|source| source.upload(cfg.full_attention.len(), stream)) + .transpose()?; let (h, kd, vd) = (cfg.hidden, cfg.key_dim(), cfg.value_dim()); let embed = weights .tensors @@ -687,11 +737,13 @@ impl Model { embed, final_norm, layers, + lora, stream, gemm, scratch: None, graph_enabled: std::env::var("CUA_S1_GRAPH").as_deref() == Ok("1"), graphs: VecDeque::new(), + _lora_stream: stream_owner, }; Ok(model) } @@ -717,6 +769,36 @@ impl Model { ) } + #[allow(clippy::too_many_arguments)] // Layer projection coordinates in shared scratch. + fn apply_lora( + &self, + layer: usize, + group: Group, + s: &Scratch, + input: usize, + output: usize, + lengths: &[usize], + ) -> Result<()> { + let Some(adapter) = &self.lora else { + return Ok(()); + }; + let work = s.lora.as_ref().context("missing language LoRA work")?; + // SAFETY: caller passes the validated projection's input/output ranges in scratch; + // per-sequence row counts are bounded by config, and work/weights belong to this Model. + unsafe { + adapter.apply( + layer, + group, + s.at(input), + s.at(output), + lengths, + work, + self.gemm, + self.stream, + ) + } + } + /// Preserve each prompt's output/down GEMM shape and split-K reduction order. fn gemm_sequences( &self, @@ -744,12 +826,16 @@ impl Model { fn prepare_scratch(&mut self, t: usize) -> Result<()> { cuda::set_device(0)?; if self.scratch.as_ref().is_none_or(|s| t > s.cap) { + if self.lora.is_some() { + cuda::synchronize(self.stream)?; + } self.graphs.clear(); self.scratch = None; self.scratch = Some(Scratch::new( &self.cfg, t.next_multiple_of(1024), self.stream, + self.lora.as_ref(), )?); } Ok(()) @@ -1112,6 +1198,7 @@ impl Model { match &layer.mixer { Mixer::Linear(la) => { self.gemm(s, s.x, &la.in_proj, s.gdn_in, t)?; + self.apply_lora(i, Group::LinearInput, s, s.x, s.gdn_in, lengths)?; let ld = w.gdn_in as i32; let z = s.gdn_in + w.conv * BF16; let b = z + vd * BF16; @@ -1185,9 +1272,11 @@ impl Model { )?; } self.gemm_sequences(s, s.ln, &la.out, s.delta, lengths)?; + self.apply_lora(i, Group::LinearOutput, s, s.ln, s.delta, lengths)?; } Mixer::Full(fa) => { self.gemm(s, s.x, &fa.qkv, s.attn_in, t)?; + self.apply_lora(i, Group::AttentionInput, s, s.x, s.attn_in, lengths)?; let ld = w.attn_in as i32; let k = s.attn_in + w.attn_q * BF16; let v = k + cfg.kv_heads * cfg.head_dim * BF16; @@ -1235,6 +1324,7 @@ impl Model { } } self.gemm_sequences(s, s.ao, &fa.o, s.delta, lengths)?; + self.apply_lora(i, Group::AttentionOutput, s, s.ao, s.delta, lengths)?; } } unsafe { @@ -1253,6 +1343,7 @@ impl Model { )?; } self.gemm(s, s.x, &layer.gate_up, s.gate_up, t)?; + self.apply_lora(i, Group::GateUp, s, s.x, s.gate_up, lengths)?; unsafe { check( (cuda::api().cs1_silu_mul)( @@ -1267,6 +1358,7 @@ impl Model { )?; } self.gemm_sequences(s, s.act, &layer.down, s.delta, lengths)?; + self.apply_lora(i, Group::Down, s, s.act, s.delta, lengths)?; let next = self .layers .get(i + 1) @@ -1360,6 +1452,14 @@ impl Model { match &layer.mixer { Mixer::Linear(la) => { self.gemm(s, s.x + qb * hb, &la.in_proj, s.gdn_in + qb * ldbb, rows)?; + self.apply_lora( + i, + Group::LinearInput, + s, + s.x + qb * hb, + s.gdn_in + qb * ldbb, + &[rows], + )?; let ld = ldb as i32; let z = s.gdn_in + w.conv * BF16; let b = z + vd * BF16; @@ -1460,10 +1560,26 @@ impl Model { )?; } self.gemm(s, s.ln + qb * vb, &la.out, s.delta + qb * hb, rows)?; + self.apply_lora( + i, + Group::LinearOutput, + s, + s.ln + qb * vb, + s.delta + qb * hb, + &[rows], + )?; la_i += 1; } Mixer::Full(fa) => { self.gemm(s, s.x + qb * hb, &fa.qkv, s.attn_in + qb * ab, rows)?; + self.apply_lora( + i, + Group::AttentionInput, + s, + s.x + qb * hb, + s.attn_in + qb * ab, + &[rows], + )?; let k = s.attn_in + w.attn_q * BF16; let v = k + kvb; let (q_base, t_flash) = (qb as i32, tend as i32); @@ -1522,6 +1638,14 @@ impl Model { )?; } self.gemm(s, s.ao + qb * ob, &fa.o, s.delta + qb * hb, rows)?; + self.apply_lora( + i, + Group::AttentionOutput, + s, + s.ao + qb * ob, + s.delta + qb * hb, + &[rows], + )?; fa_i += 1; } } @@ -1547,6 +1671,14 @@ impl Model { s.gate_up + qb * 2 * cfg.intermediate * BF16, rows, )?; + self.apply_lora( + i, + Group::GateUp, + s, + s.x + qb * hb, + s.gate_up + qb * 2 * cfg.intermediate * BF16, + &[rows], + )?; unsafe { check( (cuda::api().cs1_silu_mul)( @@ -1567,6 +1699,14 @@ impl Model { s.delta + qb * hb, rows, )?; + self.apply_lora( + i, + Group::Down, + s, + s.act + qb * cfg.intermediate * BF16, + s.delta + qb * hb, + &[rows], + )?; let next = self .layers .get(i + 1) diff --git a/tests/qwen3_5/lora.rs b/tests/qwen3_5/lora.rs new file mode 100644 index 00000000..9cffc6bf --- /dev/null +++ b/tests/qwen3_5/lora.rs @@ -0,0 +1,430 @@ +use super::*; +use safetensors::{ + Dtype, + tensor::{TensorView, serialize}, +}; + +fn config() -> Config { + Config::load(Path::new(concat!( + env!("CARGO_MANIFEST_DIR"), + "/../../../../tests/qwen3_5/data" + ))) + .unwrap() +} +fn tensors(values: &[(&str, Vec, Dtype, f32)]) -> Vec { + let data: Vec> = values + .iter() + .map(|(_, shape, dtype, value)| { + let count: usize = shape.iter().product(); + match dtype { + Dtype::F32 => value.to_le_bytes().repeat(count), + Dtype::BF16 => half::bf16::from_f32(*value).to_le_bytes().repeat(count), + _ => unreachable!(), + } + }) + .collect(); + let views: Vec<_> = values + .iter() + .zip(&data) + .map(|((name, shape, dtype, _), bytes)| { + ( + *name, + TensorView::new(*dtype, shape.clone(), bytes).unwrap(), + ) + }) + .collect(); + serialize(views, None).unwrap() +} +fn pair(target: &str, rows: usize, cols: usize, rank: usize) -> Vec { + tensors(&[ + ( + &format!("{PREFIX}{target}.lora_A.weight"), + vec![rank, cols], + Dtype::F32, + 0.125, + ), + ( + &format!("{PREFIX}{target}.lora_B.weight"), + vec![rows, rank], + Dtype::F32, + -0.25, + ), + ]) +} + +#[test] +fn layouts_cover_both_mixers_and_the_small_gate_projections() { + for (hidden, intermediate, heads, values) in [(5120, 17408, 24, 48), (2560, 9216, 16, 32)] { + let mut cfg = config(); + cfg.hidden = hidden; + cfg.intermediate = intermediate; + cfg.heads = heads; + cfg.lin_v_heads = values; + let layouts = targets(&cfg).unwrap(); + let find = |name: &str| layouts.iter().find(|t| t.name == name).unwrap(); + let vd = values * 128; + let kd = cfg.lin_k_heads * 128; + let small = find("layers.0.linear_attn.in_proj_a"); + assert_eq!( + (small.rows, small.cols, small.offset, small.stride), + ( + values, + hidden, + 2 * kd + 2 * vd + values, + 2 * kd + 2 * vd + 2 * values + ) + ); + let q = find("layers.3.self_attn.q_proj"); + let k = find("layers.3.self_attn.k_proj"); + assert_eq!((q.rows, q.cols, q.offset), (heads * 256 * 2, hidden, 0)); + assert_eq!(k.offset, q.rows); + let up = find("layers.0.mlp.up_proj"); + assert_eq!( + (up.rows, up.cols, up.offset, up.stride), + (intermediate, hidden, intermediate, 2 * intermediate) + ); + assert_eq!(layouts.len(), 496); + assert!( + !layouts + .iter() + .any(|t| t.name == "layers.0.self_attn.q_proj") + ); + } +} +#[test] +fn partial_pairs_keep_fp32_bytes_and_match_native_component_layouts() { + let cfg = config(); + for (target, rows, cols) in [ + ("layers.0.linear_attn.in_proj_a", 48, 5120), + ("layers.3.self_attn.q_proj", 12288, 5120), + ("layers.0.mlp.down_proj", 5120, 17408), + ] { + let bytes = pair(target, rows, cols, 16); + let parsed = parse(&cfg, &bytes, 2.0).unwrap(); + assert_eq!(parsed.pairs.len(), 1); + let p = &parsed.pairs[0]; + assert_eq!((p.rank, p.target.rows, p.target.cols), (16, rows, cols)); + assert_eq!(&bytes[p.a.clone()][..4], &0.125f32.to_le_bytes()); + assert_eq!(&bytes[p.b.clone()][..4], &(-0.25f32).to_le_bytes()); + } +} +#[test] +fn rank_dtype_shapes_pairing_targets_and_nonfinite_values_are_rejected() { + let cfg = config(); + for rank in [0, 129] { + assert!( + parse( + &cfg, + &pair("layers.0.linear_attn.in_proj_a", 48, 5120, rank), + 2. + ) + .is_err() + ); + } + for scale in [0., -1., f32::NAN, f32::INFINITY] { + assert!( + parse( + &cfg, + &pair("layers.0.linear_attn.in_proj_a", 48, 5120, 1), + scale + ) + .is_err() + ); + } + for (target, rows, cols) in [ + ("layers.0.linear_attn.in_proj_a", 49, 5120), + ("layers.0.linear_attn.in_proj_a", 48, 5121), + ("layers.0.self_attn.q_proj", 12288, 5120), + ("layers.64.mlp.down_proj", 5120, 17408), + ("layers.0.input_layernorm", 5120, 1), + ] { + assert!(parse(&cfg, &pair(target, rows, cols, 1), 2.).is_err()); + } + let a = format!("{PREFIX}layers.0.linear_attn.in_proj_a.lora_A.weight"); + let b = format!("{PREFIX}layers.0.linear_attn.in_proj_a.lora_B.weight"); + assert!(parse(&cfg, &tensors(&[(&a, vec![1, 5120], Dtype::F32, 0.)]), 2.).is_err()); + assert!(parse(&cfg, &tensors(&[(&b, vec![48, 1], Dtype::F32, 0.)]), 2.).is_err()); + for bad in [f32::NAN, f32::INFINITY, f32::NEG_INFINITY] { + for letter in [0, 1] { + let vals = [ + ( + &*a, + vec![1, 5120], + Dtype::F32, + if letter == 0 { bad } else { 0. }, + ), + ( + &*b, + vec![48, 1], + Dtype::F32, + if letter == 1 { bad } else { 0. }, + ), + ]; + assert!(parse(&cfg, &tensors(&vals), 2.).is_err()); + } + } + assert!( + parse( + &cfg, + &tensors(&[ + (&a, vec![1, 5120], Dtype::BF16, 0.), + (&b, vec![48, 1], Dtype::F32, 0.) + ]), + 2. + ) + .is_err() + ); + assert!(parse(&cfg, &tensors(&[]), 2.).is_err()); +} +#[test] +fn optional_work_is_absent_without_adapters_and_checked_for_overflow() { + assert!(WorkLayout::for_pairs(8192, &[]).unwrap().is_none()); + let bytes = pair("layers.0.linear_attn.in_proj_a", 48, 5120, 16); + let parsed = parse(&config(), &bytes, 2.).unwrap(); + let work = WorkLayout::for_pairs(7, &parsed.pairs).unwrap().unwrap(); + assert_eq!( + (work.input, work.rank, work.delta, work.gather), + (7 * 5120 * 4, 7 * 16 * 4, 7 * 48 * 4, 7 * 48 * 2) + ); + assert!(WorkLayout::for_pairs(usize::MAX, &parsed.pairs).is_err()); +} + +struct Temporary(std::path::PathBuf); +impl Temporary { + fn new() -> Self { + let unique = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap() + .as_nanos(); + let path = std::env::temp_dir().join(format!("qwen-lora-{}-{unique}", std::process::id())); + std::fs::create_dir(&path).unwrap(); + Self(path) + } +} +impl Drop for Temporary { + fn drop(&mut self) { + let _ = std::fs::remove_dir_all(&self.0); + } +} + +#[test] +fn malformed_adapter_is_rejected_before_loading_cuda() { + let temporary = Temporary::new(); + let path = temporary.0.join("adapter.safetensors"); + let bytes = pair("layers.0.linear_attn.in_proj_a", 49, 5120, 1); + std::fs::write(&path, bytes).unwrap(); + let config = Path::new(concat!( + env!("CARGO_MANIFEST_DIR"), + "/../../../../tests/qwen3_5/data" + )); + let error = + crate::model::Model::load_with_lora(config, Path::new("/missing/CUDA-library"), &path, 2.) + .err() + .unwrap(); + assert!( + error.to_string().contains("LoRA shape mismatch"), + "{error:#}" + ); +} + +#[cfg(unix)] +#[test] +fn cpu_cuda_boundary_checks_fp32_sums_fused_offsets_sequences_and_retirement() { + let temporary = Temporary::new(); + let source = temporary.0.join("fake.c"); + let library = temporary.0.join(if cfg!(target_os = "macos") { + "fake.dylib" + } else { + "fake.so" + }); + let mut code = include_str!("lora_fake.c").to_owned(); + let implemented = [ + "cs1_abi_version", + "cs1_error_string", + "cs1_malloc", + "cs1_free", + "cs1_stream_create", + "cs1_stream_sync", + "cs1_stream_destroy", + "cs1_upload", + "cs1_download", + "cs1_copy2d", + "cs1_vision_to_float", + "cs1_gemm_f32", + "cs1_vision_lora_add", + ]; + // Required but unused functions return an error, so an unexpected dispatch fails. + let cuda_source = include_str!("../../src/models/qwen3_5/native/src/cuda.rs"); + let abi = cuda_source + .lines() + .find(|line| line.starts_with("const ABI_VERSION:")) + .unwrap() + .split('=') + .nth(1) + .unwrap() + .trim() + .trim_end_matches(';'); + let api = cuda_source + .split("api! {") + .nth(1) + .unwrap() + .split("\n}\n") + .next() + .unwrap(); + for line in api.lines() { + let line = line.trim(); + if !line.starts_with("cs1_") { + continue; + } + let name = line.split('(').next().unwrap(); + if !implemented.contains(&name) { + code.push_str(&format!("\nint {name}(void) {{ return 77; }}\n")); + } + } + std::fs::write(&source, code).unwrap(); + let output = std::process::Command::new("cc") + .args(["-shared", "-fPIC", "-ffp-contract=off"]) + .arg(format!("-DLORA_FAKE_ABI={abi}")) + .arg(&source) + .arg("-o") + .arg(&library) + .output() + .unwrap(); + assert!( + output.status.success(), + "{}", + String::from_utf8_lossy(&output.stderr) + ); + cuda::load(&library).unwrap(); + // SAFETY: these symbols are defined immediately above and live for this test. + let metrics = unsafe { libloading::Library::new(&library).unwrap() }; + let live = unsafe { + *metrics + .get:: i32>(b"lora_fake_live\0") + .unwrap() + }; + let gemms = unsafe { + *metrics + .get:: i32>(b"lora_fake_gemms\0") + .unwrap() + }; + let get_m = unsafe { + *metrics + .get:: i32>(b"lora_fake_m\0") + .unwrap() + }; + let copies = unsafe { + *metrics + .get:: i32>(b"lora_fake_copies\0") + .unwrap() + }; + let height = unsafe { + *metrics + .get:: i32>(b"lora_fake_height\0") + .unwrap() + }; + let reset = unsafe { + *metrics + .get::(b"lora_fake_reset\0") + .unwrap() + }; + let stream = cuda::new_stream().unwrap(); + let mut cfg = config(); + cfg.hidden = 8; + cfg.intermediate = 8; + cfg.heads = 1; + cfg.kv_heads = 1; + cfg.lin_k_heads = 1; + cfg.lin_v_heads = 1; + cfg.full_attention = vec![false, true]; + for (name, layer, group) in [ + ("layers.0.linear_attn.in_proj_b", 0, Group::LinearInput), + ("layers.1.self_attn.k_proj", 1, Group::AttentionInput), + ("layers.0.mlp.up_proj", 0, Group::GateUp), + ("layers.0.mlp.down_proj", 0, Group::Down), + ] { + let target = targets(&cfg) + .unwrap() + .into_iter() + .find(|t| t.name == name) + .unwrap(); + let path = temporary.0.join("adapter.safetensors"); + std::fs::write(&path, pair(name, target.rows, target.cols, 1)).unwrap(); + let checkpoint = Checkpoint::load(&cfg, &path, 2.).unwrap(); + unsafe { + reset(); + } + { + let adapter = checkpoint.upload(2, stream).unwrap(); + let work = adapter.work(2).unwrap(); + let input: Vec<_> = (1..=5) + .flat_map(|value| { + half::bf16::from_f32(value as f32) + .to_le_bytes() + .repeat(target.cols) + }) + .collect(); + let original = half::bf16::ONE.to_le_bytes().repeat(5 * target.stride); + let x = DeviceBuffer::new(input.len()).unwrap(); + let y = DeviceBuffer::new(original.len()).unwrap(); + // Start at absolute row 1 and preserve the first/last rows and every neighboring component. + unsafe { + cuda::upload(x.at(0), &input, stream).unwrap(); + cuda::upload(y.at(0), &original, stream).unwrap(); + adapter + .apply( + layer, + group, + x.at(target.cols * 2), + y.at(target.stride * 2), + &[1, 2], + &work, + std::ptr::null_mut(), + stream, + ) + .unwrap(); + } + let mut actual = vec![0; original.len()]; + unsafe { + cuda::download(&mut actual, y.at(0), stream).unwrap(); + } + for row in 0..5 { + for col in 0..target.stride { + let offset = (row * target.stride + col) * 2; + let value = + half::bf16::from_le_bytes(actual[offset..offset + 2].try_into().unwrap()); + let expected = if (1..4).contains(&row) + && (target.offset..target.offset + target.rows).contains(&col) + { + half::bf16::from_f32( + 1. - (row + 1) as f32 * target.cols as f32 * 0.125 * 0.25 * 2., + ) + } else { + half::bf16::ONE + }; + assert_eq!(value, expected, "{name}, row {row}, col {col}"); + } + } + assert_eq!(unsafe { gemms() }, 4); + assert_eq!( + (0..4).map(|i| unsafe { get_m(i) }).collect::>(), + [1, 1, 2, 2] + ); + assert_eq!( + unsafe { copies() }, + if target.rows == target.stride { 0 } else { 4 } + ); + if target.rows != target.stride { + assert_eq!( + (0..4).map(|i| unsafe { height(i) }).collect::>(), + [1, 1, 2, 2] + ); + } + } + assert_eq!( + unsafe { live() }, + 0, + "adapter/work/input/output allocations must retire" + ); + } +} diff --git a/tests/qwen3_5/lora_fake.c b/tests/qwen3_5/lora_fake.c new file mode 100644 index 00000000..1efeb5a2 --- /dev/null +++ b/tests/qwen3_5/lora_fake.c @@ -0,0 +1,40 @@ +// CPU implementation of the CUDA boundary used only by the LoRA dispatch test. +#include +#include +#include +static int live, gemms, copies, ms[128], heights[128]; +int lora_fake_live(void) { return live; } +int lora_fake_gemms(void) { return gemms; } +int lora_fake_m(int i) { return ms[i]; } +int lora_fake_copies(void) { return copies; } +int lora_fake_height(int i) { return heights[i]; } +void lora_fake_reset(void) { gemms=0; copies=0; } +uint32_t cs1_abi_version(void) { return LORA_FAKE_ABI; } +const char* cs1_error_string(int code) { (void)code; return "unexpected fake CUDA call"; } +int cs1_malloc(void** p, size_t n) { *p=malloc(n); if (!*p) return 1; ++live; return 0; } +int cs1_free(void* p) { --live; free(p); return 0; } +int cs1_stream_create(void** stream) { *stream=(void*)1; return 0; } +int cs1_stream_sync(void* stream) { (void)stream; return 0; } +int cs1_stream_destroy(void* stream) { (void)stream; return 0; } +int cs1_upload(void* dst, const void* src, size_t n, void* stream) { (void)stream; memcpy(dst,src,n); return 0; } +int cs1_download(void* dst, const void* src, size_t n, void* stream) { (void)stream; memcpy(dst,src,n); return 0; } +int cs1_copy2d(void* dst,size_t dpitch,const void* src,size_t spitch,size_t width,size_t height,void* stream) { + (void)stream; heights[copies++]=(int)height; + for (size_t row=0;row>16)&1))>>16); } +int cs1_vision_to_float(const uint16_t* x,float* y,size_t n,void* stream) { + (void)stream; for (size_t i=0;i Date: Sat, 10 Oct 2026 22:39:18 +0800 Subject: [PATCH 08/21] Preserve original Jemm language weights and FP32 LoRA in v2 exports --- recipe/jemm/README.md | 13 +- recipe/jemm/export.py | 81 ++++++------ src/models/jemm/README.md | 17 ++- src/models/jemm/native/src/artifacts.rs | 64 +++++++++ src/models/jemm/native/src/executor.rs | 4 +- src/models/jemm/native/src/processing.rs | 4 +- tests/jemm/artifacts.rs | 131 ++++++++++++++++++- tests/jemm/test_export.py | 158 +++++++++++++++++++++++ tests/jemm/tokenization.rs | 12 +- 9 files changed, 430 insertions(+), 54 deletions(-) create mode 100644 tests/jemm/test_export.py diff --git a/recipe/jemm/README.md b/recipe/jemm/README.md index 186fb029..06a1a1a3 100644 --- a/recipe/jemm/README.md +++ b/recipe/jemm/README.md @@ -2,7 +2,9 @@ Run from the repository root on Linux with a CUDA GPU (compute capability 8.0+), nvcc/cuBLASLt and Rust. Historical validation used one A800 80 GB. The raw base -and adapter need about 55.6 GB disk and the export about 52 GB more. Export is an +and adapter need about 55.6 GB disk. V2 exports hardlink unchanged files on the +same filesystem; cross-filesystem exports copy the 17 language shards and raw +FP32 adapter, requiring additional disk for those files. Export is an offline CPU operation using torch, safetensors and Transformers; the running Rust worker needs no Python serving process. Keep inputs/exports immutable. @@ -36,6 +38,10 @@ python recipe/jemm/export.py --base "$JEMM_DATA/base" \ The checked-in pinned inventory verifies every required input file and tensor. Interrupted exports may resume only with matching inputs, producer and recorded runtime. Do not substitute similarly named checkpoints or newer revisions. +The `jemm-native/2` format keeps the original BF16 language weights and FP32 +adapter separate, with rank 16, 496 pairs and scale 2. Use a new directory for v2; +legacy premerged v1 directories are rejected rather than reused. Hardlinked +inputs and outputs share file contents and must remain immutable. ## Build, launch and request @@ -71,6 +77,8 @@ claim that those inputs now pass. Native CPU/Metal execution is unsupported. ## Checks and reference ```sh +# CPU exporter tests, in the environment containing torch and safetensors: +python -m unittest discover -s tests/jemm -p 'test_export.py' -v cargo fmt --all --check cargo clippy --workspace --locked --all-targets -- -D warnings cargo test --workspace --locked @@ -81,7 +89,8 @@ JEMM_CUDA_LIB="$PWD/target/release/libqwen3_5_cuda.so" \ --ignored --test-threads=1 ``` -CPU checks compile the worker and exercise host contracts. They cannot prove the +CPU checks compile the worker and exercise host/export contracts. The new +unmerged FP32-adapter path has no fresh GPU parity claim in these host checks. They cannot prove the real image warmup, multi-image parity or device failure recovery. Hardware tests require a separately reserved device; normal CI compiles and ignores the CUDA case. diff --git a/recipe/jemm/export.py b/recipe/jemm/export.py index 54cc23a1..21f05b20 100644 --- a/recipe/jemm/export.py +++ b/recipe/jemm/export.py @@ -3,14 +3,15 @@ No full Transformers model is instantiated. Raw inputs remain untouched. The completion marker is written last; restart with the same arguments to verify and reuse atomically completed shards under the same producer script/runtime. -Changed producers require a new export directory. Needs roughly one base shard plus a -bounded FP32 LoRA delta in RAM, rather than two copies of the full model. +Changed producers require a new export directory. Language shards and the FP32 +adapter are hardlinked or copied byte for byte, never merged into BF16 weights. """ import argparse import gc import hashlib import importlib.metadata import json +import math import os import platform import shutil @@ -78,26 +79,18 @@ def atomic_tensors(path, tensors): os.replace(temporary, path) -def safe_merge(base, a, b, scale, chunk_rows=1024): - """PEFT safe_merge semantics: FP32 B@A delta, in-place BF16 addition. - - Clone the base first and reject nonfinite adapter values and the final - rounded BF16 result. The caller's base tensor never changes. - """ - if base.dtype != torch.bfloat16 or a.dtype != torch.float32 or b.dtype != torch.float32: - raise ValueError("expected BF16 base and FP32 JEMM adapter") - if base.ndim != 2 or a.ndim != 2 or b.ndim != 2 or (b.shape[0], a.shape[1]) != tuple(base.shape) or b.shape[1] != a.shape[0]: - raise ValueError("LoRA dimensions do not match base") - if chunk_rows < 1 or not torch.isfinite(a).all() or not torch.isfinite(b).all(): - raise ValueError("nonfinite LoRA tensors or invalid chunk size") - result = base.clone() - for start in range(0, base.shape[0], chunk_rows): - end = min(start + chunk_rows, base.shape[0]) - delta = (b[start:end] @ a) * scale - result[start:end] += delta - if not torch.isfinite(result[start:end]).all(): - raise ValueError("nonfinite merged BF16 tensor") - return result +def atomic_original(source, destination): + """Publish an unchanged regular source file, using a hardlink where possible.""" + source = Path(source).resolve(strict=True) + temporary = destination.with_suffix(destination.suffix + ".tmp") + temporary.unlink(missing_ok=True) + try: + os.link(source, temporary) + except OSError: + shutil.copyfile(source, temporary) + with temporary.open("rb") as stream: + os.fsync(stream.fileno()) + os.replace(temporary, destination) def validate_artifacts(directory, inventory): @@ -141,7 +134,7 @@ def adapter_pairs(base_inventory, adapter_inventory, config): if config.get("bias", "none") != "none": raise ValueError("adapter bias unsupported") rank, alpha = config["r"], config["lora_alpha"] - if rank < 1 or alpha <= 0: + if type(rank) is not int or rank < 1 or type(alpha) not in (int, float) or not 0 < alpha < float("inf"): raise ValueError("invalid LoRA rank/alpha") pairs = {} for name, spec in adapter_inventory.items(): @@ -177,6 +170,9 @@ def export_checkpoint(base, adapter, out, provenance, *, tokenizer=None, invento raise ValueError("export directory must be separate from source checkpoints") if any(provenance.get(key) != value for key, value in PINS.items()): raise ValueError("download provenance does not match pinned revisions") + manifest_path, progress_path = out / "jemm_export.json", out / "export_progress.json" + if manifest_path.exists() and json.loads(manifest_path.read_text()).get("format") != "jemm-native/2": + raise ValueError("legacy export is unsupported; jemm-native/2 requires a new export directory") inventory = inventory or json.loads(INVENTORY_PATH.read_text()) source_hashes = {"base": validate_artifacts(base, inventory["base"]), "adapter": validate_artifacts(adapter, inventory["adapter"])} index = json.loads((base / "model.safetensors.index.json").read_text())["weight_map"] @@ -191,6 +187,8 @@ def export_checkpoint(base, adapter, out, provenance, *, tokenizer=None, invento raise ValueError("untied head shape disagrees with text configuration") adapter_config = json.loads((adapter / "adapter_config.json").read_text()) pairs, scale = adapter_pairs(inventory["base"]["tensors"], inventory["adapter"]["tensors"], adapter_config) + if inventory.get("provenance") == PINS and (adapter_config["r"] != 16 or len(pairs) != 496 or scale != 2.): + raise ValueError("pinned JEMM requires rank16, 496 FP32 LoRA pairs and scale2") calibration = json.loads((adapter / "decision_config.json").read_text()) if any(calibration.get(key) != value for key, value in CALIBRATION.items()): raise ValueError("decision calibration differs from pinned JEMM") @@ -214,10 +212,9 @@ def export_checkpoint(base, adapter, out, provenance, *, tokenizer=None, invento raise ValueError("chat template did not preserve a unique user marker") chat_prefix, chat_suffix = chat.split(marker) producer = {"exporter_sha256": sha256(Path(__file__)), "runtime": producer_runtime()} - contract = {"pins": PINS, "inventory_sha256": canonical_hash(inventory), "source_sha256": source_hashes, + contract = {"format": "jemm-native/2", "lora_mode": "unmerged_fp32", "pins": PINS, "inventory_sha256": canonical_hash(inventory), "source_sha256": source_hashes, "label_token_ids": label_ids, "chat_prefix": chat_prefix, "chat_suffix": chat_suffix, "producer": producer} fingerprint = canonical_hash(contract) - manifest_path, progress_path = out / "jemm_export.json", out / "export_progress.json" if manifest_path.exists(): manifest = json.loads(manifest_path.read_text()) if manifest.get("exporter_sha256") != producer["exporter_sha256"] or manifest.get("producer_runtime") != producer["runtime"]: @@ -262,21 +259,25 @@ def export_checkpoint(base, adapter, out, provenance, *, tokenizer=None, invento selected_head = torch.cat([view[token:token + 1] for token in label_ids], dim=0).contiguous() elif key.startswith("model.visual."): visual[key] = reader.get_tensor(key) + if not torch.isfinite(visual[key]).all(): + raise ValueError(f"nonfinite vision tensor: {key}") elif key.startswith("model.language_model."): value = reader.get_tensor(key) - if key in pairs: - pair = pairs[key] - value = safe_merge(value, lora.get_tensor(pair["A"]), lora.get_tensor(pair["B"]), scale) - language[key[len("model.language_model."):]] = value + if not torch.isfinite(value).all(): + raise ValueError(f"nonfinite base tensor: {key}") + language[key] = inventory["base"]["tensors"][key] + del value elif not key.startswith("mtp."): raise ValueError(f"unexpected base tensor outside validated native scope: {key}") outputs, mapping, language_bytes = {}, {}, 0 if language: - filename = f"model-{number:05d}.safetensors" - atomic_tensors(out / filename, language) + filename = shard + atomic_original(base / shard, out / filename) outputs[filename] = sha256(out / filename) + if outputs[filename] != source_hashes["base"][shard]: + raise ValueError(f"original language shard checksum mismatch: {shard}") mapping = {key: filename for key in language} - language_bytes = sum(value.numel() * value.element_size() for value in language.values()) + language_bytes = sum(math.prod(spec["shape"]) * 2 for spec in language.values()) if visual: filename = "vision.safetensors" if any(filename in x["outputs"] for x in progress["completed"].values()): @@ -284,6 +285,8 @@ def export_checkpoint(base, adapter, out, provenance, *, tokenizer=None, invento atomic_tensors(out / filename, visual) outputs[filename] = sha256(out / filename) if selected_head is not None: + if not torch.isfinite(selected_head).all(): + raise ValueError("nonfinite selected head") atomic_tensors(out / "jemm_lm_head.safetensors", {"weight": selected_head}) outputs["jemm_lm_head.safetensors"] = sha256(out / "jemm_lm_head.safetensors") progress["completed"][shard] = {"source_sha256": source_hashes["base"][shard], "outputs": outputs, @@ -300,22 +303,26 @@ def export_checkpoint(base, adapter, out, provenance, *, tokenizer=None, invento temporary = out / (name + ".tmp") shutil.copyfile(base / name, temporary) os.replace(temporary, out / name) - for name in ("adapter_config.json", "decision_config.json"): - shutil.copyfile(adapter / name, out / name) + for name in ("adapter_config.json", "decision_config.json", "adapter_model.safetensors"): + atomic_original(adapter / name, out / name) exports = {p.name: sha256(p) for p in sorted(out.iterdir()) if p.is_file() and not p.name.endswith(".tmp") and p.name != "export_progress.json"} + if exports["adapter_model.safetensors"] != source_hashes["adapter"]["adapter_model.safetensors"]: + raise ValueError("original FP32 adapter checksum mismatch") if "jemm_lm_head.safetensors" not in exports or "vision.safetensors" not in exports: raise ValueError("export lacks selected head or vision tensors") - manifest = {"format": "jemm-native/1", "model_id": "JEMM", **PINS, **CALIBRATION, + manifest = {"format": "jemm-native/2", "model_id": "JEMM", **PINS, **CALIBRATION, "max_tokens": 8192, "max_mm_tokens": 3072, "chat_prefix": chat_prefix, "chat_suffix": chat_suffix, "enable_thinking": False, "preserve_thinking": False, "labels": list(LABELS), "label_token_ids": label_ids, "label_head_file": "jemm_lm_head.safetensors", "label_head_tensor": "weight", "label_head_dtype": "BF16", "label_head_shape": [32, text["hidden_size"]], "untied_lm_head": True, "vision_file": "vision.safetensors", - "lora_scale": scale, "lora_merge": "FP32 delta into BF16 base, finite checked", "source_sha256": source_hashes, + "lora_mode": "unmerged_fp32", "lora_file": "adapter_model.safetensors", + "lora_rank": adapter_config["r"], "lora_pairs": len(pairs), "lora_scale": scale, + "lora_sha256": source_hashes["adapter"]["adapter_model.safetensors"], "source_sha256": source_hashes, "upstream_source_sha256": inventory.get("upstream_source_sha256", {}), "export_sha256": exports, "inventory_sha256": canonical_hash(inventory), "exporter_sha256": producer["exporter_sha256"], "producer_runtime": producer["runtime"], "download_provenance": provenance, "export_fingerprint": fingerprint, - "omitted_tensors": [k for k in index if k.startswith("mtp.") or k == "lm_head.weight"]} + "unindexed_tensors": [k for k in index if k.startswith("mtp.") or k == "lm_head.weight"]} atomic_json(manifest_path, manifest) return manifest diff --git a/src/models/jemm/README.md b/src/models/jemm/README.md index ee18a612..3674a1eb 100644 --- a/src/models/jemm/README.md +++ b/src/models/jemm/README.md @@ -13,10 +13,14 @@ unsupported. Multi-image reference parity has a known limitation below. | MaestroYan/JEMM adapter | `76e3c209e8441fa658221c7ba2725bad2f811176` | | Qwen/Qwen3.8-27B base/tokenizer/processor | `1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0` | -The exporter verifies pinned file SHA-256s and tensor inventory, merges language -LoRA in FP32 into cloned BF16 weights and extracts 32 rows from the untied LM -head. `jemm_export.json` records every exported file's SHA-256. Startup verifies -complete file/shard coverage before CUDA initialization; keep files immutable +The `jemm-native/2` exporter verifies pinned file SHA-256s and tensor inventory, +retains the 17 original BF16 language shards byte for byte, and preserves the raw +FP32 adapter separately (496 rank-16 pairs, scale 2). Its language-only index +retains original `model.language_model.*` keys. Vision tensors and 32 selected +rows from the untied LM head are extracted without merging adapter weights. +`jemm_export.json` records every exported file's SHA-256 and the adapter contract. +Startup verifies original shard and adapter pins before CUDA initialization; +legacy premerged `jemm-native/1` exports are explicitly rejected. Keep files immutable while serving. This head requires the pinned temperatures 1.3480874159655591 (text), 1.3954832341582943 (images) and threshold 0.9872681877423998. Follow-up #122 loads validated calibration from the checksum-covered decision config; @@ -80,7 +84,10 @@ There is no cross-request batching or prefix cache. ## Validation and known multi-image limitation -The historical A800 BF16 corpus (13 requests / 16 questions, text and single-image +The unmerged FP32-adapter v2 path requires a fresh device/reference campaign; +CPU export and contract checks alone do not establish its numerical parity. + +The historical premerged v1 A800 BF16 corpus (13 requests / 16 questions, text and single-image inputs) passed exact tokens/positions/grids/pixels and predeclared probability 0.02 / Score 0.1 / winner-margin 0.05 gates. Max probability drift was 0.0015338802. These results are scoped to the source revisions and corpus in diff --git a/src/models/jemm/native/src/artifacts.rs b/src/models/jemm/native/src/artifacts.rs index b3abe60d..f687dcb0 100644 --- a/src/models/jemm/native/src/artifacts.rs +++ b/src/models/jemm/native/src/artifacts.rs @@ -13,7 +13,69 @@ fn relative(name: &str) -> bool { .components() .all(|p| matches!(p, Component::Normal(_))) } +const ADAPTER_FILE: &str = "adapter_model.safetensors"; +const PINNED_INVENTORY: &str = include_str!("../../../../../recipe/jemm/pinned_inventory.json"); + +/// Validate the unmerged adapter contract before opening a device or artifact file. +pub fn lora_contract(manifest: &Value) -> Result<(&'static str, f32)> { + ensure!( + manifest["format"] == "jemm-native/2", + "expected jemm-native/2; legacy premerged BF16 exports are unsupported" + ); + ensure!( + manifest["lora_mode"] == "unmerged_fp32" + && manifest["lora_file"] == ADAPTER_FILE + && manifest["lora_rank"].as_u64() == Some(16) + && manifest["lora_pairs"].as_u64() == Some(496) + && manifest["lora_scale"].as_f64() == Some(2.) + && manifest.get("lora_merge").is_none(), + "expected unmerged FP32 JEMM LoRA: rank16, 496 pairs, scale2" + ); + let pinned: Value = serde_json::from_str(PINNED_INVENTORY)?; + let files = &pinned["adapter"]["files"]; + ensure!( + manifest["lora_sha256"] == files[ADAPTER_FILE], + "LoRA checksum differs from pinned JEMM adapter" + ); + for name in [ADAPTER_FILE, "adapter_config.json"] { + ensure!( + manifest["source_sha256"]["adapter"][name] == files[name] + && manifest["export_sha256"][name] == files[name], + "LoRA source/export checksum differs from pinned artifact {name}" + ); + } + Ok((ADAPTER_FILE, 2.)) +} + pub fn verify_export(dir: &Path, manifest: &Value) -> Result<()> { + lora_contract(manifest)?; + let pinned: Value = serde_json::from_str(PINNED_INVENTORY)?; + let expected: serde_json::Map = pinned["base"]["tensors"] + .as_object() + .context("pinned tensor inventory")? + .iter() + .filter(|(key, _)| key.starts_with("model.language_model.")) + .map(|(key, spec)| (key.clone(), spec["file"].clone())) + .collect(); + let index: Value = + serde_json::from_slice(&std::fs::read(dir.join("model.safetensors.index.json"))?)?; + ensure!( + index["weight_map"].as_object() == Some(&expected), + "expected language-only index with original pinned tensor keys/shards" + ); + for filename in expected.values() { + let filename = filename.as_str().context("pinned language shard")?; + ensure!( + manifest["export_sha256"][filename] == pinned["base"]["files"][filename] + && manifest["source_sha256"]["base"][filename] == pinned["base"]["files"][filename], + "expected original unmerged BF16 language shard {filename}" + ); + } + verify_export_files(dir, manifest) +} + +/// Stream every checksum-covered file; model-specific pins are checked by verify_export. +pub fn verify_export_files(dir: &Path, manifest: &Value) -> Result<()> { let hashes = manifest["export_sha256"] .as_object() .context("export_sha256 must contain all native artifact hashes")?; @@ -45,6 +107,8 @@ pub fn verify_export(dir: &Path, manifest: &Value) -> Result<()> { "model.safetensors.index.json", "vision.safetensors", "jemm_lm_head.safetensors", + ADAPTER_FILE, + "adapter_config.json", ] { ensure!( hashes.contains_key(name), diff --git a/src/models/jemm/native/src/executor.rs b/src/models/jemm/native/src/executor.rs index e90c6c30..3fc488af 100644 --- a/src/models/jemm/native/src/executor.rs +++ b/src/models/jemm/native/src/executor.rs @@ -192,6 +192,8 @@ pub struct Executor { impl Executor { pub async fn load(dir: &Path, library: &Path, manifest: &Value) -> Result { crate::artifacts::verify_export(dir, manifest)?; + let (adapter_file, scale) = crate::artifacts::lora_contract(manifest)?; + let adapter_path = dir.join(adapter_file); let cfg = Config::load(dir)?; ensure!( ( @@ -215,7 +217,7 @@ impl Executor { )?; let (dir, library) = (dir.to_path_buf(), library.to_path_buf()); let state = tokio::task::spawn_blocking(move || -> Result { - let language = Model::load(&dir, &library)?; + let language = Model::load_with_lora(&dir, &library, &adapter_path, scale)?; let vision = VisionModel::load(&dir, &library)?; let head = LabelHead::load(&bytes, cfg.hidden)?; Ok(Loaded { diff --git a/src/models/jemm/native/src/processing.rs b/src/models/jemm/native/src/processing.rs index 7b122ce1..3538da9f 100644 --- a/src/models/jemm/native/src/processing.rs +++ b/src/models/jemm/native/src/processing.rs @@ -148,13 +148,13 @@ pub struct Processor { impl Processor { pub fn load(dir: &Path, manifest: &Value) -> Result { ensure!( - manifest["format"] == "jemm-native/1" + manifest["format"] == "jemm-native/2" && manifest["model_id"] == "JEMM" && manifest["base_model_id"] == "Qwen/Qwen3.8-27B" && manifest["base_revision"] == contract::BASE_REVISION && manifest["checkpoint_revision"] == contract::CHECKPOINT_REVISION && manifest["source_revision"] == contract::SOURCE_REVISION, - "expected pinned JEMM native export" + "expected pinned JEMM jemm-native/2 export; legacy premerged BF16 exports are unsupported" ); ensure!( manifest["max_tokens"] == 8192 && manifest["max_mm_tokens"] == 3072, diff --git a/tests/jemm/artifacts.rs b/tests/jemm/artifacts.rs index 58fac3a6..61d7470b 100644 --- a/tests/jemm/artifacts.rs +++ b/tests/jemm/artifacts.rs @@ -1,16 +1,16 @@ -use omni_jemm_native::artifacts::verify_export; +use omni_jemm_native::artifacts::{lora_contract, verify_export, verify_export_files}; use serde_json::json; #[test] fn export_hash_map_is_mandatory_and_rejects_traversal() { let dir = std::env::temp_dir(); assert!( - verify_export(&dir, &json!({})) + verify_export_files(&dir, &json!({})) .unwrap_err() .to_string() .contains("export_sha256") ); assert!( - verify_export(&dir, &json!({"export_sha256":{"../escape":"0".repeat(64)}})) + verify_export_files(&dir, &json!({"export_sha256":{"../escape":"0".repeat(64)}})) .unwrap_err() .to_string() .contains("relative") @@ -30,6 +30,8 @@ fn hashes_every_exported_file_and_requires_all_language_shards() { "model.safetensors.index.json", "vision.safetensors", "jemm_lm_head.safetensors", + "adapter_model.safetensors", + "adapter_config.json", "model-1.safetensors", "decision_config.json", ] { @@ -42,10 +44,10 @@ fn hashes_every_exported_file_and_requires_all_language_shards() { hashes.insert(name.into(), json!(format!("{:x}", Sha256::digest(bytes)))); } let manifest = json!({"export_sha256":hashes}); - verify_export(&dir, &manifest).unwrap(); + verify_export_files(&dir, &manifest).unwrap(); std::fs::write(dir.join("decision_config.json"), b"tampered").unwrap(); assert!( - verify_export(&dir, &manifest) + verify_export_files(&dir, &manifest) .unwrap_err() .to_string() .contains("mismatch: decision_config.json") @@ -56,10 +58,127 @@ fn hashes_every_exported_file_and_requires_all_language_shards() { .unwrap() .remove("model-1.safetensors"); assert!( - verify_export(&dir, &incomplete) + verify_export_files(&dir, &incomplete) .unwrap_err() .to_string() .contains("missing language shard") ); std::fs::remove_dir_all(dir).unwrap(); } + +#[test] +fn malformed_unmerged_contracts_fail_before_file_or_device_initialization() { + let file = "adapter_model.safetensors"; + let hash = "fac397c8af737bb8d259efbe16f771507f77276fb44ab74942cd2e1bc05c3fef"; + let config_hash = "7cfc1e46979f37a20012207b4ed72a720d24d74c9205952a5b62c1f81ee11427"; + let valid = json!({"format":"jemm-native/2","lora_mode":"unmerged_fp32","lora_file":file, + "lora_rank":16,"lora_pairs":496,"lora_scale":2.0,"lora_sha256":hash, + "source_sha256":{"adapter":{file:hash,"adapter_config.json":config_hash}}, + "export_sha256":{file:hash,"adapter_config.json":config_hash}}); + assert_eq!(lora_contract(&valid).unwrap(), (file, 2.)); + for (key, value) in [ + ("lora_merge", json!("FP32 delta into BF16 base")), + ("format", json!("jemm-native/1")), + ("lora_mode", json!("merged_bf16")), + ("lora_file", json!("../adapter_model.safetensors")), + ("lora_rank", json!(8)), + ("lora_rank", json!("16")), + ("lora_pairs", json!(495)), + ("lora_scale", json!(1.0)), + ("lora_sha256", json!("0".repeat(64))), + ] { + let mut manifest = valid.clone(); + manifest[key] = value; + let error = verify_export(std::path::Path::new("/absent-jemm-export"), &manifest) + .unwrap_err() + .to_string(); + assert!( + error.contains("jemm-native/2") || error.contains("LoRA"), + "{key}: {error}" + ); + } +} + +#[tokio::test] +async fn executor_rejects_legacy_export_before_cuda_library_or_file_access() { + let error = omni_jemm_native::executor::Executor::load( + std::path::Path::new("/absent-jemm-export"), + std::path::Path::new("/absent-cuda-library"), + &json!({"format":"jemm-native/1"}), + ) + .await + .err() + .unwrap(); + assert!(error.to_string().contains("jemm-native/2"), "{error}"); +} + +#[test] +fn original_language_index_and_adapter_provenance_are_mandatory() { + let inventory: serde_json::Value = + serde_json::from_str(include_str!("../../recipe/jemm/pinned_inventory.json")).unwrap(); + let adapter_hashes = inventory["adapter"]["files"].clone(); + let mut manifest = json!({"format":"jemm-native/2","lora_mode":"unmerged_fp32", + "lora_file":"adapter_model.safetensors","lora_rank":16,"lora_pairs":496,"lora_scale":2., + "lora_sha256":adapter_hashes["adapter_model.safetensors"], + "source_sha256":{"adapter":adapter_hashes},"export_sha256":inventory["adapter"]["files"]}); + for scope in ["source_sha256", "export_sha256"] { + for name in ["adapter_model.safetensors", "adapter_config.json"] { + let mut bad = manifest.clone(); + if scope == "source_sha256" { + bad[scope]["adapter"][name] = json!("0".repeat(64)); + } else { + bad[scope][name] = json!("0".repeat(64)); + } + assert!( + lora_contract(&bad) + .unwrap_err() + .to_string() + .contains("LoRA") + ); + } + } + let dir = std::env::temp_dir().join(format!("jemm-original-index-{}", std::process::id())); + std::fs::create_dir_all(&dir).unwrap(); + std::fs::write( + dir.join("model.safetensors.index.json"), + br#"{"weight_map":{"embed_tokens.weight":"merged.safetensors"}}"#, + ) + .unwrap(); + assert!( + verify_export(&dir, &manifest) + .unwrap_err() + .to_string() + .contains("original pinned tensor keys/shards") + ); + let map: serde_json::Map = inventory["base"]["tensors"] + .as_object() + .unwrap() + .iter() + .filter(|(key, _)| key.starts_with("model.language_model.")) + .map(|(key, spec)| (key.clone(), spec["file"].clone())) + .collect(); + std::fs::write( + dir.join("model.safetensors.index.json"), + serde_json::to_vec(&json!({"weight_map":map})).unwrap(), + ) + .unwrap(); + assert!( + verify_export(&dir, &manifest) + .unwrap_err() + .to_string() + .contains("original unmerged BF16 language shard") + ); + // A valid index alone cannot bless newly merged/re-serialized shards. + manifest["source_sha256"]["base"] = inventory["base"]["files"].clone(); + for filename in map.values() { + let name = filename.as_str().unwrap(); + manifest["export_sha256"][name] = inventory["base"]["files"][name].clone(); + } + assert!( + verify_export(&dir, &manifest) + .unwrap_err() + .to_string() + .contains("required artifact config.json") + ); + std::fs::remove_dir_all(dir).unwrap(); +} diff --git a/tests/jemm/test_export.py b/tests/jemm/test_export.py new file mode 100644 index 00000000..75a652dd --- /dev/null +++ b/tests/jemm/test_export.py @@ -0,0 +1,158 @@ +"""Tiny real safetensors exports: preserve BF16 base and raw FP32 adapter bytes.""" +import errno +import importlib.util +import json +import tempfile +import unittest +from pathlib import Path +from unittest.mock import patch + +import torch +from safetensors import safe_open +from safetensors.torch import save_file + +ROOT = Path(__file__).resolve().parents[2] +SPEC = importlib.util.spec_from_file_location("jemm_export", ROOT / "recipe/jemm/export.py") +EXPORT = importlib.util.module_from_spec(SPEC) +SPEC.loader.exec_module(EXPORT) + + +class Tokenizer: + def encode(self, label, **_kwargs): + return [EXPORT.LABELS.index(label)] + + def apply_chat_template(self, messages, **_kwargs): + return "system:" + messages[0]["content"] + "\n" + messages[1]["content"] + "\n\n\n" + + +def inventory(directory, names): + files, tensors = {}, {} + for name in names: + files[name] = EXPORT.sha256(directory / name) + if name.endswith(".safetensors"): + with safe_open(str(directory / name), framework="pt", device="cpu") as reader: + for key in reader.keys(): + view = reader.get_slice(key) + tensors[key] = {"dtype": view.get_dtype(), "shape": view.get_shape(), "file": name} + return {"files": files, "tensors": tensors} + + +class ExportTest(unittest.TestCase): + def setUp(self): + self.temporary = tempfile.TemporaryDirectory() + self.root = Path(self.temporary.name) + self.base, self.adapter, self.out = (self.root / name for name in ("base", "adapter", "out")) + self.base.mkdir(); self.adapter.mkdir() + self.target = "model.language_model.layers.0.mlp.down_proj.weight" + self.language = {self.target: torch.tensor([[1., 2.], [3., 4.]], dtype=torch.bfloat16), + "model.language_model.embed_tokens.weight": torch.ones(32, 2, dtype=torch.bfloat16)} + save_file(self.language, str(self.base / "model-00001-of-00002.safetensors")) + other = {"model.visual.patch_embed.weight": torch.ones(2, 2, dtype=torch.bfloat16), + "lm_head.weight": torch.arange(64).reshape(32, 2).to(torch.bfloat16), + "mtp.fc.weight": torch.zeros(2, 2, dtype=torch.bfloat16)} + save_file(other, str(self.base / "model-00002-of-00002.safetensors")) + mapping = {key: "model-00001-of-00002.safetensors" for key in self.language} + mapping.update({key: "model-00002-of-00002.safetensors" for key in other}) + (self.base / "model.safetensors.index.json").write_text(json.dumps({"weight_map": mapping})) + (self.base / "config.json").write_text(json.dumps({"text_config": {"vocab_size": 32, "hidden_size": 2}})) + (self.base / "tokenizer.json").write_text("{}") + (self.base / "preprocessor_config.json").write_text("{}") + self.lora = {"base_model.model." + self.target.removesuffix(".weight") + ".lora_A.weight": torch.full((16, 2), 0.12345678), + "base_model.model." + self.target.removesuffix(".weight") + ".lora_B.weight": torch.full((2, 16), 0.23456789)} + self.write_adapter() + self.config = {"peft_type": "LORA", "base_model_name_or_path": EXPORT.BASE_MODEL_ID, "r": 16, "lora_alpha": 32} + (self.adapter / "adapter_config.json").write_text(json.dumps(self.config)) + (self.adapter / "decision_config.json").write_text(json.dumps(EXPORT.CALIBRATION)) + self.refresh_inventory() + + def tearDown(self): + self.temporary.cleanup() + + def write_adapter(self): + save_file(self.lora, str(self.adapter / "adapter_model.safetensors")) + + def refresh_inventory(self): + self.inventory = {"base": inventory(self.base, [p.name for p in self.base.iterdir()]), + "adapter": inventory(self.adapter, [p.name for p in self.adapter.iterdir()])} + + def run_export(self): + return EXPORT.export_checkpoint(self.base, self.adapter, self.out, EXPORT.PINS, + tokenizer=Tokenizer(), inventory=self.inventory) + + def test_preserves_original_base_and_fp32_adapter_without_merging(self): + manifest = self.run_export() + self.assertEqual(manifest["format"], "jemm-native/2") + self.assertEqual(manifest["lora_mode"], "unmerged_fp32") + self.assertEqual(manifest["lora_file"], "adapter_model.safetensors") + self.assertEqual(manifest["lora_rank"], 16) + self.assertEqual(manifest["lora_pairs"], 1) # injected tiny inventory, not production + self.assertEqual(manifest["lora_scale"], 2.) + self.assertEqual(manifest["lora_sha256"], EXPORT.sha256(self.adapter / "adapter_model.safetensors")) + mapping = json.loads((self.out / "model.safetensors.index.json").read_text())["weight_map"] + self.assertEqual(mapping, {key: "model-00001-of-00002.safetensors" for key in self.language}) + for name in set(mapping.values()): + self.assertEqual((self.out / name).read_bytes(), (self.base / name).read_bytes()) + self.assertEqual((self.out / "adapter_model.safetensors").read_bytes(), (self.adapter / "adapter_model.safetensors").read_bytes()) + with safe_open(str(self.out / mapping[self.target]), framework="pt", device="cpu") as reader: + self.assertTrue(torch.equal(reader.get_tensor(self.target), self.language[self.target])) + with safe_open(str(self.out / "adapter_model.safetensors"), framework="pt", device="cpu") as reader: + for key in self.lora: + self.assertEqual(reader.get_tensor(key).dtype, torch.float32) + with safe_open(str(self.out / "jemm_lm_head.safetensors"), framework="pt", device="cpu") as reader: + self.assertTrue(torch.equal(reader.get_tensor("weight"), torch.arange(64).reshape(32, 2).to(torch.bfloat16))) + with safe_open(str(self.out / "vision.safetensors"), framework="pt", device="cpu") as reader: + self.assertTrue(torch.equal(reader.get_tensor("model.visual.patch_embed.weight"), torch.ones(2, 2, dtype=torch.bfloat16))) + + def test_cross_device_copy_fallback_preserves_identical_source_bytes(self): + with patch.object(EXPORT.os, "link", side_effect=OSError(errno.EXDEV, "cross-device")): + manifest = self.run_export() + for name in ["model-00001-of-00002.safetensors", "adapter_model.safetensors"]: + source = self.adapter if name.startswith("adapter") else self.base + self.assertEqual((self.out / name).read_bytes(), (source / name).read_bytes()) + self.assertEqual(EXPORT.sha256(source / name), manifest["export_sha256"][name]) + + def test_partial_export_resumes_and_writes_completion_marker_last(self): + original = EXPORT.atomic_json + def interrupted(path, value): + original(path, value) + if path.name == "export_progress.json" and len(value["completed"]) == 1: + raise RuntimeError("simulated interruption") + with patch.object(EXPORT, "atomic_json", side_effect=interrupted): + with self.assertRaisesRegex(RuntimeError, "simulated interruption"): + self.run_export() + self.assertFalse((self.out / "jemm_export.json").exists()) + original_inode = (self.out / "model-00001-of-00002.safetensors").stat().st_ino + writes = [] + def recorded(path, value): + writes.append(path.name); original(path, value) + with patch.object(EXPORT, "atomic_json", side_effect=recorded): + manifest = self.run_export() + self.assertEqual(writes[-1], "jemm_export.json") + self.assertEqual(original_inode, (self.out / "model-00001-of-00002.safetensors").stat().st_ino) + self.assertEqual(self.run_export(), manifest) + + def test_nonfinite_adapter_is_refused_before_output_creation(self): + next(iter(self.lora.values()))[0, 0] = float("nan") + self.write_adapter(); self.refresh_inventory() + with self.assertRaisesRegex(ValueError, "nonfinite adapter"): + self.run_export() + self.assertFalse(self.out.exists()) + + def test_legacy_completed_export_requires_new_directory(self): + self.run_export() + path = self.out / "jemm_export.json" + manifest = json.loads(path.read_text()); manifest["format"] = "jemm-native/1" + path.write_text(json.dumps(manifest)) + with self.assertRaisesRegex(ValueError, "legacy|jemm-native/2"): + self.run_export() + + def test_pinned_inventory_is_rank16_and_has_exactly496_adapter_pairs(self): + pinned = json.loads(EXPORT.INVENTORY_PATH.read_text()) + pairs, scale = EXPORT.adapter_pairs(pinned["base"]["tensors"], pinned["adapter"]["tensors"], self.config) + self.assertEqual(len(pairs), 496); self.assertEqual(scale, 2.) + self.assertEqual(len({v["file"] for key, v in pinned["base"]["tensors"].items() + if key.startswith("model.language_model.")}), 17) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/jemm/tokenization.rs b/tests/jemm/tokenization.rs index 736d2995..f9bc30f6 100644 --- a/tests/jemm/tokenization.rs +++ b/tests/jemm/tokenization.rs @@ -42,7 +42,7 @@ fn fixture() -> (Fixture, Value) { br#"{"size":{"shortest_edge":65536,"longest_edge":16777216}}"#, ) .unwrap(); - let manifest = json!({"format":"jemm-native/1","model_id":"JEMM","base_model_id":"Qwen/Qwen3.8-27B","base_revision":contract::BASE_REVISION,"checkpoint_revision":contract::CHECKPOINT_REVISION,"source_revision":contract::SOURCE_REVISION,"max_tokens":8192,"max_mm_tokens":3072,"label_token_ids":(1..=32).collect::>(),"chat_prefix":format!("<|im_start|>system\n{}<|im_end|>\n<|im_start|>user\n",contract::SYSTEM),"chat_suffix":"<|im_end|>\n<|im_start|>assistant\n\n\n\n\n","temperature":1.3480874159655591,"mm_temperature":1.3954832341582943,"threshold":0.9872681877423998}); + let manifest = json!({"format":"jemm-native/2","model_id":"JEMM","base_model_id":"Qwen/Qwen3.8-27B","base_revision":contract::BASE_REVISION,"checkpoint_revision":contract::CHECKPOINT_REVISION,"source_revision":contract::SOURCE_REVISION,"max_tokens":8192,"max_mm_tokens":3072,"label_token_ids":(1..=32).collect::>(),"chat_prefix":format!("<|im_start|>system\n{}<|im_end|>\n<|im_start|>user\n",contract::SYSTEM),"chat_suffix":"<|im_end|>\n<|im_start|>assistant\n\n\n\n\n","temperature":1.3480874159655591,"mm_temperature":1.3954832341582943,"threshold":0.9872681877423998}); (Fixture(dir), manifest) } #[test] @@ -97,3 +97,13 @@ fn rejects_custom_calibration_for_pinned_adapter() { .contains("pinned JEMM calibration") ); } + +#[test] +fn accepts_v2_and_rejects_legacy_premerged_format_before_tokenizer_io() { + let (f, mut manifest) = fixture(); + manifest["format"] = json!("jemm-native/2"); + assert!(Processor::load(&f.0, &manifest).is_ok()); + manifest["format"] = json!("jemm-native/1"); + let error = Processor::load(&f.0, &manifest).err().unwrap().to_string(); + assert!(error.contains("jemm-native/2"), "{error}"); +} From a3c7a78187cbeead3ef4d699a2d208340f382154 Mon Sep 17 00:00:00 2001 From: levius <2114377220@qq.com> Date: Sat, 10 Oct 2026 23:01:03 +0800 Subject: [PATCH 09/21] Match original base projection shapes for unmerged LoRA models --- src/models/qwen3_5/native/src/lora.rs | 124 ++++++++++++++++++- src/models/qwen3_5/native/src/model.rs | 83 ++++++++++++- tests/qwen3_5/lora.rs | 158 +++++++++++++++++++++++++ tests/qwen3_5/lora_fake.c | 15 ++- 4 files changed, 370 insertions(+), 10 deletions(-) diff --git a/src/models/qwen3_5/native/src/lora.rs b/src/models/qwen3_5/native/src/lora.rs index 44ff43c1..cb838a49 100644 --- a/src/models/qwen3_5/native/src/lora.rs +++ b/src/models/qwen3_5/native/src/lora.rs @@ -44,6 +44,7 @@ type Components = [Option<(Range, usize)>; 2]; struct Parsed { pairs: Vec, + base_width: usize, } fn targets(cfg: &Config) -> Result> { @@ -263,7 +264,10 @@ fn parse(cfg: &Config, bytes: &[u8], scale: f32) -> Result { }) }) .collect::>()?; - Ok(Parsed { pairs }) + Ok(Parsed { + pairs, + base_width: allowed.values().map(|t| t.rows).max().unwrap(), + }) } pub(crate) struct Checkpoint { @@ -296,12 +300,20 @@ impl Checkpoint { Ok(Self { map, parsed, scale }) } pub(crate) fn upload(self, layers: usize, stream: Stream) -> Result { + let mut layout = WorkLayout::for_pairs(1, &self.parsed.pairs)?.unwrap(); + // Reuse the BF16 gather for contiguous base projection outputs. All configured + // components must fit even when only a small subset has an adapter. + layout.gather = self + .parsed + .base_width + .checked_mul(2) + .context("base projection work overflow")?; let mut result = Lora { layers: (0..layers) .map(|_| std::array::from_fn(|_| Vec::new())) .collect(), scale: self.scale, - layout: WorkLayout::for_pairs(1, &self.parsed.pairs)?.unwrap(), + layout, }; for pair in self.parsed.pairs { let a = DeviceBuffer::new(pair.a.len())?; @@ -376,6 +388,7 @@ pub(crate) struct Work { delta: DeviceBuffer, gather: DeviceBuffer, rows: usize, + gather_width: usize, } impl Lora { pub(crate) fn work(&self, rows: usize) -> Result { @@ -386,6 +399,7 @@ impl Lora { delta: DeviceBuffer::new(layout.delta)?, gather: DeviceBuffer::new(layout.gather)?, rows, + gather_width: self.layout.gather / 2, }) } #[allow(clippy::too_many_arguments)] // Validated projection layout plus exclusively owned device work. @@ -497,6 +511,112 @@ impl Lora { } } +pub(crate) fn component_rows(cfg: &Config, group: Group) -> Result<[usize; 4]> { + let mul = |a: usize, b: usize| { + a.checked_mul(b) + .context("base component dimension overflow") + }; + Ok(match group { + Group::LinearInput => { + let kd = mul(cfg.lin_k_heads, cfg.lin_k_dim)?; + let vd = mul(cfg.lin_v_heads, cfg.lin_v_dim)?; + [ + mul(2, kd)? + .checked_add(vd) + .context("base component dimension overflow")?, + vd, + cfg.lin_v_heads, + cfg.lin_v_heads, + ] + } + Group::AttentionInput => [ + mul(mul(cfg.heads, cfg.head_dim)?, 2)?, + mul(cfg.kv_heads, cfg.head_dim)?, + mul(cfg.kv_heads, cfg.head_dim)?, + 0, + ], + Group::GateUp => [cfg.intermediate, cfg.intermediate, 0, 0], + _ => anyhow::bail!("base component group is not fused"), + }) +} +/// Original component GEMMs write contiguous BF16 rows before copying into fused scratch. +/// +/// # Safety +/// Input is packed row-major [sum(lengths),cols]; weights are contiguous row-stacked +/// component matrices; output holds sum(lengths) rows of sum(components). Work and +/// GEMM belong to the serialized Model on this stream. +#[allow(clippy::too_many_arguments)] // Exact device boundary and validated component layout. +pub(crate) unsafe fn project_components( + input: *const c_void, + weights: *const c_void, + output: *mut c_void, + lengths: &[usize], + cols: usize, + components: &[usize], + work: &Work, + gemm: *mut c_void, + stream: Stream, +) -> Result<()> { + let stride = components + .iter() + .try_fold(0usize, |sum, &n| sum.checked_add(n)) + .context("base component stride overflow")?; + ensure!( + stride > 0 && stride <= i32::MAX as usize && cols > 0 && cols <= i32::MAX as usize, + "base component dimensions" + ); + ensure!( + components.iter().all(|&n| n <= work.gather_width), + "base component exceeds work width" + ); + let mut row = 0; + for &rows in lengths { + ensure!( + rows > 0 && rows <= work.rows, + "base component work row bound" + ); + let mut first = 0; + for &n in components { + if n == 0 { + continue; + } + // Match the original projection's M/N/K and contiguous input/weight/output + // strides. A strided GEMM output itself can choose a different cuBLAS plan. + unsafe { + check( + (cuda::api().cs1_gemm)( + gemm, + input.wrapping_byte_add(row * cols * 2), + weights.wrapping_byte_add(first * cols * 2), + work.gather.at(0), + rows as i32, + n as i32, + cols as i32, + n as i32, + stream, + ), + "base component GEMM", + )?; + cuda::copy2d( + output.wrapping_byte_add((row * stride + first) * 2), + stride * 2, + work.gather.at(0), + n * 2, + n * 2, + rows, + stream, + )?; + } + first += n; + } + row = row + .checked_add(rows) + .context("base component row overflow")?; + ensure!(row <= i32::MAX as usize, "base component packed row bound"); + } + Ok(()) +} + #[cfg(test)] #[path = "../../../../../tests/qwen3_5/lora.rs"] mod tests; diff --git a/src/models/qwen3_5/native/src/model.rs b/src/models/qwen3_5/native/src/model.rs index 6a353fbb..d41b19b2 100644 --- a/src/models/qwen3_5/native/src/model.rs +++ b/src/models/qwen3_5/native/src/model.rs @@ -769,6 +769,43 @@ impl Model { ) } + #[allow(clippy::too_many_arguments)] // Base projection coordinates and sequence shape. + fn gemm_group( + &self, + s: &Scratch, + x: usize, + w: &Tensor, + y: usize, + m: usize, + lengths: &[usize], + group: Group, + ) -> Result<()> { + if self.lora.is_none() { + return self.gemm(s, x, w, y, m); + } + let components = lora::component_rows(&self.cfg, group)?; + ensure!( + components.iter().sum::() == w.shape[0], + "base component weight shape" + ); + let work = s.lora.as_ref().context("missing base component work")?; + // SAFETY: stacked() validated contiguous component weights; scratch holds the + // complete packed input/output rows, and work belongs to this serialized Model. + unsafe { + lora::project_components( + s.at(x), + w.ptr, + s.at(y), + lengths, + w.shape[1], + &components, + work, + self.gemm, + self.stream, + ) + } + } + #[allow(clippy::too_many_arguments)] // Layer projection coordinates in shared scratch. fn apply_lora( &self, @@ -1197,7 +1234,15 @@ impl Model { for (i, layer) in self.layers.iter().enumerate() { match &layer.mixer { Mixer::Linear(la) => { - self.gemm(s, s.x, &la.in_proj, s.gdn_in, t)?; + self.gemm_group( + s, + s.x, + &la.in_proj, + s.gdn_in, + t, + lengths, + Group::LinearInput, + )?; self.apply_lora(i, Group::LinearInput, s, s.x, s.gdn_in, lengths)?; let ld = w.gdn_in as i32; let z = s.gdn_in + w.conv * BF16; @@ -1275,7 +1320,15 @@ impl Model { self.apply_lora(i, Group::LinearOutput, s, s.ln, s.delta, lengths)?; } Mixer::Full(fa) => { - self.gemm(s, s.x, &fa.qkv, s.attn_in, t)?; + self.gemm_group( + s, + s.x, + &fa.qkv, + s.attn_in, + t, + lengths, + Group::AttentionInput, + )?; self.apply_lora(i, Group::AttentionInput, s, s.x, s.attn_in, lengths)?; let ld = w.attn_in as i32; let k = s.attn_in + w.attn_q * BF16; @@ -1342,7 +1395,7 @@ impl Model { "post-attention norm", )?; } - self.gemm(s, s.x, &layer.gate_up, s.gate_up, t)?; + self.gemm_group(s, s.x, &layer.gate_up, s.gate_up, t, lengths, Group::GateUp)?; self.apply_lora(i, Group::GateUp, s, s.x, s.gate_up, lengths)?; unsafe { check( @@ -1451,7 +1504,15 @@ impl Model { for (i, layer) in self.layers.iter().enumerate() { match &layer.mixer { Mixer::Linear(la) => { - self.gemm(s, s.x + qb * hb, &la.in_proj, s.gdn_in + qb * ldbb, rows)?; + self.gemm_group( + s, + s.x + qb * hb, + &la.in_proj, + s.gdn_in + qb * ldbb, + rows, + &[rows], + Group::LinearInput, + )?; self.apply_lora( i, Group::LinearInput, @@ -1571,7 +1632,15 @@ impl Model { la_i += 1; } Mixer::Full(fa) => { - self.gemm(s, s.x + qb * hb, &fa.qkv, s.attn_in + qb * ab, rows)?; + self.gemm_group( + s, + s.x + qb * hb, + &fa.qkv, + s.attn_in + qb * ab, + rows, + &[rows], + Group::AttentionInput, + )?; self.apply_lora( i, Group::AttentionInput, @@ -1664,12 +1733,14 @@ impl Model { "post-attention norm", )?; } - self.gemm( + self.gemm_group( s, s.x + qb * hb, &layer.gate_up, s.gate_up + qb * 2 * cfg.intermediate * BF16, rows, + &[rows], + Group::GateUp, )?; self.apply_lora( i, diff --git a/tests/qwen3_5/lora.rs b/tests/qwen3_5/lora.rs index 9cffc6bf..f32b78c7 100644 --- a/tests/qwen3_5/lora.rs +++ b/tests/qwen3_5/lora.rs @@ -101,6 +101,10 @@ fn partial_pairs_keep_fp32_bytes_and_match_native_component_layouts() { ] { let bytes = pair(target, rows, cols, 16); let parsed = parse(&cfg, &bytes, 2.0).unwrap(); + assert_eq!( + parsed.base_width, 17408, + "partial adapters must retain all base projection widths" + ); assert_eq!(parsed.pairs.len(), 1); let p = &parsed.pairs[0]; assert_eq!((p.rank, p.target.rows, p.target.cols), (16, rows, cols)); @@ -189,6 +193,44 @@ fn optional_work_is_absent_without_adapters_and_checked_for_overflow() { assert!(WorkLayout::for_pairs(usize::MAX, &parsed.pairs).is_err()); } +#[test] +fn base_components_match_original_27b_and_4b_projection_shapes() { + for (hidden, intermediate, heads, values, linear, attention) in [ + ( + 5120, + 17408, + 24, + 48, + [10240, 6144, 48, 48], + [12288, 1024, 1024, 0], + ), + ( + 2560, + 9216, + 16, + 32, + [8192, 4096, 32, 32], + [8192, 1024, 1024, 0], + ), + ] { + let mut cfg = config(); + cfg.hidden = hidden; + cfg.intermediate = intermediate; + cfg.heads = heads; + cfg.lin_v_heads = values; + assert_eq!(component_rows(&cfg, Group::LinearInput).unwrap(), linear); + assert_eq!( + component_rows(&cfg, Group::AttentionInput).unwrap(), + attention + ); + assert_eq!( + component_rows(&cfg, Group::GateUp).unwrap(), + [intermediate, intermediate, 0, 0] + ); + assert!(component_rows(&cfg, Group::Down).is_err()); + } +} + struct Temporary(std::path::PathBuf); impl Temporary { fn new() -> Self { @@ -251,6 +293,7 @@ fn cpu_cuda_boundary_checks_fp32_sums_fused_offsets_sequences_and_retirement() { "cs1_copy2d", "cs1_vision_to_float", "cs1_gemm_f32", + "cs1_gemm", "cs1_vision_lora_add", ]; // Required but unused functions return an error, so an unexpected dispatch fails. @@ -427,4 +470,119 @@ fn cpu_cuda_boundary_checks_fp32_sums_fused_offsets_sequences_and_retirement() { "adapter/work/input/output allocations must retire" ); } + let base_calls = unsafe { + *metrics + .get:: i32>(b"lora_fake_base_calls\0") + .unwrap() + }; + let base_arg = unsafe { + *metrics + .get:: i32>(b"lora_fake_base_arg\0") + .unwrap() + }; + for (group, components) in [ + (Group::LinearInput, [384, 128, 1, 1]), + (Group::AttentionInput, [512, 256, 256, 0]), + (Group::GateUp, [8, 8, 0, 0]), + ] { + assert_eq!(component_rows(&cfg, group).unwrap(), components); + let cols = cfg.hidden; + let stride: usize = components.iter().sum(); + let input: Vec<_> = (1..=5) + .flat_map(|value| { + half::bf16::from_f32(value as f32) + .to_le_bytes() + .repeat(cols) + }) + .collect(); + let weights: Vec<_> = (1..=stride) + .flat_map(|value| { + half::bf16::from_f32(value as f32) + .to_le_bytes() + .repeat(cols) + }) + .collect(); + let original = half::bf16::ONE.to_le_bytes().repeat(5 * stride); + unsafe { + reset(); + } + { + let path = temporary.0.join("base-partial.safetensors"); + std::fs::write(&path, pair("layers.0.linear_attn.in_proj_b", 1, 8, 1)).unwrap(); + let adapter = Checkpoint::load(&cfg, &path, 2.) + .unwrap() + .upload(2, stream) + .unwrap(); + let work = adapter.work(2).unwrap(); + let x = DeviceBuffer::new(input.len()).unwrap(); + let w = DeviceBuffer::new(weights.len()).unwrap(); + let y = DeviceBuffer::new(original.len()).unwrap(); + unsafe { + cuda::upload(x.at(0), &input, stream).unwrap(); + cuda::upload(w.at(0), &weights, stream).unwrap(); + cuda::upload(y.at(0), &original, stream).unwrap(); + project_components( + x.at(cols * 2), + w.at(0), + y.at(stride * 2), + &[1, 2], + cols, + &components, + &work, + std::ptr::null_mut(), + stream, + ) + .unwrap(); + } + let mut actual = vec![0; original.len()]; + unsafe { + cuda::download(&mut actual, y.at(0), stream).unwrap(); + } + for row in 0..5 { + for col in 0..stride { + let offset = (row * stride + col) * 2; + let value = + half::bf16::from_le_bytes(actual[offset..offset + 2].try_into().unwrap()); + let expected = if (1..4).contains(&row) { + half::bf16::from_f32( + (row + 1) as f32 + * half::bf16::from_f32((col + 1) as f32).to_f32() + * cols as f32, + ) + } else { + half::bf16::ONE + }; + assert_eq!(value, expected, "{group:?}, row{row}, output{col}"); + } + } + let parts: Vec<_> = components.iter().copied().filter(|&n| n > 0).collect(); + assert_eq!(unsafe { base_calls() }, parts.len() as i32 * 2); + let mut call = 0; + for m in [1, 2] { + let mut row = 0; + for &n in &parts { + assert_eq!( + (0..4) + .map(|arg| unsafe { base_arg(call, arg) }) + .collect::>(), + [m, n as i32, cols as i32, n as i32] + ); + assert_eq!( + unsafe { base_arg(call, 4) }, + half::bf16::from_f32((row + 1) as f32).to_bits() as i32, + "component weight offset" + ); + row += n; + call += 1; + } + } + assert_eq!(unsafe { copies() }, parts.len() as i32 * 2); + assert_eq!( + unsafe { gemms() }, + 0, + "base split must not call FP32 adapter GEMM" + ); + } + assert_eq!(unsafe { live() }, 0); + } } diff --git a/tests/qwen3_5/lora_fake.c b/tests/qwen3_5/lora_fake.c index 1efeb5a2..802d2441 100644 --- a/tests/qwen3_5/lora_fake.c +++ b/tests/qwen3_5/lora_fake.c @@ -2,13 +2,13 @@ #include #include #include -static int live, gemms, copies, ms[128], heights[128]; +static int live, gemms, copies, ms[128], heights[128], base_calls, base_args[16][5]; int lora_fake_live(void) { return live; } int lora_fake_gemms(void) { return gemms; } int lora_fake_m(int i) { return ms[i]; } int lora_fake_copies(void) { return copies; } int lora_fake_height(int i) { return heights[i]; } -void lora_fake_reset(void) { gemms=0; copies=0; } +void lora_fake_reset(void) { gemms=0; copies=0; base_calls=0; } uint32_t cs1_abi_version(void) { return LORA_FAKE_ABI; } const char* cs1_error_string(int code) { (void)code; return "unexpected fake CUDA call"; } int cs1_malloc(void** p, size_t n) { *p=malloc(n); if (!*p) return 1; ++live; return 0; } @@ -38,3 +38,14 @@ int cs1_gemm_f32(void* gemm,const float* x,const float* w,float* y,int m,int n,i int cs1_vision_lora_add(uint16_t* x,const float* delta,size_t n,float scale,void* stream) { (void)stream; for(size_t i=0;i Date: Sat, 10 Oct 2026 23:05:26 +0800 Subject: [PATCH 10/21] Preserve all-image projection shapes in native 27B vision --- .../qwen3_5/native/src/vision/geometry.rs | 49 ++- src/models/qwen3_5/native/src/vision/model.rs | 198 ++++++++-- tests/qwen3_5/vision_batch.rs | 347 ++++++++++++++++++ tests/qwen3_5/vision_batch_fake.c | 54 +++ 4 files changed, 616 insertions(+), 32 deletions(-) create mode 100644 tests/qwen3_5/vision_batch.rs create mode 100644 tests/qwen3_5/vision_batch_fake.c diff --git a/src/models/qwen3_5/native/src/vision/geometry.rs b/src/models/qwen3_5/native/src/vision/geometry.rs index 179b0bfa..cc2e672c 100644 --- a/src/models/qwen3_5/native/src/vision/geometry.rs +++ b/src/models/qwen3_5/native/src/vision/geometry.rs @@ -8,7 +8,7 @@ pub struct VisionGeometry { pub sin: Vec, } impl VisionGeometry { - pub fn new([t, h, w]: [usize; 3], config: &VisionConfig) -> Result { + fn patch_count([t, h, w]: [usize; 3], config: &VisionConfig) -> Result { config.validate()?; ensure!( t == 1 && h > 0 && w > 0 && h % 2 == 0 && w % 2 == 0, @@ -17,7 +17,6 @@ impl VisionGeometry { let n = h .checked_mul(w) .ok_or_else(|| anyhow::anyhow!("vision grid overflow"))?; - // Processor bounds allow rounding up a 1,048,576-pixel source and narrow upscaled images. ensure!( if config.hidden_size == 1024 { n <= 4608 && h <= 512 && w <= 512 @@ -26,6 +25,10 @@ impl VisionGeometry { }, "vision grid exceeds processor bounds" ); + Ok(n) + } + pub fn new([t, h, w]: [usize; 3], config: &VisionConfig) -> Result { + let n = Self::patch_count([t, h, w], config)?; let mut g = Self { indices: Vec::with_capacity(n * 4), weights: Vec::with_capacity(n * 4), @@ -65,3 +68,45 @@ impl VisionGeometry { Ok(g) } } + +/// Ordered image-local geometry concatenated for all-image projection row counts. +pub(crate) struct BatchGeometry { + pub geometry: VisionGeometry, +} +impl BatchGeometry { + pub fn lengths(grids: &[[usize; 3]], config: &VisionConfig) -> Result> { + ensure!( + config.hidden_size == 1152 && (2..=4).contains(&grids.len()), + "batched vision requires 2..=4 unadapted 27B images" + ); + let lengths = grids + .iter() + .map(|&grid| VisionGeometry::patch_count(grid, config)) + .collect::>>()?; + let total = lengths.iter().try_fold(0usize, |n, &rows| { + n.checked_add(rows) + .ok_or_else(|| anyhow::anyhow!("vision batch row overflow")) + })?; + // Keep the aggregate allocation within the existing shared geometry bound. + ensure!(total <= 65536, "vision batch exceeds 65536 patches"); + Ok(lengths) + } + pub fn new(grids: &[[usize; 3]], config: &VisionConfig) -> Result { + let lengths = Self::lengths(grids, config)?; + let total: usize = lengths.iter().sum(); + let mut geometry = VisionGeometry { + indices: Vec::with_capacity(total * 4), + weights: Vec::with_capacity(total * 4), + cos: Vec::with_capacity(total * (config.head_dim() / 2)), + sin: Vec::with_capacity(total * (config.head_dim() / 2)), + }; + for &grid in grids { + let image = VisionGeometry::new(grid, config)?; + geometry.indices.extend(image.indices); + geometry.weights.extend(image.weights); + geometry.cos.extend(image.cos); + geometry.sin.extend(image.sin); + } + Ok(Self { geometry }) + } +} diff --git a/src/models/qwen3_5/native/src/vision/model.rs b/src/models/qwen3_5/native/src/vision/model.rs index 205a3066..f70e75ca 100644 --- a/src/models/qwen3_5/native/src/vision/model.rs +++ b/src/models/qwen3_5/native/src/vision/model.rs @@ -1,5 +1,8 @@ //! GPU vision forward for the structurally validated, unmerged checkpoint. -use super::{VisionCheckpoint, VisionConfig, geometry::VisionGeometry}; +use super::{ + VisionCheckpoint, VisionConfig, + geometry::{BatchGeometry, VisionGeometry}, +}; use crate::{ cuda::{self, DeviceBuffer, Stream, api, check}, image_preprocess::ProcessedImage, @@ -36,6 +39,7 @@ pub struct VisionModel { config: VisionConfig, // Captures and their buffers retire before weights, GEMM workspace and stream. scratch: VecDeque, + batch_scratch: VecDeque, graph_enabled: bool, base: BTreeMap, lora: BTreeMap, @@ -127,6 +131,7 @@ impl VisionModel { let mut model = Self { config, scratch: VecDeque::new(), + batch_scratch: VecDeque::new(), graph_enabled: std::env::var("CUA_S1_VISION_GRAPH").as_deref() == Ok("1"), base: BTreeMap::new(), lora: BTreeMap::new(), @@ -307,6 +312,100 @@ impl VisionModel { pub fn forward(&mut self, image: &ProcessedImage) -> Result> { self.run(image, None) } + /// Project all unadapted 27B image rows together, with image-local attention. + /// Empty inputs do no device work; one image delegates the existing forward exactly. + /// Multi-image scratch/captures use a separate four-entry FIFO keyed by ordered grids. + pub fn forward_images(&mut self, images: &[ProcessedImage]) -> Result> { + match images { + [] => return Ok(Vec::new()), + [image] => return self.forward(image), + _ => {} + } + ensure!( + self.lora.is_empty(), + "batched vision does not support vision adapters" + ); + let grids: Vec<_> = images.iter().map(|image| image.image_grid_thw).collect(); + let lengths = BatchGeometry::lengths(&grids, &self.config)?; + let n: usize = lengths.iter().sum(); + for (image, &rows) in images.iter().zip(&lengths) { + ensure!( + image.pixel_values.len() == rows * 1536, + "vision pixel_values length does not match grid" + ); + ensure!( + image.resized_height == image.image_grid_thw[1] * 16 + && image.resized_width == image.image_grid_thw[2] * 16, + "vision resized geometry does not match grid" + ); + ensure!( + image.pixel_values.iter().all(|v| v.is_finite()), + "vision pixels must be finite" + ); + } + let pixels: Vec<_> = images + .iter() + .flat_map(|image| image.pixel_values.iter()) + .flat_map(|&value| bf16::from_f32(value).to_le_bytes()) + .collect(); + cuda::set_device(0)?; + let index = if let Some(index) = self.batch_scratch.iter().position(|s| s.grids == grids) { + index + } else { + let geometry = BatchGeometry::new(&grids, &self.config)?.geometry; + self.synchronize()?; + if self.batch_scratch.len() == 4 { + self.batch_scratch.pop_front(); + } + let buffers = Buffers::new(self, n, *lengths.iter().max().unwrap(), geometry)?; + self.batch_scratch.push_back(BatchScratch { + graph: None, + grids: grids.clone(), + lengths, + buffers, + }); + self.batch_scratch.len() - 1 + }; + let scratch = &self.batch_scratch[index]; + // SAFETY: every image was validated, and the ordered key owns exactly n pixel rows. + unsafe { + cuda::upload(scratch.buffers.pixels.at(0), &pixels, self.stream.0)?; + } + if self.graph_enabled + && let Some(graph) = &scratch.graph + { + graph.launch(self.stream.0)?; + batch_graph_trace("replayed", &grids); + return self.read(&scratch.buffers.out, n / 4 * self.config.out_hidden_size); + } + self.enqueue(&scratch.buffers, n, &scratch.lengths, &mut None)?; + let result = self.read(&scratch.buffers.out, n / 4 * self.config.out_hidden_size)?; + if self.graph_enabled { + let captured = cuda::Graph::capture(self.stream.0, || { + self.enqueue(&scratch.buffers, n, &scratch.lengths, &mut None) + }); + match captured { + Ok(graph) => { + self.batch_scratch[index].graph = Some(graph); + batch_graph_trace("captured", &grids); + } + Err(error) => { + eprintln!("Vision CUDA Graph capture failed; using eager execution: {error:#}"); + self.graph_enabled = false; + self.clear_graphs(); + } + } + } + Ok(result) + } + fn clear_graphs(&mut self) { + for scratch in &mut self.scratch { + scratch.graph = None; + } + for scratch in &mut self.batch_scratch { + scratch.graph = None; + } + } /// Optional synchronized stage downloads for parity diagnosis; ordinary forward skips them. pub fn forward_with_trace( &mut self, @@ -360,7 +459,7 @@ impl VisionModel { let scratch = &self.scratch[index]; // SAFETY: the current image has the validated exact resident buffer size. unsafe { - cuda::upload(scratch.pixels.at(0), &pixels, self.stream.0)?; + cuda::upload(scratch.buffers.pixels.at(0), &pixels, self.stream.0)?; } if callback.is_none() && self.graph_enabled @@ -368,16 +467,18 @@ impl VisionModel { { graph.launch(self.stream.0)?; graph_trace("replayed", scratch.grid); - return self.read(&scratch.out, n / 4 * self.config.out_hidden_size); + return self.read(&scratch.buffers.out, n / 4 * self.config.out_hidden_size); } - self.enqueue(scratch, &mut callback)?; - let result = self.read(&scratch.out, n / 4 * self.config.out_hidden_size)?; + self.enqueue(&scratch.buffers, n, &[n], &mut callback)?; + let result = self.read(&scratch.buffers.out, n / 4 * self.config.out_hidden_size)?; if let Some(callback) = callback.as_mut() { callback("merger.output", &result)?; } else if self.graph_enabled { // Readback completed warmup. Capture records without executing; keep // the eager result even when recording/instantiation fails. - let captured = cuda::Graph::capture(self.stream.0, || self.enqueue(scratch, &mut None)); + let captured = cuda::Graph::capture(self.stream.0, || { + self.enqueue(&scratch.buffers, n, &[n], &mut None) + }); match captured { Ok(graph) => { self.scratch[index].graph = Some(graph); @@ -386,21 +487,24 @@ impl VisionModel { Err(error) => { eprintln!("Vision CUDA Graph capture failed; using eager execution: {error:#}"); self.graph_enabled = false; - for scratch in &mut self.scratch { - scratch.graph = None; - } + self.clear_graphs(); } } } Ok(result) } - fn enqueue(&self, scratch: &Scratch, callback: &mut Trace<'_>) -> Result<()> { - let n = scratch.grid[1] * scratch.grid[2]; + fn enqueue( + &self, + buffers: &Buffers, + n: usize, + lengths: &[usize], + callback: &mut Trace<'_>, + ) -> Result<()> { let h = self.config.hidden_size; let intermediate = self.config.intermediate_size; let merged = h * 4; let output = self.config.out_hidden_size; - let Scratch { + let Buffers { pixels, indices, weights, @@ -418,7 +522,7 @@ impl VisionModel { mlp, out, .. - } = scratch; + } = buffers; self.linear("patch_embed.proj", pixels, x, n, h, 1536, w)?; self.trace(callback, "patch_embed", x, n * h)?; unsafe { @@ -495,20 +599,24 @@ impl VisionModel { ), "vision rotary", )?; - check( - (v2.attention)( - q.at(0), - k.at(0), - qkv.at(h * 2 * 2), - attn.at(0), - n as i32, - self.config.num_heads as i32, - self.config.head_dim() as i32, - attention_workspace.at(0), - self.stream.0, - ), - "vision attention", - )?; + let mut row = 0; + for &rows in lengths { + check( + (v2.attention)( + q.at(row * h * 2), + k.at(row * h * 2), + qkv.at((row * h * 3 + h * 2) * 2), + attn.at(row * h * 2), + rows as i32, + self.config.num_heads as i32, + self.config.head_dim() as i32, + attention_workspace.at(0), + self.stream.0, + ), + "vision attention", + )?; + row += rows; + } } } self.linear(&format!("{p}.attn.proj"), attn, delta, n, h, h, w)?; @@ -577,9 +685,23 @@ fn graph_trace(action: &str, grid: [usize; 3]) { eprintln!("Vision CUDA Graph {action} grid={grid:?}"); } } +fn batch_graph_trace(action: &str, grids: &[[usize; 3]]) { + if std::env::var("CUA_S1_GRAPH_TRACE").as_deref() == Ok("1") { + eprintln!("Vision CUDA Graph {action} grids={grids:?}"); + } +} struct Scratch { graph: Option, grid: [usize; 3], + buffers: Buffers, +} +struct BatchScratch { + graph: Option, + grids: Vec<[usize; 3]>, + lengths: Vec, + buffers: Buffers, +} +struct Buffers { pixels: DeviceBuffer, indices: DeviceBuffer, weights: DeviceBuffer, @@ -600,6 +722,20 @@ struct Scratch { impl Scratch { fn new(model: &VisionModel, grid: [usize; 3], geo: VisionGeometry) -> Result { let n = grid[1] * grid[2]; + Ok(Self { + graph: None, + grid, + buffers: Buffers::new(model, n, n, geo)?, + }) + } +} +impl Buffers { + fn new( + model: &VisionModel, + n: usize, + attention_rows: usize, + geo: VisionGeometry, + ) -> Result { let h = model.config.hidden_size; let width = model .config @@ -607,8 +743,6 @@ impl Scratch { .max(h * 4) .max(model.config.out_hidden_size); Ok(Self { - graph: None, - grid, pixels: DeviceBuffer::new(n * 1536 * 2)?, indices: model.upload( &geo.indices @@ -638,7 +772,7 @@ impl Scratch { attention_workspace: DeviceBuffer::new(if h == 1024 { 0 } else { - n * model.config.num_heads * 80 * 4 * 2 + attention_rows * model.config.num_heads * 80 * 4 * 2 })?, x: DeviceBuffer::new(n * h * 2)?, norm: DeviceBuffer::new(n * h * 2)?, @@ -671,3 +805,7 @@ impl Work { #[cfg(test)] #[path = "../../../../../../tests/qwen3_5/vision_graph.rs"] mod graph_tests; + +#[cfg(test)] +#[path = "../../../../../../tests/qwen3_5/vision_batch.rs"] +mod batch_tests; diff --git a/tests/qwen3_5/vision_batch.rs b/tests/qwen3_5/vision_batch.rs new file mode 100644 index 00000000..94914de8 --- /dev/null +++ b/tests/qwen3_5/vision_batch.rs @@ -0,0 +1,347 @@ +use super::*; + +fn config() -> VisionConfig { + VisionConfig::from_value(serde_json::json!({ + "depth":27,"hidden_size":1152,"intermediate_size":4304,"num_heads":16, + "num_position_embeddings":2304,"out_hidden_size":5120,"in_channels":3,"patch_size":16, + "temporal_patch_size":2,"spatial_merge_size":2,"hidden_act":"gelu_pytorch_tanh", + "deepstack_visual_indexes":[],"model_type":"qwen3_5" + })) + .unwrap() +} +fn image(grid: [usize; 3]) -> ProcessedImage { + ProcessedImage { + pixel_values: vec![0.; grid[1] * grid[2] * 1536], + image_grid_thw: grid, + resized_height: grid[1] * 16, + resized_width: grid[2] * 16, + } +} +#[cfg(unix)] +#[test] +fn cpu_batch_uses_aggregate_gemms_and_image_local_attention() { + if std::env::var_os("QWEN_TEST_VISION_BATCH_CHILD").is_none() { + let name = format!( + "{}::cpu_batch_uses_aggregate_gemms_and_image_local_attention", + module_path!().split_once("::").unwrap().1 + ); + let output = std::process::Command::new(std::env::current_exe().unwrap()) + .args(["--exact", &name, "--nocapture"]) + .env("QWEN_TEST_VISION_BATCH_CHILD", "1") + .output() + .unwrap(); + assert!( + output.status.success(), + "{}\n{}", + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr) + ); + return; + } + let directory = std::env::temp_dir().join(format!("qwen-vision-batch-{}", std::process::id())); + std::fs::create_dir(&directory).unwrap(); + let library = directory.join(if cfg!(target_os = "macos") { + "fake.dylib" + } else { + "fake.so" + }); + let source = directory.join("fake.c"); + let mut code = include_str!("vision_batch_fake.c").to_owned(); + let implemented = [ + "cs1_abi_version", + "cs1_error_string", + "cs1_set_device", + "cs1_malloc", + "cs1_free", + "cs1_stream_create", + "cs1_stream_sync", + "cs1_stream_destroy", + "cs1_upload", + "cs1_download", + "cs1_gemm_create", + "cs1_gemm_destroy", + "cs1_vision_linear", + "cs1_vision_norm", + "cs1_vision_bias", + "cs1_vision_gelu", + "cs1_vision_add", + "cs1_graph_begin", + "cs1_graph_end", + "cs1_graph_launch", + "cs1_graph_destroy", + ]; + let api_source = include_str!("../../src/models/qwen3_5/native/src/cuda.rs"); + for line in api_source + .split("api! {") + .nth(1) + .unwrap() + .split("\n}\n") + .next() + .unwrap() + .lines() + { + let line = line.trim(); + if line.starts_with("cs1_") { + let name = line.split('(').next().unwrap(); + if !implemented.contains(&name) { + code.push_str(&format!("\nint {name}(void){{return 77;}}\n")); + } + } + } + std::fs::write(&source, code).unwrap(); + let compiled = std::process::Command::new("cc") + .args(["-shared", "-fPIC"]) + .arg(&source) + .arg("-o") + .arg(&library) + .output() + .unwrap(); + assert!( + compiled.status.success(), + "{}", + String::from_utf8_lossy(&compiled.stderr) + ); + cuda::load(&library).unwrap(); + // SAFETY: these signatures are defined by the test-only C library above. + let metrics = unsafe { libloading::Library::new(&library).unwrap() }; + let count = unsafe { + *metrics + .get:: i32>(b"batch_linear_count\0") + .unwrap() + }; + let rows = unsafe { + *metrics + .get:: i32>(b"batch_linear_rows\0") + .unwrap() + }; + let out = unsafe { + *metrics + .get:: i32>(b"batch_linear_out\0") + .unwrap() + }; + let attention_count = unsafe { + *metrics + .get:: i32>(b"batch_attention_count\0") + .unwrap() + }; + let attention_rows = unsafe { + *metrics + .get:: i32>(b"batch_attention_rows\0") + .unwrap() + }; + let q = unsafe { + *metrics + .get:: usize>(b"batch_q\0") + .unwrap() + }; + let k = unsafe { + *metrics + .get:: usize>(b"batch_k\0") + .unwrap() + }; + let v = unsafe { + *metrics + .get:: usize>(b"batch_v\0") + .unwrap() + }; + let output = unsafe { + *metrics + .get:: usize>(b"batch_out\0") + .unwrap() + }; + let qkv_base = unsafe { + *metrics + .get:: usize>(b"batch_qkv_base\0") + .unwrap() + }; + let captures = unsafe { + *metrics + .get:: i32>(b"batch_captures\0") + .unwrap() + }; + let launches = unsafe { + *metrics + .get:: i32>(b"batch_launches\0") + .unwrap() + }; + let destroyed = unsafe { + *metrics + .get:: i32>(b"batch_destroyed\0") + .unwrap() + }; + let fail_capture = unsafe { + *metrics + .get::(b"batch_fail_capture\0") + .unwrap() + }; + let live = unsafe { + *metrics + .get:: i32>(b"batch_live\0") + .unwrap() + }; + let config = config(); + let mut base = BTreeMap::new(); + for name in config.inventory().keys() { + base.insert( + name.strip_prefix("model.visual.").unwrap().to_owned(), + DeviceBuffer::new(0).unwrap(), + ); + } + base.insert("__zero_bias".into(), DeviceBuffer::new(0).unwrap()); + let mut model = VisionModel { + config, + scratch: VecDeque::new(), + batch_scratch: VecDeque::new(), + graph_enabled: false, + base, + lora: BTreeMap::new(), + gemm: Gemm(unsafe { (api().cs1_gemm_create)(32 << 20) }), + stream: OwnedStream(cuda::new_stream().unwrap()), + }; + let images = [image([1, 2, 2]), image([1, 2, 4])]; + let result = model.forward_images(&images).unwrap(); + assert_eq!(result.len(), 3 * 5120); + for index in 0..unsafe { count() } { + let expected = if unsafe { out(index) } == 4608 || unsafe { out(index) } == 5120 { + 3 + } else { + 12 + }; + assert_eq!( + unsafe { rows(index) }, + expected, + "all-row vision GEMM must preserve aggregate M (call {index})" + ); + } + assert_eq!(unsafe { attention_count() }, 27 * 2); + for index in 0..54 { + assert_eq!( + unsafe { attention_rows(index) }, + if index % 2 == 0 { 4 } else { 8 } + ); + } + assert_eq!(unsafe { q(1) - q(0) }, 4 * 1152 * 2); + assert_eq!(unsafe { k(1) - k(0) }, 4 * 1152 * 2); + assert_eq!(unsafe { v(1) - v(0) }, 4 * 1152 * 3 * 2); + assert_eq!(unsafe { v(0) - qkv_base() }, 2 * 1152 * 2); + assert_eq!(unsafe { output(1) - output(0) }, 4 * 1152 * 2); + assert_eq!(model.batch_scratch.len(), 1); + assert!(model.scratch.is_empty()); + model.graph_enabled = true; + model.forward_images(&images).unwrap(); + assert_eq!(unsafe { captures() }, 1); + let mut changed = [image([1, 2, 2]), image([1, 2, 4])]; + changed[0].pixel_values[0] = 1.; + model.forward_images(&changed).unwrap(); + assert_eq!(unsafe { launches() }, 1); + let mut pixel = [0u8; 2]; + unsafe { + cuda::download( + &mut pixel, + model.batch_scratch[0].buffers.pixels.at(0), + model.stream.0, + ) + .unwrap(); + } + assert_eq!(bf16::from_le_bytes(pixel), bf16::ONE); + // Equal aggregate rows with a different order require a distinct capture. + model + .forward_images(&[image([1, 2, 4]), image([1, 2, 2])]) + .unwrap(); + assert_eq!(unsafe { captures() }, 2); + assert_eq!(model.batch_scratch.len(), 2); + let allocations = unsafe { live() }; + let dispatched = unsafe { count() }; + let mut invalid = [image([1, 2, 2]), image([1, 2, 4])]; + invalid[1].pixel_values[0] = f32::NAN; + assert!(model.forward_images(&invalid).is_err()); + assert_eq!(unsafe { live() }, allocations); + assert_eq!(unsafe { count() }, dispatched); + assert_eq!(model.batch_scratch.len(), 2); + for grids in [ + [[1, 2, 2], [1, 4, 2]], + [[1, 4, 2], [1, 2, 2]], + [[1, 2, 2], [1, 4, 4]], + ] { + model.forward_images(&grids.map(image)).unwrap(); + } + assert_eq!(model.batch_scratch.len(), 4); + assert_eq!(unsafe { destroyed() }, 1); + assert!( + !model + .batch_scratch + .iter() + .any(|s| s.grids == [[1, 2, 2], [1, 2, 4]]) + ); + // Single-image dispatch still uses the original separate cache/pipeline. + model.forward_images(&[image([1, 2, 2])]).unwrap(); + assert_eq!(model.scratch.len(), 1); + assert_eq!(model.batch_scratch.len(), 4); + assert!(model.forward_images(&[]).unwrap().is_empty()); + unsafe { + fail_capture(1); + } + let fallback = model + .forward_images(&[image([1, 4, 4]), image([1, 2, 4])]) + .unwrap(); + assert_eq!(fallback.len(), 6 * 5120); + assert!(!model.graph_enabled); + assert!(model.scratch.iter().all(|s| s.graph.is_none())); + assert!(model.batch_scratch.iter().all(|s| s.graph.is_none())); + unsafe { + fail_capture(0); + } + let capture_count = unsafe { captures() }; + model.forward_images(&images).unwrap(); + assert_eq!(unsafe { captures() }, capture_count); + + drop(model); + assert_eq!(unsafe { live() }, 0); + drop(metrics); + std::fs::remove_dir_all(directory).unwrap(); +} + +#[test] +fn batch_geometry_preserves_each_image_coordinates_and_merge_boundaries() { + let config = config(); + let grids = [[1, 2, 2], [1, 2, 4], [1, 4, 2]]; + assert_eq!(BatchGeometry::lengths(&grids, &config).unwrap(), [4, 8, 8]); + let batch = BatchGeometry::new(&grids, &config).unwrap().geometry; + let images: Vec<_> = grids + .iter() + .map(|&g| VisionGeometry::new(g, &config).unwrap()) + .collect(); + assert_eq!( + batch.indices, + images + .iter() + .flat_map(|g| g.indices.iter().copied()) + .collect::>() + ); + assert_eq!( + batch.weights, + images + .iter() + .flat_map(|g| g.weights.iter().copied()) + .collect::>() + ); + assert_eq!( + batch.cos, + images + .iter() + .flat_map(|g| g.cos.iter().copied()) + .collect::>() + ); + assert_eq!( + batch.sin, + images + .iter() + .flat_map(|g| g.sin.iter().copied()) + .collect::>() + ); + assert_eq!(batch.cos.len(), 20 * 36); + assert!(BatchGeometry::lengths(&[[1, 256, 256], [1, 2, 2]], &config).is_err()); + assert!(BatchGeometry::lengths(&[[1, 2, 2]; 5], &config).is_err()); + assert!(BatchGeometry::lengths(&[[1, 2, 2], [2, 2, 2]], &config).is_err()); + assert!(BatchGeometry::lengths(&[[1, usize::MAX - 1, 2], [1, 2, 2]], &config).is_err()); +} diff --git a/tests/qwen3_5/vision_batch_fake.c b/tests/qwen3_5/vision_batch_fake.c new file mode 100644 index 00000000..560cdb57 --- /dev/null +++ b/tests/qwen3_5/vision_batch_fake.c @@ -0,0 +1,54 @@ +#include +#include +#include +static int live, linear_count, attention_count, captures, launches, destroyed, fail_capture; +static int rows[16384], outs[16384], lens[16384]; +static uintptr_t qkv_base; +static uintptr_t qs[16384], ks[16384], vs[16384], os[16384]; +unsigned cs1_abi_version(void){return 6;} +const char *cs1_error_string(int c){(void)c;return "fake CUDA error";} +int cs1_set_device(int d){(void)d;return 0;} +int cs1_malloc(void **p,size_t n){*p=calloc(1,n);if(!*p)return 1;live++;return 0;} +int cs1_free(void *p){free(p);live--;return 0;} +int cs1_stream_create(void **p){*p=(void*)1;return 0;} +int cs1_stream_sync(void *s){(void)s;return 0;} +int cs1_stream_destroy(void *s){(void)s;return 0;} +int cs1_upload(void *d,const void *s,size_t n,void *stream){(void)stream;memcpy(d,s,n);return 0;} +int cs1_download(void *d,const void *s,size_t n,void *stream){(void)stream;memcpy(d,s,n);return 0;} +void *cs1_gemm_create(size_t n){(void)n;return (void*)1;} +void cs1_gemm_destroy(void *g){(void)g;} +int cs1_vision_linear(void *g,const void *x,const void *w,const void *bias,void *y,int m,int n,int k,void *s){ + (void)g;(void)x;(void)w;(void)bias;(void)y;(void)k;(void)s; + rows[linear_count]=m;outs[linear_count]=n;linear_count++;return 0; +} +int cs1_vision_norm(const void*x,const void*w,const void*b,void*y,int n,int h,void*s){(void)x;(void)w;(void)b;(void)y;(void)n;(void)h;(void)s;return 0;} +int cs1_vision_bias(void*x,const void*b,size_t n,int h,void*s){(void)x;(void)b;(void)n;(void)h;(void)s;return 0;} +int cs1_vision_gelu(void*x,size_t n,int e,void*s){(void)x;(void)n;(void)e;(void)s;return 0;} +int cs1_vision_add(void*x,const void*d,size_t n,void*s){(void)x;(void)d;(void)n;(void)s;return 0;} +int cs1_vision_position_v2(void*x,const void*t,const int*i,const float*w,int n,int h,void*s){(void)x;(void)t;(void)i;(void)w;(void)n;(void)h;(void)s;return 0;} +int cs1_vision_rope_v2(const void*x,const float*c,const float*si,void*q,void*k,int n,int h,int d,void*s){if(attention_count==0)qkv_base=(uintptr_t)x;(void)c;(void)si;(void)q;(void)k;(void)n;(void)h;(void)d;(void)s;return 0;} +int cs1_vision_attention_v2(const void*q,const void*k,const void*v,void*o,int n,int h,int d,void*w,void*s){ + (void)h;(void)d;(void)w;(void)s; + lens[attention_count]=n;qs[attention_count]=(uintptr_t)q;ks[attention_count]=(uintptr_t)k;vs[attention_count]=(uintptr_t)v;os[attention_count]=(uintptr_t)o;attention_count++;return 0; +} +int cs1_graph_begin(void*s){(void)s;return 0;} +int cs1_graph_end(void*s,void**p){(void)s;if(fail_capture){*p=NULL;return 1;}*p=malloc(1);captures++;return 0;} +int cs1_graph_launch(void*g,void*s){(void)g;(void)s;launches++;return 0;} +int cs1_graph_destroy(void*g){free(g);destroyed++;return 0;} +int batch_live(void){return live;} +int batch_linear_count(void){return linear_count;} +int batch_linear_rows(int i){return rows[i];} +int batch_linear_out(int i){return outs[i];} +int batch_attention_count(void){return attention_count;} +int batch_attention_rows(int i){return lens[i];} +uintptr_t batch_q(int i){return qs[i];} +uintptr_t batch_k(int i){return ks[i];} +uintptr_t batch_v(int i){return vs[i];} +uintptr_t batch_out(int i){return os[i];} +int batch_captures(void){return captures;} +int batch_launches(void){return launches;} +int batch_destroyed(void){return destroyed;} + +uintptr_t batch_qkv_base(void){return qkv_base;} + +void batch_fail_capture(int value){fail_capture=value;} From 3a2ff031daf6e2a441902ff34cdf6efe72f3d24b Mon Sep 17 00:00:00 2001 From: levius <2114377220@qq.com> Date: Sat, 10 Oct 2026 23:08:45 +0800 Subject: [PATCH 11/21] Run Jemm vision projections over all request images together --- src/models/jemm/native/src/executor.rs | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/models/jemm/native/src/executor.rs b/src/models/jemm/native/src/executor.rs index 3fc488af..345372d5 100644 --- a/src/models/jemm/native/src/executor.rs +++ b/src/models/jemm/native/src/executor.rs @@ -159,9 +159,7 @@ impl Loaded { fn execute(&mut self, inputs: &[Input]) -> Result>> { let mut embeddings = Vec::new(); if let Some(first) = inputs.first() { - for image in first.images.iter() { - embeddings.extend(self.vision.forward(image)?); - } + embeddings = self.vision.forward_images(&first.images)?; } let mut rows = Vec::with_capacity(inputs.len()); for input in inputs { From 06ea197326d53416e90df282249a3c92eb947d8a Mon Sep 17 00:00:00 2001 From: levius <2114377220@qq.com> Date: Sun, 11 Oct 2026 01:18:53 +0800 Subject: [PATCH 12/21] jemm: align unmerged multi-image execution with reference numerics --- .../jemm-reference-20261011/README.md | 77 +++ recipe/jemm/README.md | 37 +- src/backends/cuda/qwen3_5/README.md | 17 + src/backends/cuda/qwen3_5/build.sh | 10 +- src/backends/cuda/qwen3_5/gemm.cu | 10 +- src/backends/cuda/qwen3_5/ops.h | 28 + .../cuda/qwen3_5/reference_attention.cuh | 33 + src/backends/cuda/qwen3_5/reference_gdn.cu | 634 ++++++++++++++++++ .../qwen3_5/reference_language_attention.cu | 192 ++++++ src/backends/cuda/qwen3_5/reference_norm.cu | 34 + src/backends/cuda/qwen3_5/reference_patch.cu | 83 +++ src/backends/cuda/qwen3_5/reference_prep.cu | 83 +++ src/backends/cuda/qwen3_5/reference_vision.cu | 100 +++ .../qwen3_5/reference_vision_attention.cu | 196 ++++++ src/models/jemm/README.md | 45 +- src/models/jemm/native/src/executor.rs | 8 +- src/models/qwen3_5/native/src/cuda.rs | 13 +- .../qwen3_5/native/src/cuda/reference.rs | 139 ++++ src/models/qwen3_5/native/src/model.rs | 136 +++- .../qwen3_5/native/src/vision/geometry.rs | 20 + src/models/qwen3_5/native/src/vision/model.rs | 127 +++- tests/jemm/data/parity/corpus.jsonl | 16 + tests/jemm/data/parity/reference.json | 528 +++++++++++++++ tests/jemm/parity.py | 100 +++ tests/jemm/reference_kernels.py | 133 ++++ tests/qwen3_5/vision_batch.rs | 1 + 26 files changed, 2724 insertions(+), 76 deletions(-) create mode 100644 docs/benchmarks/jemm-reference-20261011/README.md create mode 100644 src/backends/cuda/qwen3_5/reference_attention.cuh create mode 100644 src/backends/cuda/qwen3_5/reference_gdn.cu create mode 100644 src/backends/cuda/qwen3_5/reference_language_attention.cu create mode 100644 src/backends/cuda/qwen3_5/reference_norm.cu create mode 100644 src/backends/cuda/qwen3_5/reference_patch.cu create mode 100644 src/backends/cuda/qwen3_5/reference_prep.cu create mode 100644 src/backends/cuda/qwen3_5/reference_vision.cu create mode 100644 src/backends/cuda/qwen3_5/reference_vision_attention.cu create mode 100644 src/models/qwen3_5/native/src/cuda/reference.rs create mode 100644 tests/jemm/data/parity/corpus.jsonl create mode 100644 tests/jemm/data/parity/reference.json create mode 100644 tests/jemm/parity.py create mode 100644 tests/jemm/reference_kernels.py diff --git a/docs/benchmarks/jemm-reference-20261011/README.md b/docs/benchmarks/jemm-reference-20261011/README.md new file mode 100644 index 00000000..4a16bb59 --- /dev/null +++ b/docs/benchmarks/jemm-reference-20261011/README.md @@ -0,0 +1,77 @@ +# JEMM 1–4-image numerical regression + +The optional reference-numerics path passes the frozen 16-request / 25-question +corpus on one NVIDIA A800 80 GB. Maximum probability drift is +**0.0166172279642417**. The unchanged gates are probability/confidence/noul 0.02, +Score expected value 0.1, and winner agreement when the reference margin is at +least 0.05. Low-margin cases are retained. Two- and four-image results agree to +floating-point host-softmax noise; the maximum is the three-image Choice case. + +This is a correctness campaign, not a throughput or latency benchmark. Each +candidate was declared before its one complete pass. Failed diagnostic +candidates remain in the evidence archive. No tolerance was widened and no +failed case was removed. Broader prompts, other accelerators/framework versions, +labelled task accuracy, speed and peak memory are unverified. + +## Reproduce + +The maintained [corpus](../../../tests/jemm/data/parity/corpus.jsonl) and +[official reference oracle](../../../tests/jemm/data/parity/reference.json) are +consumed by [the HTTP validator](../../../tests/jemm/parity.py). Images are small +synthetic deployment-console fixtures; request text contains no customer data. +The corpus SHA-256 is +`82c05de9cb21b7763864dde6e80cc6bed588219d4d53da2d8157c0ec27a3663c`. + +Follow the [JEMM recipe](../../../recipe/jemm/README.md) to export the pinned BF16 +base and separate FP32 rank-16 adapter, build with cuDNN 9, and start the worker. +The worker verifies all checkpoint hashes. Use an exclusively reserved GPU and +submit the corpus sequentially (concurrency 1): + +```sh +python tests/jemm/parity.py --url http://127.0.0.1:8000 \ + --corpus tests/jemm/data/parity/corpus.jsonl \ + --reference tests/jemm/data/parity/reference.json \ + --report /tmp/jemm-parity.json +``` + +The reference oracle uses JEMM source +`6822fe0fd53c5e6670af6ba99fb2c857a661e532`, base +`1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0`, and adapter +`76e3c209e8441fa658221c7ba2725bad2f811176`. Its active GDN implementation is FLA, +not the FP32 fallback function whose name appears in the Transformers wrapper. +The measured environment used Torch 2.12.1+cu130, Transformers 5.17.0, +FLA 0.5.2, Triton 3.7.1, CUDA 13.0 and cuDNN 9. + +## What was checked + +| Layer | Result and scope | +| --- | --- | +| Full native forward | 16 requests / 25 questions pass, graphs disabled, no explicit warmup or retries. | +| Actual HTTP worker | The same corpus passes after the worker's two required startup forwards. Text-language and vision graphs enabled; multimodal language remains eager. Health READY before/after; no capture error. | +| Reference kernels | 11 fixed-seed checks: GDN and language attention at 63/278/475 tokens, RMSNorm at 1/278 rows, vision attention at 256/512 patches, and patch convolution. Every BF16 output equals the pinned framework; graph replay equals eager output. | +| Existing CUDA API | All 11 checkpoint-free kernel regressions pass, including prefix continuation and capture recovery. The checkpoint-dependent prefix test is not included. | +| Selected label head | Real CUDA constant-row projection test passes. | +| Compatibility build | CUDA library builds with and without cuDNN; reference numerics require the cuDNN-enabled variant. | + +The [operator test](../../../tests/jemm/reference_kernels.py) requires the pinned +Torch/FLA environment. The existing CUDA tests remain under +[`tests/qwen3_5/kernels.rs`](../../../tests/qwen3_5/kernels.rs). Model-scale 4B +checkpoint execution was not repeated; legacy source entry points are preserved +and their kernel regressions were run. + +## Evidence and review scope + +[Raw evidence archive](https://github.com/Levius-Fubuki/system1-omni/releases/tag/jemm-reference-20261011) +contains native/HTTP responses, predeclared protocols, exact library/binary/source +archive hashes, operator results, build/test logs, and failed diagnostic records. +The measured source is parent `3a2ff031daf6e2a441902ff34cdf6efe72f3d24b` plus the +reference API integration recorded in `production-source-v1.tar.gz`; later +comment, documentation and test additions do not change that execution path. +The source archive, not a later untested commit label, identifies the run. + +The fix couples export semantics, projection shapes and arithmetic: preserve +BF16 base/FP32 adapters, use original component GEMM dimensions, aggregate vision +projection rows with image-local attention, and select reference CUDA functions +only for JEMM. Review in that order, then inspect model resource ownership, +response reconstruction and tests. Existing Qwen API callers retain legacy +arithmetic. The reference path does not support prefix caching. diff --git a/recipe/jemm/README.md b/recipe/jemm/README.md index 06a1a1a3..10fa1f22 100644 --- a/recipe/jemm/README.md +++ b/recipe/jemm/README.md @@ -1,7 +1,7 @@ # JEMM setup and validation Run from the repository root on Linux with a CUDA GPU (compute capability 8.0+), -nvcc/cuBLASLt and Rust. Historical validation used one A800 80 GB. The raw base +nvcc/cuBLASLt, cuDNN 9 headers/library and Rust. Historical validation used one A800 80 GB. The raw base and adapter need about 55.6 GB disk. V2 exports hardlink unchanged files on the same filesystem; cross-filesystem exports copy the 17 language shards and raw FP32 adapter, requiring additional disk for those files. Export is an @@ -46,7 +46,9 @@ inputs and outputs share file contents and must remain immutable. ## Build, launch and request ```sh -src/backends/cuda/qwen3_5/build.sh target/release 80 +CUDNN_INCLUDE_DIR=/path/to/cudnn/include \ +CUDNN_LIB_DIR=/path/to/cudnn/lib \ + src/backends/cuda/qwen3_5/build.sh target/release 80 cargo build --release --locked -p omni-jemm-native -p omni-jev JEMM_MODEL="$JEMM_DATA/native" \ JEMM_CUDA_LIB="$PWD/target/release/libqwen3_5_cuda.so" \ @@ -54,7 +56,8 @@ JEMM_CUDA_LIB="$PWD/target/release/libqwen3_5_cuda.so" \ ``` Replace `80` with the GPU's compute capability. Rebuild the CUDA library and -consumers together; JEMM's 27B vision path requires the additive v2 symbols. +consumers together; JEMM requires the additive reference-numerics symbols and a +cuDNN-enabled build. The other Qwen workers keep their existing CUDA entry points. The default listener is `127.0.0.1:8000`, set by `JEMM_HOST`/`JEMM_PORT`. It binds after text and image warmup complete. In another terminal: @@ -68,11 +71,10 @@ curl --fail http://127.0.0.1:8080/v1/systemone \ ``` Optional `images` contains up to four base64 PNG/JPEG/WebP strings; images precede -text and are shared across questions. **Two-/four-image reference parity exceeds -the predeclared probability gate in the supplemental corpus.** See the -[model contract and numerical limitation](../../src/models/jemm/README.md#validation-and-known-multi-image-limitation) -before relying on multi-image outputs. This is a documented limitation, not a -claim that those inputs now pass. Native CPU/Metal execution is unsupported. +text and are shared across questions. The frozen 16-request / 25-question corpus +passes the original numerical gates, including 2/3/4-image cases. See the +[validation scope and evidence](../../docs/benchmarks/jemm-reference-20261011/README.md). +Native CPU/Metal execution is unsupported. ## Checks and reference @@ -89,10 +91,21 @@ JEMM_CUDA_LIB="$PWD/target/release/libqwen3_5_cuda.so" \ --ignored --test-threads=1 ``` -CPU checks compile the worker and exercise host/export contracts. The new -unmerged FP32-adapter path has no fresh GPU parity claim in these host checks. They cannot prove the -real image warmup, multi-image parity or device failure recovery. Hardware tests -require a separately reserved device; normal CI compiles and ignores the CUDA case. +CPU checks compile the worker and exercise host/export contracts. Device checks +require a separately reserved GPU; normal CI compiles and ignores CUDA tests. +The [frozen numerical campaign](../../docs/benchmarks/jemm-reference-20261011/README.md) +records the unmerged adapter path and its validation scope. + +```sh +# Pinned Torch/FLA environment; tests graph replay as well as eager values: +python tests/jemm/reference_kernels.py target/release/libqwen3_5_cuda.so \ + --report /tmp/jemm-operators.json +# Against the running native worker, on the reserved device: +python tests/jemm/parity.py --url http://127.0.0.1:8000 \ + --corpus tests/jemm/data/parity/corpus.jsonl \ + --reference tests/jemm/data/parity/reference.json \ + --report /tmp/jemm-parity.json +``` For the official reference, clone ypcypc/JEMM and detach at `6822fe0fd53c5e6670af6ba99fb2c857a661e532`, then use `python -m jemm.serve diff --git a/src/backends/cuda/qwen3_5/README.md b/src/backends/cuda/qwen3_5/README.md index 63825533..6fed31de 100644 --- a/src/backends/cuda/qwen3_5/README.md +++ b/src/backends/cuda/qwen3_5/README.md @@ -27,3 +27,20 @@ experimental JEV-VL worker; existing full-prefill callers keep their own paths. ABI 6 adds mandatory copy and prefix entry points after ABI 5 native vision. Rebuild the library and **all** Qwen worker binaries together; older libraries are rejected. Current-branch GPU regression is required before release. + +## Optional JEMM reference arithmetic + +JEMM enables the additive `cs1_reference_*` API group after model load and before +any forward. It requires cuDNN 9; build with `CUDNN_INCLUDE_DIR` and +`CUDNN_LIB_DIR`. Builds without those variables retain the legacy ABI-6 functions; +JEMM reports a missing-capability error rather than silently selecting older numerics. + +The reference path uses a per-handle GEMM reduction policy, separate normalization, +attention and GDN functions, and shape-owned cuDNN Conv3D plans. Attention and +convolution workspaces allocate before capture and retire with their model/shape. +The rotary entry point consumes FP32 angles and computes trigonometry on device; +legacy rotary functions still consume cos/sin tables. Prefix caching is unsupported +on this opt-in path. The shared workers' default arithmetic remains unchanged. + +The [JEMM numerical record](../../../../docs/benchmarks/jemm-reference-20261011/README.md) +contains a frozen 1–4-image corpus and reproducible operator/HTTP checks. diff --git a/src/backends/cuda/qwen3_5/build.sh b/src/backends/cuda/qwen3_5/build.sh index ef2dad6b..bdf323ef 100755 --- a/src/backends/cuda/qwen3_5/build.sh +++ b/src/backends/cuda/qwen3_5/build.sh @@ -20,8 +20,16 @@ link=(-lcublasLt) if lib=$(cd "$(dirname "$nvcc")/../lib64" 2>/dev/null && pwd); then link=(-L"$lib" -lcublasLt -Xlinker -rpath -Xlinker "$lib") fi +extra=() +# Optional JEMM framework-compatible Conv3D. Legacy builds do not require cuDNN. +if [ -n "${CUDNN_INCLUDE_DIR:-}" ] || [ -n "${CUDNN_LIB_DIR:-}" ]; then + : "${CUDNN_INCLUDE_DIR:?set CUDNN_INCLUDE_DIR and CUDNN_LIB_DIR together}" + : "${CUDNN_LIB_DIR:?set CUDNN_INCLUDE_DIR and CUDNN_LIB_DIR together}" + extra=(-DCS1_REFERENCE_CUDNN -I"$CUDNN_INCLUDE_DIR") + link+=(-L"$CUDNN_LIB_DIR" -l:libcudnn.so.9 -Xlinker -rpath -Xlinker "$CUDNN_LIB_DIR") +fi mkdir -p "$out" "$nvcc" -O3 -std=c++17 -gencode "arch=compute_${arch},code=[sm_${arch},compute_${arch}]" \ -shared -Xcompiler -fPIC -Xcompiler -Wall,-Wextra -I"$here" "$here"/*.cu \ - "${link[@]}" -o "$out/libqwen3_5_cuda.so" + "${extra[@]}" "${link[@]}" -o "$out/libqwen3_5_cuda.so" echo "built $out/libqwen3_5_cuda.so for sm_${arch}" diff --git a/src/backends/cuda/qwen3_5/gemm.cu b/src/backends/cuda/qwen3_5/gemm.cu index 154ad698..b7f8d63d 100644 --- a/src/backends/cuda/qwen3_5/gemm.cu +++ b/src/backends/cuda/qwen3_5/gemm.cu @@ -28,6 +28,7 @@ struct Gemm { cublasLtHandle_t handle = nullptr; void* workspace = nullptr; size_t workspace_bytes = 0; + bool reference = false; std::map plans; }; @@ -66,7 +67,7 @@ int first_choice(Gemm& g, Plan& p, bool vision) { if (s != CUBLAS_STATUS_SUCCESS) return status(s); cublasLtMatmulPreferenceSetAttribute(pref, CUBLASLT_MATMUL_PREF_MAX_WORKSPACE_BYTES, &g.workspace_bytes, sizeof(g.workspace_bytes)); - const uint32_t schemes = vision ? CUBLASLT_REDUCTION_SCHEME_NONE : (CUBLASLT_REDUCTION_SCHEME_MASK & ~CUBLASLT_REDUCTION_SCHEME_INPLACE); + const uint32_t schemes = g.reference ? CUBLASLT_REDUCTION_SCHEME_MASK : vision ? CUBLASLT_REDUCTION_SCHEME_NONE : (CUBLASLT_REDUCTION_SCHEME_MASK & ~CUBLASLT_REDUCTION_SCHEME_INPLACE); cublasLtMatmulPreferenceSetAttribute(pref, CUBLASLT_MATMUL_PREF_REDUCTION_SCHEME_MASK, &schemes, sizeof(schemes)); cublasLtMatmulHeuristicResult_t r{}; @@ -161,3 +162,10 @@ extern "C" int cs1_gemm_f32(void* gemm, const float* x, const float* w, float* y return status(cublasLtMatmul(g->handle, p->op, &alpha, w, p->a, x, p->b, &beta, y, p->c, y, p->c, &p->algo, g->workspace, g->workspace_bytes, static_cast(stream))); } + +// JEMM matches the pinned framework heuristic, including all reduction schemes. +extern "C" void* cs1_reference_gemm_create(size_t workspace_bytes) { + auto* g=static_cast(cs1_gemm_create(workspace_bytes)); + if(g) g->reference=true; + return g; +} diff --git a/src/backends/cuda/qwen3_5/ops.h b/src/backends/cuda/qwen3_5/ops.h index 984db6f6..487aab3a 100644 --- a/src/backends/cuda/qwen3_5/ops.h +++ b/src/backends/cuda/qwen3_5/ops.h @@ -142,6 +142,34 @@ int cs1_vision_add(void* x, const void* delta, size_t n, void* stream); int cs1_vision_to_float(const void* x, float* out, size_t n, void* stream); int cs1_vision_lora_add(void* x, const float* delta, size_t n, float scale, void* stream); + +// ---- Optional JEMM framework numerics (ABI 6 additive) ---- +void* cs1_reference_gemm_create(size_t workspace_bytes); +int cs1_reference_rms_norm(const void* x, const void* w, void* out, int rows, int D, float eps, void* stream); +int cs1_reference_add_rms_norm(void* residual, const void* delta, const void* w, void* out, int rows, int D, + float eps, void* stream); +size_t cs1_reference_gdn_workspace_floats(int T, int H); +int cs1_reference_gdn_prefill(const void* q, const void* k, const void* v, const float* g, const void* beta, + void* o, float* workspace, int T, int H, int HK, float scale, void* stream); +int cs1_reference_attn_prep(const void* qg, const void* kr, int ld, const void* qw, const void* kw, + const void* cos, const void* sin, void* q, void* gate, void* k, int T, int Hq, int Hk, + int Dh, int half, float eps, void* stream); +int cs1_reference_vision_norm(const void* x, const void* w, const void* b, void* y, int rows, int d, void* stream); +int cs1_reference_vision_rope(const void* qkv, const float* co, const float* si, void* q, void* k, int n, int hidden, int head_dim, void* stream); +int cs1_reference_vision_attention(const void* q, const void* k, const void* v, void* out, int n, int heads, int head_dim, void* workspace, void* stream); +// Rotary's co parameter is FP32 angles, si is unused; unlike the legacy cos/sin API. +// Language workspace is measured in FP32 elements and covers every length <= capacity. +// Vision workspace adds 4*capacity*16*80 BF16 padding elements before that FP32 region. +int cs1_reference_available(void); +size_t cs1_reference_language_attention_workspace_floats(int capacity,int heads); +size_t cs1_reference_vision_attention_workspace_floats(int capacity,int heads); +int cs1_reference_attention(const void* q,const void* k,const void* v,int ldv,const void* gate,void* out,int tokens,int heads,int kv_heads,float scale,void* workspace,void* stream); +int cs1_reference_vision_attention_padded(const void* q,const void* k,const void* v,void* out,int tokens,int heads,void* workspace,void* stream); +// Create/destroy outside capture, on the owning device. The caller synchronizes before destroy. +void* cs1_reference_patch_create(int rows,void* stream); +int cs1_reference_patch(void* plan,const void* x,const void* weight,void* out); +void cs1_reference_patch_destroy(void* plan); + #ifdef __cplusplus } #endif diff --git a/src/backends/cuda/qwen3_5/reference_attention.cuh b/src/backends/cuda/qwen3_5/reference_attention.cuh new file mode 100644 index 00000000..2284692d --- /dev/null +++ b/src/backends/cuda/qwen3_5/reference_attention.cuh @@ -0,0 +1,33 @@ +#pragma once +#include "common.cuh" +namespace cs1 { namespace reference { +inline int split_count(int tokens,int heads,int tile,int sms) { + int blocks=(tokens+tile-1)/tile,workers=heads*((tokens+63)/64),splits=1; + if(workers<0.8f*(sms*2)) { + int maximum=min(min(128,sms*2),blocks);float best=0.f; + for(int pass=0;pass<2;pass++) for(int n=1;n<=maximum;n++) { + if(n>1 && (blocks+n-1)/n==(blocks+n-2)/(n-1)) continue; + float waves=(float)(workers*n)/(sms*2),efficiency=waves/ceilf(waves); + if(pass==0) best=fmaxf(best,efficiency); + else if(efficiency>=0.85f*best) {splits=n;break;} + } + } + return splits; +} +inline int sm_count() { + int device=0,sms=0; + if(cudaGetDevice(&device)!=cudaSuccess || cudaDeviceGetAttribute(&sms,cudaDevAttrMultiProcessorCount,device)!=cudaSuccess) return 0; + return sms; +} +inline size_t workspace_floats(int capacity,int heads,int dim,int tile) { + if(capacity<=0 || capacity>65536 || heads<=0 || heads>48) return 0; + int sms=sm_count();if(sms<=0) return 0; + size_t maximum=0; + for(int end=64;endcount?maximum:count; + } + return maximum; +} +} } diff --git a/src/backends/cuda/qwen3_5/reference_gdn.cu b/src/backends/cuda/qwen3_5/reference_gdn.cu new file mode 100644 index 00000000..fe7f0fe2 --- /dev/null +++ b/src/backends/cuda/qwen3_5/reference_gdn.cu @@ -0,0 +1,634 @@ +// Optional JEMM Gated DeltaNet arithmetic, matched to the pinned active FLA path. +// Chunks of 64: BF16 Q/K products, FP32 diagonal inverse, layout-specific TF32 +// off-diagonal inverse products, BF16 U/W and recurrent-state snapshots. +// Rounding and reduction order are intentional: rare BF16 changes in early +// layers otherwise amplify across the 64-layer JEMM language model. +#include +#include +#include +#include + +#include "mma.cuh" +#include "ops.h" + +using namespace nvcuda; + +namespace { +__device__ __forceinline__ float reference_inverse_norm(float squared) { + float root,inverse; + asm("sqrt.approx.ftz.f32 %0, %1;" : "=f"(root) : "f"(squared)); + asm("div.full.f32 %0, %1, %2;" : "=f"(inverse) : "f"(1.f), "f"(root)); + return inverse; +} +// Triton operand layouts select even/odd K order for the first product and +// natural K order for the second. Keep multiple products in one accumulator. +template +__device__ __forceinline__ void reference_dot16(const float* a,int lda,const float* b,int ldb,float (&acc)[2][4]) { + int lane=threadIdx.x&31,row=lane/4,col=lane%4; +#pragma unroll + for(int k=0;k<16;k+=8) { + const int k0=EvenOdd ? ((k+col)*2)%16+(k+col)/8 : k+col, k1=EvenOdd ? ((k+col+4)*2)%16+(k+col+4)/8 : k+col+4; + uint32_t av[4]={__float_as_uint(a[row*lda+k0])&0xffffe000u, + __float_as_uint(a[(row+8)*lda+k0])&0xffffe000u, + __float_as_uint(a[row*lda+k1])&0xffffe000u, + __float_as_uint(a[(row+8)*lda+k1])&0xffffe000u}; +#pragma unroll + for(int n=0;n<2;n++) { + uint32_t b0=__float_as_uint(b[k0*ldb+row+n*8])&0xffffe000u; + uint32_t b1=__float_as_uint(b[k1*ldb+row+n*8])&0xffffe000u; + asm volatile("mma.sync.aligned.m16n8k8.row.col.f32.tf32.tf32.f32 {%0,%1,%2,%3}, {%4,%5,%6,%7}, {%8,%9}, {%0,%1,%2,%3};" + : "+f"(acc[n][0]),"+f"(acc[n][1]),"+f"(acc[n][2]),"+f"(acc[n][3]) + : "r"(av[0]),"r"(av[1]),"r"(av[2]),"r"(av[3]),"r"(b0),"r"(b1)); + } + } +} + +__device__ __forceinline__ float fla_exp2(float x) { float y; asm("ex2.approx.ftz.f32 %0, %1;" : "=f"(y) : "f"(x)); return y; } + +using cs1::bf16; +using cs1::warp_sum; + +constexpr int C = 64; // chunk length +constexpr int K = 128; // key head dim +constexpr int V = 128; // value head dim +constexpr int THREADS = 256; +// Padded row strides for the shared matrices. +constexpr int KP = K + 4; // packed TF32 component planes +constexpr int CP = C + 4; // float +constexpr int HB = K + 8; // bfloat16 +constexpr int TB = C + 8; // bfloat16 + +// Retain the original shared-memory reservation; these regions now hold +// normalized BF16 Q/K, then are reused for inverse/output operands. +struct Tf32Row { + uint16_t lo[KP]; + uint8_t hi[KP]; +}; +static_assert(sizeof(Tf32Row) == KP * 3); + + + +struct Work { + bf16* u; // [H, NCC, V] + bf16* w; // [H, NCC, K] + bf16* qd; // [H, NCC, K] normalized Q + bf16* kd; // [H, NCC, K] normalized K + float* p; // [H, NC, C, C] float32 scratch of q.k + bf16* pb; // [H, NC, C, C] (q.k) exp(cum_i - cum_j), j <= i + float* cumulative; // [H, NCC] cumulative base-2 gates + float* decay; // [H, NC] exp(cum_last) + bf16* s; // [H, NC, K, V] state before each chunk + bf16* vn; // [H, NCC, V] v_new +}; + +// Byte offsets of the workspace parts, 256-byte aligned. +struct Layout { + size_t u, w, qd, kd, p, pb, decay, s, vn, cumulative, total; + Layout(int T, int H) { + const size_t NC = (T + C - 1) / C, NCC = NC * C; + size_t at = 0; + auto take = [&](size_t bytes) { + const size_t off = at; + at = (at + bytes + 255) / 256 * 256; + return off; + }; + u = take((size_t)H * NCC * V * 2); + w = take((size_t)H * NCC * K * 2); + qd = take((size_t)H * NCC * K * 2); + kd = take((size_t)H * NCC * K * 2); + p = take((size_t)H * NC * C * C * 4); + pb = take((size_t)H * NC * C * C * 2); + decay = take((size_t)H * NC * 4); + cumulative = take((size_t)H * NCC * 4); + s = take((size_t)H * NC * K * V * 2); + vn = take((size_t)H * NCC * V * 2); + total = at; + } +}; + +// kernel 1 shared memory, bytes +constexpr int TBF = C * CP * 4 + 3 * 256 * 4; // offset of T as bf16 inside R2 +constexpr int R1 = 0; // kn packed TF32 [C]; later v as bf16 [C][HB] +constexpr int R2 = R1 + C * sizeof(Tf32Row); // qn packed TF32; later T + scratch, then T as bf16 +constexpr int R3 = R2 + TBF + C * TB * 2; // A float [C][CP]; later k exp(cum) as bf16 [C][HB] +constexpr int R4 = R3 + C * CP * 4; // cum, beta, exp(cum), exp(cum_last - cum), k norm inverse +constexpr size_t SMEM1_BYTES = R4 + 5 * C * 4; +static_assert(C * HB * 2 <= C * CP * 4, "k exp(cum) as bf16 fits over A"); +static_assert(C * sizeof(Tf32Row) <= R3 - R2, "packed qn fits in R2"); +static_assert(C * HB * 2 <= R2 - R1, "v as bf16 fits over kn"); + +__global__ void __launch_bounds__(THREADS) gdn_chunk_prep( + const __nv_bfloat16* __restrict__ q, const __nv_bfloat16* __restrict__ k, + const __nv_bfloat16* __restrict__ v, const float* __restrict__ g, + const __nv_bfloat16* __restrict__ beta, Work ws, int T, int H, int HK, float scale) { + extern __shared__ __align__(128) unsigned char sm[]; + Tf32Row* kn = reinterpret_cast(sm + R1); + Tf32Row* qn = reinterpret_cast(sm + R2); + float* tm = reinterpret_cast(sm + R2); + float* sc = tm + C * CP; + float* am = reinterpret_cast(sm + R3); + float* cum = reinterpret_cast(sm + R4); + float* bet = cum + C; + float* ecum = bet + C; + float* erem = ecum + C; + float* kinvs = erem + C; + __nv_bfloat16* vb = reinterpret_cast<__nv_bfloat16*>(sm + R1); + __nv_bfloat16* tb = reinterpret_cast<__nv_bfloat16*>(sm + R2 + TBF); + __nv_bfloat16* kw = reinterpret_cast<__nv_bfloat16*>(sm + R3); + + const int c = blockIdx.x, h = blockIdx.y, NC = gridDim.x, NCC = NC * C; + const int hk = h / (H / HK); + const int tid = threadIdx.x, lane = tid & 31, warp = tid >> 5; + + // 1. cumulative decays and learning rates (padding rows stay zero) + if (tid < C) { + const int t = c * C + tid; + cum[tid] = t < T ? g[(size_t)t * H + h] : 0.f; + bet[tid] = t < T ? __bfloat162float(beta[(size_t)t * H + h]) : 0.f; + } + __syncthreads(); + if(tid=step) value+=previous; + } + cum[tid]=value; + } + __syncthreads(); + if(tid>=32&&tid0;step/=2) { + ks+=__shfl_xor_sync(0xffffffff,ks,step,16); + qs+=__shfl_xor_sync(0xffffffff,qs,step,16); + } + const float kinv = reference_inverse_norm(ks + 1e-6f), qinv = reference_inverse_norm(qs + 1e-6f); + if (lane%16 == 0) kinvs[r] = kinv; +#pragma unroll + for (int e = 0; e < 8; e++) { + const int d = (lane%16)*8+e; + const float kx = __bfloat162float(__float2bfloat16(kv[e] * kinv)), qx = __bfloat162float(__float2bfloat16(qv[e] * qinv)); + reinterpret_cast(kn)[r*HB+d]=__float2bfloat16(kx); + reinterpret_cast(qn)[r*HB+d]=__float2bfloat16(qx); + const size_t row = ((size_t)h * NCC + c * C + r) * K + d; + ws.qd[row] = __float2bfloat16(qx); + ws.kd[row] = __float2bfloat16(kx); + if (d == 0) ws.cumulative[(size_t)h * NCC + c * C + r] = cum[r]; + } + } + __syncthreads(); + + // 3. pair products on tensor cores: the 10 lower 16x16 tiles of k.k (into A) and of + // q.k (into P, in global memory), then the masks and decays elementwise + float* pout = ws.p + ((size_t)h * NC + c) * C * C; + for (int e = warp; e < 20; e += THREADS / 32) { + const int tile = e % 10; + const int it = tile < 1 ? 0 : tile < 3 ? 1 : tile < 6 ? 2 : 3; + const int jt = tile - it * (it + 1) / 2; + const bf16* asrc = reinterpret_cast(e < 10 ? kn : qn); + const int group = lane / 4, thread = lane % 4; + float acc[2][4] = {}; +#pragma unroll + for (int k0=0;k0(kn),HB,k0,jt*16,lane); + cs1::mma16816(acc[0],a,b[0],b[1]); + cs1::mma16816(acc[1],a,b[2],b[3]); + } +#pragma unroll + for (int r = 0; r < 2; r++) { + const int row = it * 16 + group + r * 8; +#pragma unroll + for (int nt = 0; nt < 2; nt++) { + const int col = jt * 16 + 2 * thread + nt * 8; + float* dst = (e < 10 ? am + row * CP : pout + row * C) + col; + *reinterpret_cast(dst) = make_float2(acc[nt][2 * r], acc[nt][2 * r + 1]); + } + } + } + __syncthreads(); + __nv_bfloat16* pb = ws.pb + ((size_t)h * NC + c) * C * C; + for (int x = tid; x < C * C; x += THREADS) { + const int i = x / C, j = x % C; + const float dec = j <= i ? fla_exp2(cum[i] - cum[j]) : 0.f; + am[i * CP + j] = j < i ? __fmul_rn(__fmul_rn(am[i * CP + j], dec), bet[i]) : 0.f; + pb[i * C + j] = __float2bfloat16(j <= i ? pout[i * C + j] * dec : 0.f); + } + __syncthreads(); + + // 4. T = (I + A)^-1 in 16x16 blocks. The first reduction pair is an FMA. + for (int x = tid; x < C * CP; x += THREADS) tm[x] = 0.f; + __syncthreads(); + if (tid < C) { + const int base = 16 * (tid / 16), col = tid % 16; + float xs[16] = {}; + for(int r=1;r<16;r++) { + float terms[16]; +#pragma unroll + for(int j=0;j<8;j++) { + float low=j0;step/=2) { +#pragma unroll + for(int j=0;j(tm+16*bi*CP+16*bi,CP,am+16*bi*CP+16*bj,CP,acc); + else for(int kb=bj;kb(am+16*bi*CP+16*kb,CP,tm+16*kb*CP+16*bj,CP,acc); + } + int row=lane/4,col=2*(lane%4); +#pragma unroll + for(int n=0;n<2;n++) for(int e=0;e<4;e++) sc[warp*256+(row+(e/2)*8)*16+col+n*8+e%2]=acc[n][e]; + } + __syncthreads(); + if(warp(sc+warp*256,16,tm+16*bj*CP+16*bj,CP,acc); + else reference_dot16(tm+16*bi*CP+16*bi,CP,sc+warp*256,16,acc); + int row=lane/4,col=2*(lane%4); +#pragma unroll + for(int n=0;n<2;n++) for(int e=0;e<4;e++) tm[(bi*16+row+(e/2)*8)*CP+bj*16+col+n*8+e%2]=-acc[n][e]; + } + __syncthreads(); + } + + // 5. bfloat16 operands: k exp(cum) over A, T beta after T, then v over kn; + // u = T (beta v) and w = T (beta exp(cum) k) on tensor cores, stored as bfloat16 + for (int x = tid; x < C * K; x += THREADS) { + const int j = x / K, d = x % K; + const int t = c * C + j; + const float kx = t < T ? __bfloat162float(k[((size_t)t * HK + hk) * K + d]) : 0.f; + const float normalized = __bfloat162float(__float2bfloat16(kx * kinvs[j])); + kw[j * HB + d] = __float2bfloat16(__bfloat162float(__float2bfloat16(normalized * bet[j])) * ecum[j]); + } + for (int x = tid; x < C * C; x += THREADS) { + const int i = x / C, j = x % C; + tb[i * TB + j] = __float2bfloat16(tm[i * CP + j]); + + } + __syncthreads(); + for (int x = tid; x < C * V; x += THREADS) { + const int j = x / V, d = x % V; + const int t = c * C + j; + vb[j * HB + d] = t < T ? __float2bfloat16(__bfloat162float(v[((size_t)t * H + h) * V + d]) * bet[j]) : __float2bfloat16(0.f); + } + __syncthreads(); + for (int e = warp; e < 64; e += THREADS / 32) { + const bool is_u = e < 32; + const int it = (e % 32) / 8, dt = e % 8; + const __nv_bfloat16* bsrc = is_u ? vb : kw; + float acc[2][4] = {}; + for (int kb = 0; kb <= it; kb++) { // T is lower triangular + uint32_t a[4], b[4]; + cs1::load_a(a, tb, TB, it * 16, kb * 16, lane); + cs1::load_b_kn(b, bsrc, HB, kb * 16, dt * 16, lane); + cs1::mma16816(acc[0], a, b[0], b[1]); + cs1::mma16816(acc[1], a, b[2], b[3]); + } +#pragma unroll + for (int r = 0; r < 2; r++) { + const int row = it * 16 + lane / 4 + r * 8; + bf16* dst = (is_u ? ws.u : ws.w) + ((size_t)h * NCC + c * C + row) * K + dt * 16 + 2 * (lane % 4); +#pragma unroll + for (int nt = 0; nt < 2; nt++) + *reinterpret_cast(dst + nt * 8) = cs1::pack_bf16(acc[nt][2 * r], acc[nt][2 * r + 1]); + } + } +} + +// ---- kernel 2: the state, chunk by chunk ---- + +constexpr int BVS = 32; // value columns per block +constexpr int ST_THREADS = 128; +constexpr int WS_LD = K + 8; // bfloat16 row stride of staged w and kd +constexpr int SS_LD = BVS + 8; // bfloat16 row stride of the S copy and v_new +constexpr int STAGE = C * WS_LD; // elements of one staged w or kd +constexpr size_t SMEM2_BYTES = (4 * STAGE + K * SS_LD + C * SS_LD) * 2; + +__global__ void __launch_bounds__(ST_THREADS) gdn_chunk_state(Work ws, int NC, const float* __restrict__ S_IN, + float* __restrict__ S_OUT) { + extern __shared__ __align__(128) unsigned char sm[]; + bf16* wbuf = reinterpret_cast(sm); // [2][C][WS_LD] + bf16* kbuf = wbuf + 2 * STAGE; // [2][C][WS_LD] + bf16* scopy = kbuf + 2 * STAGE; // [K][SS_LD] + bf16* vnew = scopy + K * SS_LD; // [C][SS_LD] + const int h = blockIdx.x, vb0 = blockIdx.y * BVS, NCC = NC * C; + const int tid = threadIdx.x, warp = tid / 32, lane = tid % 32, g = lane / 4, t = lane % 4; + + auto load = [&](int c, int buf) { + const size_t base = ((size_t)h * NCC + (size_t)c * C) * K; + for (int x = tid; x < C * K / 8; x += ST_THREADS) { + const int r = x / (K / 8), col = (x % (K / 8)) * 8; + cs1::cp_async16(wbuf + buf * STAGE + r * WS_LD + col, ws.w + base + (size_t)r * K + col); + cs1::cp_async16(kbuf + buf * STAGE + r * WS_LD + col, ws.kd + base + (size_t)r * K + col); + } + cs1::cp_async_commit(); + }; + + // S rows warp * 32 + mt * 16 + {g, g + 8}, columns nt * 8 + {2t, 2t + 1} + float st[2][4][4]; +#pragma unroll + for (int mt = 0; mt < 2; mt++) +#pragma unroll + for (int nt = 0; nt < 4; nt++) st[mt][nt][0] = st[mt][nt][1] = st[mt][nt][2] = st[mt][nt][3] = 0.f; + if (S_IN != nullptr) { + // float32 state seeded for a continuation: identical to the registers a + // full pass would carry into this chunk, so the scan repeats one pass. + const float* si = S_IN + (size_t)h * K * V; +#pragma unroll + for (int mt = 0; mt < 2; mt++) +#pragma unroll + for (int nt = 0; nt < 4; nt++) +#pragma unroll + for (int e = 0; e < 4; e++) { + const int row = warp * 32 + mt * 16 + g + (e >> 1) * 8; + const int col = vb0 + nt * 8 + 2 * t + (e & 1); + st[mt][nt][e] = si[(size_t)row * V + col]; + } + } + + load(0, 0); + for (int c = 0; c < NC; c++) { + const int buf = c & 1; + cs1::cp_async_wait<0>(); + __syncthreads(); // chunk c is staged, and chunk c - 1 is done with the other buffer + if (c + 1 < NC) load(c + 1, buf ^ 1); + // 1. S as bfloat16, to shared memory for w S and to global memory for the output + bf16* sg = ws.s + ((size_t)h * NC + c) * K * V + vb0; +#pragma unroll + for (int mt = 0; mt < 2; mt++) +#pragma unroll + for (int nt = 0; nt < 4; nt++) +#pragma unroll + for (int r = 0; r < 2; r++) { + const int row = warp * 32 + mt * 16 + g + r * 8, col = nt * 8 + 2 * t; + const uint32_t pk = cs1::pack_bf16(st[mt][nt][2 * r], st[mt][nt][2 * r + 1]); + *reinterpret_cast(scopy + row * SS_LD + col) = pk; + *reinterpret_cast(sg + (size_t)row * V + col) = pk; + } + __syncthreads(); + // 2. v_new = u - w S, rows warp * 16 .. + const bf16* wc = wbuf + buf * STAGE; + float acc[4][4]; +#pragma unroll + for (int nt = 0; nt < 4; nt++) acc[nt][0] = acc[nt][1] = acc[nt][2] = acc[nt][3] = 0.f; +#pragma unroll + for (int kk = 0; kk < K; kk += 16) { + uint32_t a[4]; + cs1::load_a(a, wc, WS_LD, warp * 16, kk, lane); +#pragma unroll + for (int nt = 0; nt < 4; nt += 2) { + uint32_t b[4]; + cs1::load_b_kn(b, scopy, SS_LD, kk, nt * 8, lane); + cs1::mma16816(acc[nt], a, b[0], b[1]); + cs1::mma16816(acc[nt + 1], a, b[2], b[3]); + } + } + const size_t row0 = (size_t)h * NCC + (size_t)c * C + warp * 16; +#pragma unroll + for (int nt = 0; nt < 4; nt++) +#pragma unroll + for (int r = 0; r < 2; r++) { + const int row = g + r * 8, col = nt * 8 + 2 * t; + const __nv_bfloat162 uv = + *reinterpret_cast(ws.u + (row0 + row) * V + vb0 + col); + const float x = __low2float(uv) - acc[nt][2 * r], y = __high2float(uv) - acc[nt][2 * r + 1]; + const float erem = fla_exp2(ws.cumulative[(size_t)h*NCC + c*C + C-1] - ws.cumulative[row0 + row]); + const uint32_t pk = cs1::pack_bf16(x, y); + *reinterpret_cast(vnew + (warp * 16 + row) * SS_LD + col) = cs1::pack_bf16(x * erem, y * erem); + *reinterpret_cast(ws.vn + (row0 + row) * V + vb0 + col) = pk; + } + __syncthreads(); + // 3. S = decay S + kd^T v_new, rows warp * 32 .. + const float dec = ws.decay[h * NC + c]; +#pragma unroll + for (int mt = 0; mt < 2; mt++) +#pragma unroll + for (int nt = 0; nt < 4; nt++) +#pragma unroll + for (int e = 0; e < 4; e++) st[mt][nt][e] *= dec; + const bf16* kc = kbuf + buf * STAGE; +#pragma unroll + for (int kk = 0; kk < C; kk += 16) { +#pragma unroll + for (int mt = 0; mt < 2; mt++) { + uint32_t a[4]; + cs1::load_a_trans(a, kc, WS_LD, warp * 32 + mt * 16, kk, lane); +#pragma unroll + for (int nt = 0; nt < 4; nt += 2) { + uint32_t b[4]; + cs1::load_b_kn(b, vnew, SS_LD, kk, nt * 8, lane); + cs1::mma16816(st[mt][nt], a, b[0], b[1]); + cs1::mma16816(st[mt][nt + 1], a, b[2], b[3]); + } + } + } + } + if (S_OUT != nullptr) { + float* so = S_OUT + (size_t)h * K * V; +#pragma unroll + for (int mt = 0; mt < 2; mt++) +#pragma unroll + for (int nt = 0; nt < 4; nt++) +#pragma unroll + for (int r = 0; r < 2; r++) { + const int row = warp * 32 + mt * 16 + g + r * 8; + const int col = vb0 + nt * 8 + 2 * t; + *reinterpret_cast(so + (size_t)row * V + col) = + make_float2(st[mt][nt][2 * r], st[mt][nt][2 * r + 1]); + } + } +} + +// ---- kernel 3: the output, per chunk ---- + +constexpr int OUT_THREADS = 128; +constexpr int O_LD = K + 8; // bfloat16 row stride of qd, S and v_new (all 128 wide) +constexpr int P_LD = C + 8; // bfloat16 row stride of P +constexpr size_t SMEM3_BYTES = (C * O_LD + K * O_LD + C * P_LD + C * O_LD) * 2; +static_assert(K == V, "S, qd and v_new share a row stride"); + +__global__ void __launch_bounds__(OUT_THREADS) gdn_chunk_out(Work ws, bf16* __restrict__ o, int T, int H, float scale) { + extern __shared__ __align__(128) unsigned char sm[]; + bf16* qs = reinterpret_cast(sm); // [C][O_LD] + bf16* ss = qs + C * O_LD; // [K][O_LD] + bf16* ps = ss + K * O_LD; // [C][P_LD] + bf16* vs = ps + C * P_LD; // [C][O_LD] + const int c = blockIdx.x, h = blockIdx.y, NC = gridDim.x, NCC = NC * C; + const int tid = threadIdx.x, warp = tid / 32, lane = tid % 32, g = lane / 4, t = lane % 4; + const size_t rows = (size_t)h * NCC + (size_t)c * C; + for (int x = tid; x < C * K / 8; x += OUT_THREADS) { + const int r = x / (K / 8), col = (x % (K / 8)) * 8; + cs1::cp_async16(qs + r * O_LD + col, ws.qd + (rows + r) * K + col); + cs1::cp_async16(vs + r * O_LD + col, ws.vn + (rows + r) * V + col); + } + for (int x = tid; x < K * V / 8; x += OUT_THREADS) { + const int r = x / (V / 8), col = (x % (V / 8)) * 8; + cs1::cp_async16(ss + r * O_LD + col, ws.s + ((size_t)h * NC + c) * K * V + (size_t)r * V + col); + } + for (int x = tid; x < C * C / 8; x += OUT_THREADS) { + const int r = x / (C / 8), col = (x % (C / 8)) * 8; + cs1::cp_async16(ps + r * P_LD + col, ws.pb + ((size_t)h * NC + c) * C * C + r * C + col); + } + cs1::cp_async_commit(); + cs1::cp_async_wait<0>(); + __syncthreads(); + + float acc[V / 8][4]; +#pragma unroll + for (int nt = 0; nt < V / 8; nt++) acc[nt][0] = acc[nt][1] = acc[nt][2] = acc[nt][3] = 0.f; + // qd S +#pragma unroll 2 + for (int kk = 0; kk < K; kk += 16) { + uint32_t a[4]; + cs1::load_a(a, qs, O_LD, warp * 16, kk, lane); +#pragma unroll + for (int nt = 0; nt < V / 8; nt += 2) { + uint32_t b[4]; + cs1::load_b_kn(b, ss, O_LD, kk, nt * 8, lane); + cs1::mma16816(acc[nt], a, b[0], b[1]); + cs1::mma16816(acc[nt + 1], a, b[2], b[3]); + } + } + float intra[V / 8][4] = {}; + // P v_new; P is lower triangular, so these rows need positions below (warp + 1) * 16 + for (int kk = 0; kk < (warp + 1) * 16; kk += 16) { + uint32_t a[4]; + cs1::load_a(a, ps, P_LD, warp * 16, kk, lane); +#pragma unroll + for (int nt = 0; nt < V / 8; nt += 2) { + uint32_t b[4]; + cs1::load_b_kn(b, vs, O_LD, kk, nt * 8, lane); + cs1::mma16816(intra[nt], a, b[0], b[1]); + cs1::mma16816(intra[nt + 1], a, b[2], b[3]); + } + } +#pragma unroll + for (int r = 0; r < 2; r++) { + const int tok = c * C + warp * 16 + g + r * 8; + if (tok >= T) continue; + bf16* dst = o + ((size_t)tok * H + h) * V + 2 * t; +#pragma unroll + for (int nt = 0; nt < V / 8; nt++) + *reinterpret_cast(dst + nt * 8) = cs1::pack_bf16(fmaf(intra[nt][2*r], scale, __fmul_rn(__fmul_rn(acc[nt][2*r], fla_exp2(ws.cumulative[rows + warp*16+g+r*8])), scale)), fmaf(intra[nt][2*r+1], scale, __fmul_rn(__fmul_rn(acc[nt][2*r+1], fla_exp2(ws.cumulative[rows + warp*16+g+r*8])), scale))); + } +} + +Work split(float* workspace, int T, int H) { + const Layout l(T, H); + unsigned char* b = reinterpret_cast(workspace); + Work ws; + ws.u = reinterpret_cast(b + l.u); + ws.w = reinterpret_cast(b + l.w); + ws.qd = reinterpret_cast(b + l.qd); + ws.kd = reinterpret_cast(b + l.kd); + ws.p = reinterpret_cast(b + l.p); + ws.pb = reinterpret_cast(b + l.pb); + ws.decay = reinterpret_cast(b + l.decay); + ws.cumulative = reinterpret_cast(b + l.cumulative); + ws.s = reinterpret_cast(b + l.s); + ws.vn = reinterpret_cast(b + l.vn); + return ws; +} + +} // namespace + +extern "C" { + +size_t cs1_reference_gdn_workspace_floats(int T, int H) { return (Layout(T, H).total + 3) / 4; } + +static int gdn_prefill_run(const void* q, const void* k, const void* v, const float* g, + const void* beta, void* o, float* workspace, int T, int H, int HK, + float scale, const void* s_in, void* s_out, void* stream) { + if (T < 0 || HK <= 0 || H % HK != 0) return cudaErrorInvalidValue; + const float* si = static_cast(s_in); + float* so = static_cast(s_out); + if (T == 0) { + // No tokens: the state passes through unchanged, for symmetric capture. + if (so == nullptr) return cudaSuccess; + const size_t bytes = (size_t)H * K * V * sizeof(float); + cudaStream_t st = static_cast(stream); + return si != nullptr ? cudaMemcpyAsync(so, si, bytes, cudaMemcpyDeviceToDevice, st) + : cudaMemsetAsync(so, 0, bytes, st); + } + // once per process (for the device current at the first call) + static const cudaError_t configured = [] { + cudaError_t e = cudaFuncSetAttribute(gdn_chunk_prep, cudaFuncAttributeMaxDynamicSharedMemorySize, + (int)SMEM1_BYTES); + if (e == cudaSuccess) + e = cudaFuncSetAttribute(gdn_chunk_state, cudaFuncAttributeMaxDynamicSharedMemorySize, + (int)SMEM2_BYTES); + if (e == cudaSuccess) + e = cudaFuncSetAttribute(gdn_chunk_out, cudaFuncAttributeMaxDynamicSharedMemorySize, + (int)SMEM3_BYTES); + return e; + }(); + if (configured != cudaSuccess) return configured; + const int NC = (T + C - 1) / C; + const Work ws = split(workspace, T, H); + cudaStream_t st = static_cast(stream); + gdn_chunk_prep<<>>( + static_cast(q), static_cast(k), + static_cast(v), g, static_cast(beta), ws, T, H, HK, scale); + gdn_chunk_state<<>>(ws, NC, si, so); + gdn_chunk_out<<>>(ws, static_cast(o), T, H, scale); + return cudaGetLastError(); +} + +int cs1_reference_gdn_prefill(const void* q, const void* k, const void* v, const float* g, const void* beta, + void* o, float* workspace, int T, int H, int HK, float scale, void* stream) { + return gdn_prefill_run(q, k, v, g, beta, o, workspace, T, H, HK, scale, nullptr, nullptr, stream); +} + +int cs1_reference_gdn_prefill_x(const void* q, const void* k, const void* v, const float* g, const void* beta, + void* o, float* workspace, int T, int H, int HK, float scale, + const void* s_in, void* s_out, void* stream) { + return gdn_prefill_run(q, k, v, g, beta, o, workspace, T, H, HK, scale, s_in, s_out, stream); +} + +} // extern "C" diff --git a/src/backends/cuda/qwen3_5/reference_language_attention.cu b/src/backends/cuda/qwen3_5/reference_language_attention.cu new file mode 100644 index 00000000..48a0fceb --- /dev/null +++ b/src/backends/cuda/qwen3_5/reference_language_attention.cu @@ -0,0 +1,192 @@ +#include "reference_attention.cuh" +#include "common.cuh" +#include "mma.cuh" +namespace cs1 { namespace language_reference { +constexpr int BM = 64, THREADS = 128; +template constexpr int SMEM_BYTES = (BM + 2 * BN) * (D + 8) * 2; + +template +__global__ void __launch_bounds__(THREADS) + flash_kernel(const bf16* __restrict__ q, const bf16* __restrict__ k, const bf16* __restrict__ v, int ldv, + float* __restrict__ out, float* __restrict__ lse, int T, int Hq, int Hk, float scale, int splits) { + constexpr int LDS = D + 8; + const float scale_log2=(float)((double)scale*1.4426950408889634); + extern __shared__ __align__(16) unsigned char smem[]; + bf16* qs = reinterpret_cast(smem); + bf16* ks = qs + BM * LDS; + bf16* vs = ks + BN * LDS; + const int h = blockIdx.y, hk = h / (Hq / Hk); + const int q0 = (gridDim.x - 1 - blockIdx.x) * BM; // the longest blocks first + const int tid = threadIdx.x, warp = tid / 32, lane = tid % 32; + const int g = lane / 4, t = lane % 4; + const int row0 = q0 + warp * 16; // this warp's first query + + for (int c = tid; c < BM * (D / 8); c += THREADS) { + const int r = c / (D / 8), col = (c % (D / 8)) * 8, row = q0 + r; + cp_async16(qs + r * LDS + col, q + ((size_t)min(row, T - 1) * Hq + h) * D + col, row < T); + } + cp_async_commit(); + + float o[D / 8][4]; +#pragma unroll + for (int n = 0; n < D / 8; n++) o[n][0] = o[n][1] = o[n][2] = o[n][3] = 0.f; + float m[2] = {-INFINITY, -INFINITY}, l[2] = {0.f, 0.f}; + + int per_split=((T+BN-1)/BN+splits-1)/splits; + int first=blockIdx.z*per_split*BN; + int last=min(min((T+BN-1)/BN*BN,q0+BM),first+per_split*BN); + for (int k0=last-BN;k0>=first;k0-=BN) { + for (int c = tid; c < BN * (D / 8); c += THREADS) { + const int r = c / (D / 8), col = (c % (D / 8)) * 8, s = k0 + r; + cp_async16(ks + r * LDS + col, k + ((size_t)min(s, T - 1) * Hk + hk) * D + col, s < T); + } + cp_async_commit(); + for (int c = tid; c < BN * (D / 8); c += THREADS) { + const int r = c / (D / 8), col = (c % (D / 8)) * 8, s = k0 + r; + cp_async16(vs + r * LDS + col, v + (size_t)min(s, T - 1) * ldv + (size_t)hk * D + col, s < T); + } + cp_async_commit(); + cp_async_wait<1>(); // Q and K + __syncthreads(); + + // keys past every query of this warp contribute nothing + const bool active = k0 <= row0+15; + float sc[BN / 8][4]; +#pragma unroll + for (int n = 0; n < BN / 8; n++) sc[n][0] = sc[n][1] = sc[n][2] = sc[n][3] = 0.f; + if (active) { +#pragma unroll + for (int kk = 0; kk < D; kk += 16) { + uint32_t a[4]; + load_a(a, qs, LDS, warp * 16, kk, lane); +#pragma unroll + for (int n = 0; n < BN / 8; n += 2) { + uint32_t b[4]; + load_b_nk(b, ks, LDS, kk, n * 8, lane); + mma16816(sc[n], a, b[0], b[1]); + mma16816(sc[n + 1], a, b[2], b[3]); + } + } + } + uint32_t p[BN / 16][4]; + if (active) { + // causal and length mask, then the online softmax in base 2 + float mx[2] = {-INFINITY, -INFINITY}; +#pragma unroll + for (int n = 0; n < BN / 8; n++) { +#pragma unroll + for (int e = 0; e < 4; e++) { + const int key = k0 + n * 8 + 2 * t + (e & 1); + sc[n][e] = (key < T && key <= row0+g+(e>>1)*8) ? sc[n][e] : -INFINITY; + mx[e >> 1] = fmaxf(mx[e >> 1], sc[n][e]); + } + } + float alpha[2], base[2]; +#pragma unroll + for (int r = 0; r < 2; r++) { + mx[r] = fmaxf(mx[r], __shfl_xor_sync(0xffffffffu, mx[r], 1)); + mx[r] = fmaxf(mx[r], __shfl_xor_sync(0xffffffffu, mx[r], 2)); + const float mn = fmaxf(m[r], mx[r]); + base[r] = mn == -INFINITY ? 0.f : mn; + alpha[r] = exp2f((m[r] - base[r])*scale_log2); + m[r] = mn; + l[r] *= alpha[r]; + } +#pragma unroll + for (int n = 0; n < BN / 8; n++) { +#pragma unroll + for (int e = 0; e < 4; e++) { + sc[n][e] = exp2f(__fmul_rn(sc[n][e],scale_log2)-__fmul_rn(base[e >> 1],scale_log2)); + l[e >> 1] += sc[n][e]; + } + } +#pragma unroll + for (int n = 0; n < D / 8; n++) { + o[n][0] *= alpha[0]; + o[n][1] *= alpha[0]; + o[n][2] *= alpha[1]; + o[n][3] *= alpha[1]; + } + // the score accumulators, two 8-key tiles at a time, are the A fragments of P*V +#pragma unroll + for (int j = 0; j < BN / 16; j++) { + p[j][0] = pack_bf16(sc[2 * j][0], sc[2 * j][1]); + p[j][1] = pack_bf16(sc[2 * j][2], sc[2 * j][3]); + p[j][2] = pack_bf16(sc[2 * j + 1][0], sc[2 * j + 1][1]); + p[j][3] = pack_bf16(sc[2 * j + 1][2], sc[2 * j + 1][3]); + } + } + cp_async_wait<0>(); // V + __syncthreads(); + if (active) { +#pragma unroll + for (int j = 0; j < BN / 16; j++) { +#pragma unroll + for (int n = 0; n < D / 8; n += 2) { + uint32_t b[4]; + load_b_kn(b, vs, LDS, j * 16, n * 8, lane); + mma16816(o[n], p[j], b[0], b[1]); + mma16816(o[n + 1], p[j], b[2], b[3]); + } + } + } + __syncthreads(); // before the next tile overwrites K and V + } + + // the four lanes of a row each summed a quarter of its keys +#pragma unroll + for (int r = 0; r < 2; r++) { + l[r] += __shfl_xor_sync(0xffffffffu, l[r], 2); + l[r] += __shfl_xor_sync(0xffffffffu, l[r], 1); + } + const float inv[2] = {l[0]==0.f ? 1.f : 1.f/l[0], l[1]==0.f ? 1.f : 1.f/l[1]}; +#pragma unroll + for (int r = 0; r < 2; r++) { + const int row = row0 + g + r * 8; + if (row >= T) continue; + float* dst=out+(((size_t)blockIdx.z*T+row)*Hq+h)*D+2*t; + if(t==0) lse[((size_t)blockIdx.z*T+row)*Hq+h]=l[r]==0.f ? -INFINITY : fmaf(m[r],scale,__logf(l[r])); +#pragma unroll + for(int n=0;n=total) return; + if(splits==1) { float result=partial[index];if(gate) result=round_bf16(result)*round_bf16(sigmoid(f32(gate[index])));out[index]=to_bf16(result);return; } + size_t row=index/D,rows=(size_t)T*H; + float maximum=-INFINITY,values[128];int power=1;while(power0;step/=2) for(int i=0;i65536 || H<=0 || !workspace || sms<=0) return cudaErrorInvalidValue; + int splits=reference::split_count(T,H,64,sms); + float* memory=(float*)workspace;size_t count=(size_t)splits*T*H*256; + cudaError_t status=cudaSuccess; + float* lse=memory+count; + cudaFuncSetAttribute(f::flash_kernel<256,64>,cudaFuncAttributeMaxDynamicSharedMemorySize,f::SMEM_BYTES<256,64>); + f::flash_kernel<256,64><<,(cudaStream_t)stream>>>((const bf16*)q,(const bf16*)k,(const bf16*)v,ldv,memory,lse,T,H,HK,scale,splits); + status=cudaGetLastError(); + if(status==cudaSuccess) { + f::combine<<<((size_t)T*H*256+127)/128,128,0,(cudaStream_t)stream>>>(memory,lse,(bf16*)out,(const bf16*)gate,T,H,256,splits); + status=cudaGetLastError(); + } + return status; +} + +extern "C" size_t cs1_reference_language_attention_workspace_floats(int capacity,int heads) { return cs1::reference::workspace_floats(capacity,heads,256,64); } diff --git a/src/backends/cuda/qwen3_5/reference_norm.cu b/src/backends/cuda/qwen3_5/reference_norm.cu new file mode 100644 index 00000000..61f7cdeb --- /dev/null +++ b/src/backends/cuda/qwen3_5/reference_norm.cu @@ -0,0 +1,34 @@ +#include "common.cuh" +using namespace cs1; +template +__global__ void reference_norm(const bf16* x,const bf16* delta,const bf16* w,bf16* out,int D,float eps) { + __shared__ float shared[512]; + int lane=threadIdx.x;size_t row=blockIdx.x;x+=row*D;if(delta)delta+=row*D;out+=row*D;float sum[4]={}; + for(int base=lane*4;base(x)[base+j]=to_bf16(v);}sum[j]=__fadd_rn(sum[j],__fmul_rn(v,v));} + } + float value=__fadd_rn(__fadd_rn(__fadd_rn(sum[0],sum[1]),sum[2]),sum[3]); + shared[lane]=value; + for(int step=blockDim.x/2;step>=32;step/=2) {__syncthreads();if(lane0;step/=2)value=__fadd_rn(value,__shfl_down_sync(0xffffffff,value,step));if(lane==0)shared[0]=rsqrtf(__fadd_rn(__fmul_rn(value,1.f/D),eps));} + __syncthreads();float inverse=shared[0]; + for(int i=lane;i=min(height*16,256))threads*=height; + if(delta)reference_norm<<>>((const bf16*)x,(const bf16*)delta,(const bf16*)w,(bf16*)out,D,eps); + else reference_norm<<>>((const bf16*)x,nullptr,(const bf16*)w,(bf16*)out,D,eps); + return cudaGetLastError(); +} + +extern "C" int cs1_reference_rms_norm(const void* x,const void* w,void* out,int rows,int d,float eps,void* stream) { + if(rows<=0 || d!=5120) return cudaErrorInvalidValue; + return reference_norm_run(x,nullptr,w,out,rows,d,eps,stream); +} +extern "C" int cs1_reference_add_rms_norm(void* x,const void* delta,const void* w,void* out,int rows,int d,float eps,void* stream) { + if(rows<=0 || d!=5120) return cudaErrorInvalidValue; + return reference_norm_run(x,delta,w,out,rows,d,eps,stream); +} diff --git a/src/backends/cuda/qwen3_5/reference_patch.cu b/src/backends/cuda/qwen3_5/reference_patch.cu new file mode 100644 index 00000000..62c071cc --- /dev/null +++ b/src/backends/cuda/qwen3_5/reference_patch.cu @@ -0,0 +1,83 @@ +#include +// A shape-owned cuDNN plan: allocation and descriptor setup occur before graph capture. +#include "ops.h" +#include +#ifdef CS1_REFERENCE_CUDNN +#include +namespace { +struct PatchPlan { + cudnnHandle_t handle=nullptr; + cudnnTensorDescriptor_t input=nullptr,output=nullptr; + cudnnFilterDescriptor_t filter=nullptr; + cudnnConvolutionDescriptor_t conv=nullptr; + void* workspace=nullptr; + size_t bytes=0; + ~PatchPlan() { + if(workspace) cudaFree(workspace); + if(conv) cudnnDestroyConvolutionDescriptor(conv); + if(filter) cudnnDestroyFilterDescriptor(filter); + if(output) cudnnDestroyTensorDescriptor(output); + if(input) cudnnDestroyTensorDescriptor(input); + if(handle) cudnnDestroy(handle); + } +}; +} +#endif +extern "C" int cs1_reference_available() { +#ifdef CS1_REFERENCE_CUDNN + return 1; +#else + return 0; +#endif +} +extern "C" void* cs1_reference_patch_create(int rows,void* stream) { +#ifdef CS1_REFERENCE_CUDNN + if(rows<=0 || rows>65536) return nullptr; + PatchPlan* p=new(std::nothrow) PatchPlan; + if(!p) return nullptr; + bool ready=false; + #define CHECK_CUDNN(expression) if((expression)!=CUDNN_STATUS_SUCCESS) break + do { + CHECK_CUDNN(cudnnCreate(&p->handle)); + CHECK_CUDNN(cudnnSetStream(p->handle,(cudaStream_t)stream)); + CHECK_CUDNN(cudnnCreateTensorDescriptor(&p->input)); + CHECK_CUDNN(cudnnCreateTensorDescriptor(&p->output)); + CHECK_CUDNN(cudnnCreateFilterDescriptor(&p->filter)); + CHECK_CUDNN(cudnnCreateConvolutionDescriptor(&p->conv)); + int xs[]={rows,3,2,16,16},xt[]={1536,512,256,16,1}; + int ys[]={rows,1152,1,1,1},yt[]={1152,1,1,1,1}; + int ws[]={1152,3,2,16,16},pad[]={0,0,0},step[]={2,16,16},dilation[]={1,1,1}; + CHECK_CUDNN(cudnnSetTensorNdDescriptor(p->input,CUDNN_DATA_BFLOAT16,5,xs,xt)); + CHECK_CUDNN(cudnnSetTensorNdDescriptor(p->output,CUDNN_DATA_BFLOAT16,5,ys,yt)); + CHECK_CUDNN(cudnnSetFilterNdDescriptor(p->filter,CUDNN_DATA_BFLOAT16,CUDNN_TENSOR_NCHW,5,ws)); + CHECK_CUDNN(cudnnSetConvolutionNdDescriptor(p->conv,3,pad,step,dilation,CUDNN_CROSS_CORRELATION,CUDNN_DATA_FLOAT)); + CHECK_CUDNN(cudnnSetConvolutionMathType(p->conv,CUDNN_TENSOR_OP_MATH)); + CHECK_CUDNN(cudnnGetConvolutionForwardWorkspaceSize(p->handle,p->input,p->filter,p->conv,p->output,CUDNN_CONVOLUTION_FWD_ALGO_IMPLICIT_PRECOMP_GEMM,&p->bytes)); + if(p->bytes && cudaMalloc(&p->workspace,p->bytes)!=cudaSuccess) break; + ready=true; + } while(false); + #undef CHECK_CUDNN + if(!ready) {delete p;return nullptr;} + return p; +#else + (void)rows;(void)stream;return nullptr; +#endif +} +extern "C" int cs1_reference_patch(void* plan,const void* x,const void* w,void* y) { +#ifdef CS1_REFERENCE_CUDNN + auto* p=static_cast(plan); + if(!p || !x || !w || !y) return cudaErrorInvalidValue; + const float alpha=1.f,beta=0.f; + auto status=cudnnConvolutionForward(p->handle,&alpha,p->input,x,p->filter,w,p->conv,CUDNN_CONVOLUTION_FWD_ALGO_IMPLICIT_PRECOMP_GEMM,p->workspace,p->bytes,&beta,p->output,y); + return status==CUDNN_STATUS_SUCCESS ? 0 : 20000+(int)status; +#else + (void)plan;(void)x;(void)w;(void)y;return cudaErrorNotSupported; +#endif +} +extern "C" void cs1_reference_patch_destroy(void* plan) { +#ifdef CS1_REFERENCE_CUDNN + delete static_cast(plan); +#else + (void)plan; +#endif +} diff --git a/src/backends/cuda/qwen3_5/reference_prep.cu b/src/backends/cuda/qwen3_5/reference_prep.cu new file mode 100644 index 00000000..138d99a9 --- /dev/null +++ b/src/backends/cuda/qwen3_5/reference_prep.cu @@ -0,0 +1,83 @@ +#include "common.cuh" +namespace cs1 { namespace reference_prep { +constexpr int DH = 256; // head dim +constexpr int PER = DH / 32; // values per lane + +// One warp per (token, head), q heads first, then k heads. Each lane holds 8 +// consecutive dims, so the rotary partner of dim i < 32 (dim i + 32) sits in lane ^ 4. +__global__ void attn_prep_kernel(const bf16* __restrict__ qg, const bf16* __restrict__ kr, int ld, + const bf16* __restrict__ qw, const bf16* __restrict__ kw, + const bf16* __restrict__ cos, const bf16* __restrict__ sin, + bf16* __restrict__ q, bf16* __restrict__ gate, bf16* __restrict__ k, int T, int Hq, + int Hk, int half, float eps) { + const int warp = blockIdx.x * (blockDim.x / 32) + threadIdx.x / 32, lane = threadIdx.x & 31; + const int heads = Hq + Hk; + if (warp >= T * heads) return; + const int t = warp / heads, hh = warp % heads; + const bool is_q = hh < Hq; + const int h = is_q ? hh : hh - Hq; + const bf16* src = is_q ? qg + (size_t)t * ld + (size_t)h * 2 * DH : kr + (size_t)t * ld + (size_t)h * DH; + const bf16* w = is_q ? qw : kw; + const int d0 = lane * PER; + + float x[PER]; + load8(src + d0, x); + float sums[4]={}; +#pragma unroll + for(int j=0;j<4;j++) { + float a=f32(src[lane*4+j]),b=f32(src[128+lane*4+j]); + sums[j]=__fadd_rn(__fmul_rn(a,a),__fmul_rn(b,b)); + } + float ss=__fadd_rn(__fadd_rn(__fadd_rn(sums[0],sums[1]),sums[2]),sums[3]); + for(int step=16;step>0;step/=2)ss=__fadd_rn(ss,__shfl_down_sync(0xffffffff,ss,step)); + ss=__shfl_sync(0xffffffff,ss,0); + const float inv=rsqrtf(__fadd_rn(__fmul_rn(ss,1.f/DH),eps)); + float wv[PER]; + load8(w + d0, wv); +#pragma unroll + for (int i = 0; i < PER; i++) x[i] = round_bf16(x[i] * inv * (1.f + wv[i])); + + // rotate_half on the first 2 * half dims: out = x * cos + rotate_half(x) * sin, + // each product and the sum rounded to bfloat16 as in the reference. + const int rot = 2 * half; + float y[PER]; +#pragma unroll + for (int i = 0; i < PER; i++) { + const float partner = __shfl_xor_sync(0xffffffffu, x[i], (half / PER)); + const int d = d0 + i; + y[i] = x[i]; + if (d < rot) { + const int fi = d % half; + const float c = f32(cos[(size_t)t * half + fi]), s = f32(sin[(size_t)t * half + fi]); + const float r = d < half ? -partner : partner; + y[i] = round_bf16(round_bf16(x[i] * c) + round_bf16(r * s)); + } + } + if (is_q) { + store8(q + ((size_t)t * Hq + h) * DH + d0, y); + float gv[PER]; + load8(src + DH + d0, gv); + store8(gate + ((size_t)t * Hq + h) * DH + d0, gv); + } else { + store8(k + ((size_t)t * Hk + h) * DH + d0, y); + } +} + +} } +using namespace cs1; +using namespace cs1::reference_prep; +extern "C" int cs1_reference_attn_prep(const void* qg, const void* kr, int ld, const void* qw, const void* kw, + const void* cos, const void* sin, void* q, void* gate, void* k, int T, int Hq, int Hk, + int Dh, int half, float eps, void* stream) { + if (ld % 8 != 0 || T < 0) return cudaErrorInvalidValue; + // the lane ^ (half / 8) partner exchange needs 2 * half <= 256 and half a multiple of 8 + if (Dh != DH || half % PER != 0 || 2 * half > DH || (half / PER) & ((half / PER) - 1)) return cudaErrorInvalidValue; + const int warps = T * (Hq + Hk); + if (warps == 0) return cudaSuccess; + constexpr int WARPS = 8; + attn_prep_kernel<<<(warps + WARPS - 1) / WARPS, WARPS * 32, 0, static_cast(stream)>>>( + static_cast(qg), static_cast(kr), ld, static_cast(qw), + static_cast(kw), static_cast(cos), static_cast(sin), + static_cast(q), static_cast(gate), static_cast(k), T, Hq, Hk, half, eps); + return cudaGetLastError(); +} diff --git a/src/backends/cuda/qwen3_5/reference_vision.cu b/src/backends/cuda/qwen3_5/reference_vision.cu new file mode 100644 index 00000000..17dc582d --- /dev/null +++ b/src/backends/cuda/qwen3_5/reference_vision.cu @@ -0,0 +1,100 @@ +#include "common.cuh" +#include "ops.h" +namespace cs1 { namespace reference_vision { +struct Moments { float m, v, n; }; +__device__ Moments join_moments(Moments current, Moments other) { + float count=current.n+other.n; + if (count==0.f) return {0.f,0.f,0.f}; + float inv=1.f/count; + float left=other.n*inv, right=current.n*inv; + float delta=current.m-other.m; + return {left*other.m+right*current.m, + other.v+current.v+delta*delta*other.n*right,count}; +} +__global__ void reference_norm_kernel(const bf16* x,const bf16* w,const bf16* b,bf16* y,int d) { + __shared__ Moments warps[4]; + int tid=threadIdx.x, lane=tid%32, warp=tid/32; + size_t base=(size_t)blockIdx.x*d; + Moments a{0.f,0.f,0.f}; + for (int group=tid;group0;step/=2) { + Moments other{__shfl_down_sync(0xffffffff,a.m,step),__shfl_down_sync(0xffffffff,a.v,step),__shfl_down_sync(0xffffffff,a.n,step)}; + a=join_moments(a,other); + } + if(lane==0) warps[warp]=a; + __syncthreads(); + for(int step=2;step>0;step/=2) { + if(lane==0&&warp=n) return; + const int token=i/hidden,d=i%dh,channel=i%hidden,half=dh/2; + const float angle=co[(size_t)token*half+d%half]; + const float c=cosf(angle),s=sinf(angle); + const int partner=channel+(d=size) return; + const int d=i%80,head=(i/80)%heads; + const size_t token=i/(heads*80),compact=(token*heads+head)*dh+d; + pq[i]=d>>((const bf16*)x,(const bf16*)w,(const bf16*)b,(bf16*)y,d); + return cudaGetLastError(); +} +extern "C" int cs1_reference_vision_rope(const void* qkv,const float* co,const float* si,void* q,void* k,int n,int hidden,int dh,void* stream) { + if(n<=0 || (hidden!=1152 || dh!=72)) return cudaErrorInvalidValue; + const size_t count=(size_t)n*hidden; + reference_vision::rope_v2_kernel<<<(count+255)/256,256,0,(cudaStream_t)stream>>>((const bf16*)qkv,co,si,(bf16*)q,(bf16*)k,count,hidden,dh); + return cudaGetLastError(); +} +extern "C" int cs1_reference_vision_attention(const void* q,const void* k,const void* v,void* out,int n,int heads,int dh,void* workspace,void* stream) { + if(n<=0 || heads!=16 || dh!=72) return cudaErrorInvalidValue; + if(!workspace) return cudaErrorInvalidValue; + const size_t count=(size_t)n*heads*80; + bf16* pq=(bf16*)workspace; bf16* pk=pq+count; bf16* pv=pk+count; bf16* po=pv+count; + reference_vision::pad_attention_kernel<<<(count+255)/256,256,0,(cudaStream_t)stream>>>((const bf16*)q,(const bf16*)k,(const bf16*)v,pq,pk,pv,count,heads,dh); + cudaError_t status=cudaGetLastError(); if(status!=cudaSuccess) return status; + status=(cudaError_t)cs1_reference_vision_attention_padded(pq,pk,pv,po,n,heads,po+count,stream); + if(status!=cudaSuccess) return status; + status=cudaGetLastError(); if(status!=cudaSuccess) return status; + reference_vision::unpack_attention_kernel<<<((size_t)n*heads*dh+255)/256,256,0,(cudaStream_t)stream>>>(po,(bf16*)out,(size_t)n*heads*dh,dh); + return cudaGetLastError(); +} diff --git a/src/backends/cuda/qwen3_5/reference_vision_attention.cu b/src/backends/cuda/qwen3_5/reference_vision_attention.cu new file mode 100644 index 00000000..4c8202f5 --- /dev/null +++ b/src/backends/cuda/qwen3_5/reference_vision_attention.cu @@ -0,0 +1,196 @@ +#include "reference_attention.cuh" +#include "common.cuh" +#include "mma.cuh" +namespace cs1 { namespace vision_reference { +constexpr int BM = 64, THREADS = 128; +template constexpr int SMEM_BYTES = (BM + 2 * BN) * (D + 8) * 2; + +template +__global__ void __launch_bounds__(THREADS) + flash_kernel(const bf16* __restrict__ q, const bf16* __restrict__ k, const bf16* __restrict__ v, int ldv, + float* __restrict__ out, float* __restrict__ lse, int T, int Hq, int Hk, float scale, int splits) { + constexpr int LDS = D + 8; + const float scale_log2=(float)((double)scale*1.4426950408889634); + extern __shared__ __align__(16) unsigned char smem[]; + bf16* qs = reinterpret_cast(smem); + bf16* ks = qs + BM * LDS; + bf16* vs = ks + BN * LDS; + const int h = blockIdx.y, hk = h / (Hq / Hk); + const int q0 = (gridDim.x - 1 - blockIdx.x) * BM; // the longest blocks first + const int tid = threadIdx.x, warp = tid / 32, lane = tid % 32; + const int g = lane / 4, t = lane % 4; + const int row0 = q0 + warp * 16; // this warp's first query + + for (int c = tid; c < BM * (D / 8); c += THREADS) { + const int r = c / (D / 8), col = (c % (D / 8)) * 8, row = q0 + r; + cp_async16(qs + r * LDS + col, q + ((size_t)min(row, T - 1) * Hq + h) * D + col, row < T); + } + cp_async_commit(); + + float o[D / 8][4]; +#pragma unroll + for (int n = 0; n < D / 8; n++) o[n][0] = o[n][1] = o[n][2] = o[n][3] = 0.f; + float m[2] = {-INFINITY, -INFINITY}, l[2] = {0.f, 0.f}; + + int per_split=((T+BN-1)/BN+splits-1)/splits; + int first=blockIdx.z*per_split*BN; + int last=min((T+BN-1)/BN*BN,first+per_split*BN); + for (int k0=last-BN;k0>=first;k0-=BN) { + for (int c = tid; c < BN * (D / 8); c += THREADS) { + const int r = c / (D / 8), col = (c % (D / 8)) * 8, s = k0 + r; + cp_async16(ks + r * LDS + col, k + ((size_t)min(s, T - 1) * Hk + hk) * D + col, s < T); + } + cp_async_commit(); + for (int c = tid; c < BN * (D / 8); c += THREADS) { + const int r = c / (D / 8), col = (c % (D / 8)) * 8, s = k0 + r; + cp_async16(vs + r * LDS + col, v + (size_t)min(s, T - 1) * ldv + (size_t)hk * D + col, s < T); + } + cp_async_commit(); + cp_async_wait<1>(); // Q and K + __syncthreads(); + + // keys past every query of this warp contribute nothing + const bool active = true; + float sc[BN / 8][4]; +#pragma unroll + for (int n = 0; n < BN / 8; n++) sc[n][0] = sc[n][1] = sc[n][2] = sc[n][3] = 0.f; + if (active) { +#pragma unroll + for (int kk = 0; kk < D; kk += 16) { + uint32_t a[4]; + load_a(a, qs, LDS, warp * 16, kk, lane); +#pragma unroll + for (int n = 0; n < BN / 8; n += 2) { + uint32_t b[4]; + load_b_nk(b, ks, LDS, kk, n * 8, lane); + mma16816(sc[n], a, b[0], b[1]); + mma16816(sc[n + 1], a, b[2], b[3]); + } + } + } + uint32_t p[BN / 16][4]; + if (active) { + // causal and length mask, then the online softmax in base 2 + float mx[2] = {-INFINITY, -INFINITY}; +#pragma unroll + for (int n = 0; n < BN / 8; n++) { +#pragma unroll + for (int e = 0; e < 4; e++) { + const int key = k0 + n * 8 + 2 * t + (e & 1); + sc[n][e] = (key < T) ? sc[n][e] : -INFINITY; + mx[e >> 1] = fmaxf(mx[e >> 1], sc[n][e]); + } + } + float alpha[2], base[2]; +#pragma unroll + for (int r = 0; r < 2; r++) { + mx[r] = fmaxf(mx[r], __shfl_xor_sync(0xffffffffu, mx[r], 1)); + mx[r] = fmaxf(mx[r], __shfl_xor_sync(0xffffffffu, mx[r], 2)); + const float mn = fmaxf(m[r], mx[r]); + base[r] = mn == -INFINITY ? 0.f : mn; + alpha[r] = exp2f((m[r] - base[r])*scale_log2); + m[r] = mn; + l[r] *= alpha[r]; + } +#pragma unroll + for (int n = 0; n < BN / 8; n++) { +#pragma unroll + for (int e = 0; e < 4; e++) { + sc[n][e] = exp2f(__fmul_rn(sc[n][e],scale_log2)-__fmul_rn(base[e >> 1],scale_log2)); + l[e >> 1] += sc[n][e]; + } + } +#pragma unroll + for (int n = 0; n < D / 8; n++) { + o[n][0] *= alpha[0]; + o[n][1] *= alpha[0]; + o[n][2] *= alpha[1]; + o[n][3] *= alpha[1]; + } + // the score accumulators, two 8-key tiles at a time, are the A fragments of P*V +#pragma unroll + for (int j = 0; j < BN / 16; j++) { + p[j][0] = pack_bf16(sc[2 * j][0], sc[2 * j][1]); + p[j][1] = pack_bf16(sc[2 * j][2], sc[2 * j][3]); + p[j][2] = pack_bf16(sc[2 * j + 1][0], sc[2 * j + 1][1]); + p[j][3] = pack_bf16(sc[2 * j + 1][2], sc[2 * j + 1][3]); + } + } + cp_async_wait<0>(); // V + __syncthreads(); + if (active) { +#pragma unroll + for (int j = 0; j < BN / 16; j++) { +#pragma unroll + for (int n = 0; n < D / 8; n += 2) { + uint32_t b[4]; + load_b_kn(b, vs, LDS, j * 16, n * 8, lane); + mma16816(o[n], p[j], b[0], b[1]); + mma16816(o[n + 1], p[j], b[2], b[3]); + } + } + } + __syncthreads(); // before the next tile overwrites K and V + } + + // the four lanes of a row each summed a quarter of its keys +#pragma unroll + for (int r = 0; r < 2; r++) { + l[r] += __shfl_xor_sync(0xffffffffu, l[r], 2); + l[r] += __shfl_xor_sync(0xffffffffu, l[r], 1); + } + const float inv[2] = {1.f / l[0], 1.f / l[1]}; +#pragma unroll + for (int r = 0; r < 2; r++) { + const int row = row0 + g + r * 8; + if (row >= T) continue; + float* dst=out+(((size_t)blockIdx.z*T+row)*Hq+h)*D+2*t; + if(t==0) lse[((size_t)blockIdx.z*T+row)*Hq+h]=fmaf(m[r],scale,__logf(l[r])); +#pragma unroll + for(int n=0;n=total) return; + if(splits==1) { out[index]=to_bf16(partial[index]);return; } + size_t row=index/D,rows=(size_t)T*H; + float maximum=-INFINITY,values[128];int power=1;while(power0;step/=2) for(int i=0;i65536 || H<=0 || !workspace || sms<=0) return cudaErrorInvalidValue; + int splits=reference::split_count(T,H,128,sms); + float* memory=(float*)workspace;size_t count=(size_t)splits*T*H*80; + cudaError_t status=cudaSuccess; + float* lse=memory+count;float scale=(float)(1.0/sqrt(72.0)); + if(splits==1) { + cudaFuncSetAttribute(f::flash_kernel<80,64>,cudaFuncAttributeMaxDynamicSharedMemorySize,f::SMEM_BYTES<80,64>); + f::flash_kernel<80,64><<,(cudaStream_t)stream>>>((const bf16*)q,(const bf16*)k,(const bf16*)v,H*80,memory,lse,T,H,H,scale,splits); + } else { + cudaFuncSetAttribute(f::flash_kernel<80,128>,cudaFuncAttributeMaxDynamicSharedMemorySize,f::SMEM_BYTES<80,128>); + f::flash_kernel<80,128><<,(cudaStream_t)stream>>>((const bf16*)q,(const bf16*)k,(const bf16*)v,H*80,memory,lse,T,H,H,scale,splits); + } + status=cudaGetLastError(); + if(status==cudaSuccess) { + f::combine<<<((size_t)T*H*80+127)/128,128,0,(cudaStream_t)stream>>>(memory,lse,(bf16*)out,T,H,80,splits); + status=cudaGetLastError(); + } + return status; +} + +extern "C" size_t cs1_reference_vision_attention_workspace_floats(int capacity,int heads) { return cs1::reference::workspace_floats(capacity,heads,80,128); } diff --git a/src/models/jemm/README.md b/src/models/jemm/README.md index 3674a1eb..5238ebc1 100644 --- a/src/models/jemm/README.md +++ b/src/models/jemm/README.md @@ -3,7 +3,7 @@ The Rust/CUDA worker serves ordered `choice`, `score` and `noul` questions over text and up to four PNG/JPEG/WebP images. See the [recipe](../../../recipe/jemm/README.md) for download, export, launch and validation. CPU inference and native Metal are -unsupported. Multi-image reference parity has a known limitation below. +unsupported. Numerical validation is scoped to the frozen corpus below. ## Pinned artifacts @@ -82,27 +82,28 @@ covers the entire request. Both streams synchronize before resource release; caught execution errors/panics clear atomic readiness and retire loaded state. There is no cross-request batching or prefix cache. -## Validation and known multi-image limitation - -The unmerged FP32-adapter v2 path requires a fresh device/reference campaign; -CPU export and contract checks alone do not establish its numerical parity. - -The historical premerged v1 A800 BF16 corpus (13 requests / 16 questions, text and single-image -inputs) passed exact tokens/positions/grids/pixels and predeclared probability -0.02 / Score 0.1 / winner-margin 0.05 gates. Max probability drift was 0.0015338802. -These results are scoped to the source revisions and corpus in -[the evidence release](https://github.com/Levius-Fubuki/system1-omni/releases/tag/jemm-a800-20261008). - -**Two- and four-image reference parity is not established.** The supplemental -2/3/4-PNG-image feasibility corpus exceeded the unchanged 0.02 probability gate -for two and four images, with max drift **0.042138323189940485**; three images -passed. All nine question records had exact official tokens, positions, grids, -candidate counts and FP32 pixels. #118 and #122 produced identical answers and -usage for these requests. This narrows investigation to execution/merge numerics, -but does not identify the cause. Candidate repeated rounds stopped after the -failed feasibility gate; failures were retained. The worker accepts these input -classes, but applications requiring verified multi-image fidelity must validate -their own workloads or use the pinned reference. No tolerance was relaxed. +## Validation + +The unmerged FP32-adapter v2 path passes the frozen **16 requests / 25 questions** +covering text and 1–4 images on A800. Maximum probability drift is +**0.0166172279642417**, under the unchanged 0.02 gate. Score tolerance remains +0.1 and winner agreement is required at reference margins of at least 0.05; +low-margin cases remain in the corpus. See the +[requests, reference outputs and evidence](../../../docs/benchmarks/jemm-reference-20261011/README.md). +These results do not establish parity for every possible prompt, GPU or framework version. + +JEMM opts into separate reference CUDA functions. They preserve the active +framework's BF16 boundaries, TF32 matrix-product order, normalization reductions, +FlashAttention split combination and cuDNN patch convolution. FP32 adapters stay +unmerged, component projections keep their original GEMM shapes, and multi-image +vision projections use aggregate rows while attention remains image-local. +Existing Cua/Open-Jev/JEV-VL CUDA entry points retain their previous behavior. + +The historical premerged v1 corpus passed text/single-image checks but failed the +supplemental two-/four-image gate (maximum drift 0.042138323189940485). +That failed evidence remains in the +[original release](https://github.com/Levius-Fubuki/system1-omni/releases/tag/jemm-a800-20261008); +it is not evidence for the new v2 path. Root `tests/jemm/` covers prompt order/coercion, candidate limits, the 64-question resource boundary, response math, image decoding/positions, tokenizer and token diff --git a/src/models/jemm/native/src/executor.rs b/src/models/jemm/native/src/executor.rs index 345372d5..a11baab4 100644 --- a/src/models/jemm/native/src/executor.rs +++ b/src/models/jemm/native/src/executor.rs @@ -81,7 +81,7 @@ impl LabelHead { cuda::upload(weight.at(0), bytes, stream.0)?; } // SAFETY: backend initialization returns an owning handle or null. - let gemm = unsafe { (cuda::api().cs1_gemm_create)(32 << 20) }; + let gemm = unsafe { (cuda::api().reference()?.gemm_create)(32 << 20) }; ensure!(!gemm.is_null(), "LM head cuBLASLt setup failed"); Ok(Self { stream, @@ -215,8 +215,10 @@ impl Executor { )?; let (dir, library) = (dir.to_path_buf(), library.to_path_buf()); let state = tokio::task::spawn_blocking(move || -> Result { - let language = Model::load_with_lora(&dir, &library, &adapter_path, scale)?; - let vision = VisionModel::load(&dir, &library)?; + let mut language = Model::load_with_lora(&dir, &library, &adapter_path, scale)?; + language.enable_reference_numerics()?; + let mut vision = VisionModel::load(&dir, &library)?; + vision.enable_reference_numerics()?; let head = LabelHead::load(&bytes, cfg.hidden)?; Ok(Loaded { language, diff --git a/src/models/qwen3_5/native/src/cuda.rs b/src/models/qwen3_5/native/src/cuda.rs index c3d1ac8b..277f565e 100644 --- a/src/models/qwen3_5/native/src/cuda.rs +++ b/src/models/qwen3_5/native/src/cuda.rs @@ -8,6 +8,9 @@ use std::sync::OnceLock; use anyhow::{Context, Result, bail, ensure}; +mod reference; +pub use reference::Reference; + /// `CS1_ABI_VERSION` in ops.h. const ABI_VERSION: u32 = 6; pub const LIBRARY: &str = "libqwen3_5_cuda.so"; @@ -73,10 +76,17 @@ macro_rules! api { pub struct Api { _lib: libloading::Library, vision_v2: Option, + reference: Option, $(pub $name: unsafe extern "C" fn($($ty),*) $(-> $ret)?,)* } impl Api { + pub fn reference(&self) -> Result<&Reference> { + let reference=self.reference.as_ref().context("JEMM requires the optional reference CUDA symbols; rebuild libqwen3_5_cuda.so")?; + // SAFETY: this capability query takes no arguments. + ensure!(unsafe { (reference.available)() } == 1, "JEMM reference numerics require a cuDNN 9 build; set CUDNN_INCLUDE_DIR and CUDNN_LIB_DIR"); + Ok(reference) + } pub fn vision_v2(&self) -> Result<&VisionV2> { self.vision_v2.as_ref().context("27B vision needs additive v2 CUDA symbols; rebuild libqwen3_5_cuda.so") } @@ -92,7 +102,8 @@ macro_rules! api { .with_context(|| format!("{} has no {}", LIBRARY, stringify!($name)))?; )* let vision_v2=VisionV2::resolve(&lib); - Ok(Self { _lib: lib, vision_v2, $($name,)* }) + let reference=Reference::resolve(&lib); + Ok(Self { _lib: lib, vision_v2, reference, $($name,)* }) } } }; diff --git a/src/models/qwen3_5/native/src/cuda/reference.rs b/src/models/qwen3_5/native/src/cuda/reference.rs new file mode 100644 index 00000000..bd93b388 --- /dev/null +++ b/src/models/qwen3_5/native/src/cuda/reference.rs @@ -0,0 +1,139 @@ +//! Optional JEMM numerics; legacy ABI-6 libraries may omit the entire group. +use super::{Stream, c_int, c_void}; +pub struct Reference { + pub gemm_create: unsafe extern "C" fn(usize) -> *mut c_void, + pub rms_norm: unsafe extern "C" fn( + *const c_void, + *const c_void, + *mut c_void, + c_int, + c_int, + f32, + Stream, + ) -> c_int, + pub add_rms_norm: unsafe extern "C" fn( + *mut c_void, + *const c_void, + *const c_void, + *mut c_void, + c_int, + c_int, + f32, + Stream, + ) -> c_int, + pub gdn_workspace_floats: unsafe extern "C" fn(c_int, c_int) -> usize, + pub gdn_prefill: unsafe extern "C" fn( + *const c_void, + *const c_void, + *const c_void, + *const f32, + *const c_void, + *mut c_void, + *mut f32, + c_int, + c_int, + c_int, + f32, + Stream, + ) -> c_int, + pub attn_prep: unsafe extern "C" fn( + *const c_void, + *const c_void, + c_int, + *const c_void, + *const c_void, + *const c_void, + *const c_void, + *mut c_void, + *mut c_void, + *mut c_void, + c_int, + c_int, + c_int, + c_int, + c_int, + f32, + Stream, + ) -> c_int, + pub vision_norm: unsafe extern "C" fn( + *const c_void, + *const c_void, + *const c_void, + *mut c_void, + c_int, + c_int, + Stream, + ) -> c_int, + pub vision_rope: unsafe extern "C" fn( + *const c_void, + *const f32, + *const f32, + *mut c_void, + *mut c_void, + c_int, + c_int, + c_int, + Stream, + ) -> c_int, + pub vision_attention: unsafe extern "C" fn( + *const c_void, + *const c_void, + *const c_void, + *mut c_void, + c_int, + c_int, + c_int, + *mut c_void, + Stream, + ) -> c_int, + pub available: unsafe extern "C" fn() -> c_int, + pub language_attention_workspace_floats: unsafe extern "C" fn(c_int, c_int) -> usize, + pub vision_attention_workspace_floats: unsafe extern "C" fn(c_int, c_int) -> usize, + pub attention: unsafe extern "C" fn( + *const c_void, + *const c_void, + *const c_void, + c_int, + *const c_void, + *mut c_void, + c_int, + c_int, + c_int, + f32, + *mut c_void, + Stream, + ) -> c_int, + pub patch_create: unsafe extern "C" fn(c_int, Stream) -> *mut c_void, + pub patch: + unsafe extern "C" fn(*mut c_void, *const c_void, *const c_void, *mut c_void) -> c_int, + pub patch_destroy: unsafe extern "C" fn(*mut c_void) -> (), +} +impl Reference { + pub(super) fn resolve(lib: &libloading::Library) -> Option { + // SAFETY: signatures match the additive cs1_reference_* declarations in ops.h. + unsafe { + Some(Self { + gemm_create: *lib.get(b"cs1_reference_gemm_create\0").ok()?, + rms_norm: *lib.get(b"cs1_reference_rms_norm\0").ok()?, + add_rms_norm: *lib.get(b"cs1_reference_add_rms_norm\0").ok()?, + gdn_workspace_floats: *lib.get(b"cs1_reference_gdn_workspace_floats\0").ok()?, + gdn_prefill: *lib.get(b"cs1_reference_gdn_prefill\0").ok()?, + attn_prep: *lib.get(b"cs1_reference_attn_prep\0").ok()?, + vision_norm: *lib.get(b"cs1_reference_vision_norm\0").ok()?, + vision_rope: *lib.get(b"cs1_reference_vision_rope\0").ok()?, + vision_attention: *lib.get(b"cs1_reference_vision_attention\0").ok()?, + available: *lib.get(b"cs1_reference_available\0").ok()?, + language_attention_workspace_floats: *lib + .get(b"cs1_reference_language_attention_workspace_floats\0") + .ok()?, + vision_attention_workspace_floats: *lib + .get(b"cs1_reference_vision_attention_workspace_floats\0") + .ok()?, + attention: *lib.get(b"cs1_reference_attention\0").ok()?, + patch_create: *lib.get(b"cs1_reference_patch_create\0").ok()?, + patch: *lib.get(b"cs1_reference_patch\0").ok()?, + patch_destroy: *lib.get(b"cs1_reference_patch_destroy\0").ok()?, + }) + } + } +} diff --git a/src/models/qwen3_5/native/src/model.rs b/src/models/qwen3_5/native/src/model.rs index d41b19b2..0d27abef 100644 --- a/src/models/qwen3_5/native/src/model.rs +++ b/src/models/qwen3_5/native/src/model.rs @@ -444,7 +444,13 @@ struct Scratch { } impl Scratch { - fn new(cfg: &Config, cap: usize, stream: Stream, lora: Option<&lora::Lora>) -> Result { + fn new( + cfg: &Config, + cap: usize, + stream: Stream, + lora: Option<&lora::Lora>, + reference: Option<&cuda::Reference>, + ) -> Result { let (h, kd, vd, hv) = (cfg.hidden, cfg.key_dim(), cfg.value_dim(), cfg.lin_v_heads); let (hq, hk, hd) = (cfg.heads, cfg.kv_heads, cfg.head_dim); let w = Widths::of(cfg); @@ -455,7 +461,18 @@ impl Scratch { off }; // SAFETY: pure function of its arguments. - let ws_floats = unsafe { (cuda::api().cs1_gdn_workspace_floats)(cap as i32, hv as i32) }; + let ws_floats = if let Some(reference) = reference { + // SAFETY: capacity and heads derive from validated architecture/request limits. + unsafe { + let gdn = (reference.gdn_workspace_floats)(cap as i32, hv as i32); + let attention = + (reference.language_attention_workspace_floats)(cap as i32, hq as i32); + ensure!(attention > 0, "cannot size reference attention workspace"); + gdn.max(attention) + } + } else { + unsafe { (cuda::api().cs1_gdn_workspace_floats)(cap as i32, hv as i32) } + }; let offsets = [ take(cap * 4), take(cap * h * BF16), @@ -602,6 +619,7 @@ pub struct Model { lora: Option, stream: Stream, gemm: *mut c_void, + reference: Option<&'static cuda::Reference>, /// Buffers for the largest packed token count so far; grows as needed. scratch: Option, /// Opt-in replay with at most 64 captures keyed by ordered sequence lengths. @@ -740,6 +758,7 @@ impl Model { lora, stream, gemm, + reference: None, scratch: None, graph_enabled: std::env::var("CUA_S1_GRAPH").as_deref() == Ok("1"), graphs: VecDeque::new(), @@ -748,6 +767,33 @@ impl Model { Ok(model) } + /// Select JEMM's pinned framework arithmetic before the first forward. + /// Prefix caching is outside this execution contract. + pub fn enable_reference_numerics(&mut self) -> Result<()> { + ensure!( + self.cfg.hidden == 5120 && self.cfg.head_dim == 256 && self.lora.is_some(), + "reference numerics require the unmerged JEMM 27B model" + ); + ensure!( + self.scratch.is_none() && self.graphs.is_empty(), + "select reference numerics before the first forward" + ); + if self.reference.is_some() { + return Ok(()); + } + cuda::set_device(0)?; + let reference = cuda::api().reference()?; + cuda::synchronize(self.stream)?; + // SAFETY: the model owns both handles, with no queued work or captured graphs. + let gemm = unsafe { (reference.gemm_create)(GEMM_WORKSPACE) }; + ensure!(!gemm.is_null(), "reference cuBLASLt setup failed"); + unsafe { (cuda::api().cs1_gemm_destroy)(self.gemm) }; + self.gemm = gemm; + self.reference = Some(reference); + self.prefix_owner = Arc::new(()); + Ok(()) + } + fn gemm(&self, s: &Scratch, x: usize, w: &Tensor, y: usize, m: usize) -> Result<()> { let (n, k) = (w.shape[0] as i32, w.shape[1] as i32); // SAFETY: x and y are scratch buffers sized for m rows of w's shape. @@ -873,6 +919,7 @@ impl Model { t.next_multiple_of(1024), self.stream, self.lora.as_ref(), + self.reference, )?); } Ok(()) @@ -1028,6 +1075,10 @@ impl Model { /// Allocate an uninitialized prefix of `len` tokens (a multiple of 64). /// Capture must finish successfully on this model before continuation. pub fn alloc_prefix(&self, len: usize) -> Result { + ensure!( + self.reference.is_none(), + "JEMM reference numerics do not support prefix caching" + ); ensure!( len >= 64 && len.is_multiple_of(64), "cached prefix length must be a positive multiple of 64" @@ -1219,7 +1270,9 @@ impl Model { // scratch buffers laid out for at least t tokens with the widths used here. unsafe { check( - (cuda::api().cs1_rms_norm)( + (self + .reference + .map_or(cuda::api().cs1_rms_norm, |r| r.rms_norm))( p(s.res), self.layers[0].input_norm.ptr, p(s.x), @@ -1283,7 +1336,9 @@ impl Model { )?; for &(offset, length) in &sequences { check( - (cuda::api().cs1_gdn_prefill)( + (self + .reference + .map_or(cuda::api().cs1_gdn_prefill, |r| r.gdn_prefill))( p(s.lq + offset * kd * BF16), p(s.lk + offset * kd * BF16), p(s.lv + offset * vd * BF16), @@ -1336,7 +1391,9 @@ impl Model { unsafe { for &(offset, length) in &sequences { check( - (cuda::api().cs1_attn_prep)( + (self + .reference + .map_or(cuda::api().cs1_attn_prep, |r| r.attn_prep))( p(s.attn_in + offset * w.attn_in * BF16), p(k + offset * w.attn_in * BF16), ld, @@ -1358,20 +1415,37 @@ impl Model { "attention prep", )?; check( - (cuda::api().cs1_attention_gated)( - p(s.aq + offset * cfg.heads * cfg.head_dim * BF16), - p(s.ak + offset * cfg.kv_heads * cfg.head_dim * BF16), - p(v + offset * w.attn_in * BF16), - ld, - p(s.agate + offset * cfg.heads * cfg.head_dim * BF16), - p(s.ao + offset * cfg.heads * cfg.head_dim * BF16), - length, - hq, - hk, - hd, - (cfg.head_dim as f32).powf(-0.5), - st, - ), + if let Some(reference) = self.reference { + (reference.attention)( + p(s.aq + offset * cfg.heads * cfg.head_dim * BF16), + p(s.ak + offset * cfg.kv_heads * cfg.head_dim * BF16), + p(v + offset * w.attn_in * BF16), + ld, + p(s.agate + offset * cfg.heads * cfg.head_dim * BF16), + p(s.ao + offset * cfg.heads * cfg.head_dim * BF16), + length, + hq, + hk, + (cfg.head_dim as f32).powf(-0.5), + p(s.workspace), + st, + ) + } else { + (cuda::api().cs1_attention_gated)( + p(s.aq + offset * cfg.heads * cfg.head_dim * BF16), + p(s.ak + offset * cfg.kv_heads * cfg.head_dim * BF16), + p(v + offset * w.attn_in * BF16), + ld, + p(s.agate + offset * cfg.heads * cfg.head_dim * BF16), + p(s.ao + offset * cfg.heads * cfg.head_dim * BF16), + length, + hq, + hk, + hd, + (cfg.head_dim as f32).powf(-0.5), + st, + ) + }, "gated attention", )?; } @@ -1382,7 +1456,9 @@ impl Model { } unsafe { check( - (cuda::api().cs1_add_rms_norm)( + (self + .reference + .map_or(cuda::api().cs1_add_rms_norm, |r| r.add_rms_norm))( p(s.res), p(s.delta), layer.post_norm.ptr, @@ -1418,7 +1494,9 @@ impl Model { .map_or(&self.final_norm, |l| &l.input_norm); unsafe { check( - (cuda::api().cs1_add_rms_norm)( + (self + .reference + .map_or(cuda::api().cs1_add_rms_norm, |r| r.add_rms_norm))( p(s.res), p(s.delta), next.ptr, @@ -1489,7 +1567,9 @@ impl Model { // buffers hold exactly the shapes allocated by alloc_prefix(qb). unsafe { check( - (cuda::api().cs1_rms_norm)( + (self + .reference + .map_or(cuda::api().cs1_rms_norm, |r| r.rms_norm))( p(s.res + qb * hb), self.layers[0].input_norm.ptr, p(s.x + qb * hb), @@ -1661,7 +1741,9 @@ impl Model { cuda::copy_dd(p(s.ak), pre.attn_kv[fa_i].0.at(0), qb * kvb, st)?; } check( - (cuda::api().cs1_attn_prep)( + (self + .reference + .map_or(cuda::api().cs1_attn_prep, |r| r.attn_prep))( p(s.attn_in + qb * ab), p(k + qb * ab), w.attn_in as i32, @@ -1720,7 +1802,9 @@ impl Model { } unsafe { check( - (cuda::api().cs1_add_rms_norm)( + (self + .reference + .map_or(cuda::api().cs1_add_rms_norm, |r| r.add_rms_norm))( p(s.res + qb * hb), p(s.delta + qb * hb), layer.post_norm.ptr, @@ -1784,7 +1868,9 @@ impl Model { .map_or(&self.final_norm, |l| &l.input_norm); unsafe { check( - (cuda::api().cs1_add_rms_norm)( + (self + .reference + .map_or(cuda::api().cs1_add_rms_norm, |r| r.add_rms_norm))( p(s.res + qb * hb), p(s.delta + qb * hb), next.ptr, diff --git a/src/models/qwen3_5/native/src/vision/geometry.rs b/src/models/qwen3_5/native/src/vision/geometry.rs index cc2e672c..fe89a151 100644 --- a/src/models/qwen3_5/native/src/vision/geometry.rs +++ b/src/models/qwen3_5/native/src/vision/geometry.rs @@ -27,6 +27,26 @@ impl VisionGeometry { ); Ok(n) } + pub(crate) fn use_reference_angles(&mut self, grids: &[[usize; 3]], config: &VisionConfig) { + self.cos.clear(); + let quarter = config.head_dim() / 4; + for &[_, h, w] in grids { + for br in 0..h / 2 { + for bc in 0..w / 2 { + for ir in 0..2 { + for ic in 0..2 { + for pos in [br * 2 + ir, bc * 2 + ic] { + for i in 0..quarter { + let inv = 1.0f32 / 10000f32.powf(i as f32 / quarter as f32); + self.cos.push(pos as f32 * inv); + } + } + } + } + } + } + } + } pub fn new([t, h, w]: [usize; 3], config: &VisionConfig) -> Result { let n = Self::patch_count([t, h, w], config)?; let mut g = Self { diff --git a/src/models/qwen3_5/native/src/vision/model.rs b/src/models/qwen3_5/native/src/vision/model.rs index f70e75ca..ce21d398 100644 --- a/src/models/qwen3_5/native/src/vision/model.rs +++ b/src/models/qwen3_5/native/src/vision/model.rs @@ -44,6 +44,7 @@ pub struct VisionModel { base: BTreeMap, lora: BTreeMap, gemm: Gemm, + reference: Option<&'static cuda::Reference>, stream: OwnedStream, } impl VisionModel { @@ -137,6 +138,7 @@ impl VisionModel { lora: BTreeMap::new(), stream, gemm, + reference: None, }; for (name, tensor) in base { model.base.insert( @@ -158,6 +160,32 @@ impl VisionModel { ); Ok(model) } + /// Select the JEMM 27B reference path before allocating shape buffers or graphs. + pub fn enable_reference_numerics(&mut self) -> Result<()> { + ensure!( + self.config.hidden_size == 1152 && self.lora.is_empty(), + "reference vision requires unadapted 27B vision" + ); + ensure!( + self.scratch.is_empty() && self.batch_scratch.is_empty(), + "select reference vision before the first forward" + ); + if self.reference.is_some() { + return Ok(()); + } + cuda::set_device(0)?; + let reference = api().reference()?; + self.synchronize()?; + // SAFETY: allocation occurs before capture, and the old handle has no queued work. + let gemm = Gemm(unsafe { (reference.gemm_create)(32 << 20) }); + ensure!( + !gemm.0.is_null(), + "cannot create reference vision GEMM handle" + ); + self.gemm = gemm; + self.reference = Some(reference); + Ok(()) + } fn upload(&self, bytes: &[u8]) -> Result { let buffer = DeviceBuffer::new(bytes.len())?; unsafe { @@ -171,7 +199,9 @@ impl VisionModel { fn norm(&self, name: &str, x: &DeviceBuffer, y: &DeviceBuffer, rows: usize) -> Result<()> { unsafe { check( - (api().cs1_vision_norm)( + (self + .reference + .map_or(api().cs1_vision_norm, |r| r.vision_norm))( x.at(0), self.weight(&format!("{name}.weight")), self.weight(&format!("{name}.bias")), @@ -352,7 +382,10 @@ impl VisionModel { let index = if let Some(index) = self.batch_scratch.iter().position(|s| s.grids == grids) { index } else { - let geometry = BatchGeometry::new(&grids, &self.config)?.geometry; + let mut geometry = BatchGeometry::new(&grids, &self.config)?.geometry; + if self.reference.is_some() { + geometry.use_reference_angles(&grids, &self.config); + } self.synchronize()?; if self.batch_scratch.len() == 4 { self.batch_scratch.pop_front(); @@ -421,7 +454,13 @@ impl VisionModel { .iter() .position(|s| s.grid == image.image_grid_thw); let geometry = if cached.is_none() { - Some(VisionGeometry::new(image.image_grid_thw, &self.config)?) + Some({ + let mut geometry = VisionGeometry::new(image.image_grid_thw, &self.config)?; + if self.reference.is_some() { + geometry.use_reference_angles(&[image.image_grid_thw], &self.config); + } + geometry + }) } else { None }; @@ -523,7 +562,33 @@ impl VisionModel { out, .. } = buffers; - self.linear("patch_embed.proj", pixels, x, n, h, 1536, w)?; + if let Some(plan) = &buffers.patch { + let reference = self.reference.expect("reference patch plan"); + // SAFETY: the plan and buffers have the same validated aggregate row count. + unsafe { + check( + (reference.patch)( + plan.0, + pixels.at(0), + self.weight("patch_embed.proj.weight"), + x.at(0), + ), + "reference patch convolution", + )?; + check( + (api().cs1_vision_bias)( + x.at(0), + self.weight("patch_embed.proj.bias"), + n * h, + h as i32, + self.stream.0, + ), + "patch bias", + )?; + } + } else { + self.linear("patch_embed.proj", pixels, x, n, h, 1536, w)?; + } self.trace(callback, "patch_embed", x, n * h)?; unsafe { if h == 1024 { @@ -586,7 +651,7 @@ impl VisionModel { } else { let v2 = api().vision_v2()?; check( - (v2.rope)( + (self.reference.map_or(v2.rope, |r| r.vision_rope))( qkv.at(0), co.at(0).cast(), si.at(0).cast(), @@ -602,7 +667,7 @@ impl VisionModel { let mut row = 0; for &rows in lengths { check( - (v2.attention)( + (self.reference.map_or(v2.attention, |r| r.vision_attention))( q.at(row * h * 2), k.at(row * h * 2), qkv.at((row * h * 3 + h * 2) * 2), @@ -701,7 +766,22 @@ struct BatchScratch { lengths: Vec, buffers: Buffers, } +struct PatchPlan(*mut c_void); +// SAFETY: the enclosing model serializes access and keeps the owning stream alive. +unsafe impl Send for PatchPlan {} +impl Drop for PatchPlan { + fn drop(&mut self) { + // SAFETY: the enclosing model synchronizes before retiring buffers. + unsafe { + (api() + .reference() + .expect("loaded reference API") + .patch_destroy)(self.0) + }; + } +} struct Buffers { + patch: Option, pixels: DeviceBuffer, indices: DeviceBuffer, weights: DeviceBuffer, @@ -742,7 +822,36 @@ impl Buffers { .intermediate_size .max(h * 4) .max(model.config.out_hidden_size); + let patch = if let Some(reference) = model.reference { + // SAFETY: descriptors/workspace are allocated outside capture for validated rows. + let plan = PatchPlan(unsafe { (reference.patch_create)(n as i32, model.stream.0) }); + ensure!( + !plan.0.is_null(), + "cannot prepare cuDNN reference patch convolution" + ); + Some(plan) + } else { + None + }; + let attention_bytes = if let Some(reference) = model.reference { + let floats = unsafe { + (reference.vision_attention_workspace_floats)( + attention_rows as i32, + model.config.num_heads as i32, + ) + }; + ensure!( + floats > 0, + "cannot size reference vision attention workspace" + ); + attention_rows * model.config.num_heads * 80 * 4 * 2 + floats * 4 + } else if h == 1024 { + 0 + } else { + attention_rows * model.config.num_heads * 80 * 4 * 2 + }; Ok(Self { + patch, pixels: DeviceBuffer::new(n * 1536 * 2)?, indices: model.upload( &geo.indices @@ -769,11 +878,7 @@ impl Buffers { .collect::>(), )?, w: Work::new(if model.lora.is_empty() { 0 } else { n }, width)?, - attention_workspace: DeviceBuffer::new(if h == 1024 { - 0 - } else { - attention_rows * model.config.num_heads * 80 * 4 * 2 - })?, + attention_workspace: DeviceBuffer::new(attention_bytes)?, x: DeviceBuffer::new(n * h * 2)?, norm: DeviceBuffer::new(n * h * 2)?, qkv: DeviceBuffer::new(n * h * 3 * 2)?, diff --git a/tests/jemm/data/parity/corpus.jsonl b/tests/jemm/data/parity/corpus.jsonl new file mode 100644 index 00000000..ec2ef93b --- /dev/null +++ b/tests/jemm/data/parity/corpus.jsonl @@ -0,0 +1,16 @@ +{"id":"text-choice-2","modality":"text","expected":{"select":"option-01"},"request":{"state":{"workspaces":2,"requested_workspace":1,"note":"café 工作区"},"questions":{"select":{"type":"choice","instructions":"Which action opens the requested workspace?","criteria":{"option-00":{"description":"Open workspace number 0."},"option-01":{"description":"Open workspace number 1."}}}}}} +{"id":"text-choice-3","modality":"text","expected":{"select":"option-02"},"request":{"state":{"workspaces":3,"requested_workspace":2,"note":"café 工作区"},"questions":{"select":{"type":"choice","instructions":"Which action opens the requested workspace?","criteria":{"option-00":{"description":"Open workspace number 0."},"option-01":{"description":"Open workspace number 1."},"option-02":{"description":"Open workspace number 2."}}}}}} +{"id":"text-choice-10","modality":"text","expected":{"select":"option-09"},"request":{"state":{"workspaces":10,"requested_workspace":9,"note":"café 工作区"},"questions":{"select":{"type":"choice","instructions":"Which action opens the requested workspace?","criteria":{"option-00":{"description":"Open workspace number 0."},"option-01":{"description":"Open workspace number 1."},"option-02":{"description":"Open workspace number 2."},"option-03":{"description":"Open workspace number 3."},"option-04":{"description":"Open workspace number 4."},"option-05":{"description":"Open workspace number 5."},"option-06":{"description":"Open workspace number 6."},"option-07":{"description":"Open workspace number 7."},"option-08":{"description":"Open workspace number 8."},"option-09":{"description":"Open workspace number 9."}}}}}} +{"id":"text-choice-26","modality":"text","expected":{"select":"option-25"},"request":{"state":{"workspaces":26,"requested_workspace":25,"note":"café 工作区"},"questions":{"select":{"type":"choice","instructions":"Which action opens the requested workspace?","criteria":{"option-00":{"description":"Open workspace number 0."},"option-01":{"description":"Open workspace number 1."},"option-02":{"description":"Open workspace number 2."},"option-03":{"description":"Open workspace number 3."},"option-04":{"description":"Open workspace number 4."},"option-05":{"description":"Open workspace number 5."},"option-06":{"description":"Open workspace number 6."},"option-07":{"description":"Open workspace number 7."},"option-08":{"description":"Open workspace number 8."},"option-09":{"description":"Open workspace number 9."},"option-10":{"description":"Open workspace number 10."},"option-11":{"description":"Open workspace number 11."},"option-12":{"description":"Open workspace number 12."},"option-13":{"description":"Open workspace number 13."},"option-14":{"description":"Open workspace number 14."},"option-15":{"description":"Open workspace number 15."},"option-16":{"description":"Open workspace number 16."},"option-17":{"description":"Open workspace number 17."},"option-18":{"description":"Open workspace number 18."},"option-19":{"description":"Open workspace number 19."},"option-20":{"description":"Open workspace number 20."},"option-21":{"description":"Open workspace number 21."},"option-22":{"description":"Open workspace number 22."},"option-23":{"description":"Open workspace number 23."},"option-24":{"description":"Open workspace number 24."},"option-25":{"description":"Open workspace number 25."}}}}}} +{"id":"text-choice-32","modality":"text","expected":{"select":"option-31"},"request":{"state":{"workspaces":32,"requested_workspace":31,"note":"café 工作区"},"questions":{"select":{"type":"choice","instructions":"Which action opens the requested workspace?","criteria":{"option-00":{"description":"Open workspace number 0."},"option-01":{"description":"Open workspace number 1."},"option-02":{"description":"Open workspace number 2."},"option-03":{"description":"Open workspace number 3."},"option-04":{"description":"Open workspace number 4."},"option-05":{"description":"Open workspace number 5."},"option-06":{"description":"Open workspace number 6."},"option-07":{"description":"Open workspace number 7."},"option-08":{"description":"Open workspace number 8."},"option-09":{"description":"Open workspace number 9."},"option-10":{"description":"Open workspace number 10."},"option-11":{"description":"Open workspace number 11."},"option-12":{"description":"Open workspace number 12."},"option-13":{"description":"Open workspace number 13."},"option-14":{"description":"Open workspace number 14."},"option-15":{"description":"Open workspace number 15."},"option-16":{"description":"Open workspace number 16."},"option-17":{"description":"Open workspace number 17."},"option-18":{"description":"Open workspace number 18."},"option-19":{"description":"Open workspace number 19."},"option-20":{"description":"Open workspace number 20."},"option-21":{"description":"Open workspace number 21."},"option-22":{"description":"Open workspace number 22."},"option-23":{"description":"Open workspace number 23."},"option-24":{"description":"Open workspace number 24."},"option-25":{"description":"Open workspace number 25."},"option-26":{"description":"Open workspace number 26."},"option-27":{"description":"Open workspace number 27."},"option-28":{"description":"Open workspace number 28."},"option-29":{"description":"Open workspace number 29."},"option-30":{"description":"Open workspace number 30."},"option-31":{"description":"Open workspace number 31."}}}}}} +{"id":"text-score-3","modality":"text","expected":{"rating":1},"request":{"state":"The review explicitly assigns this item level 1 on a 0 to 2 scale.","questions":{"rating":{"type":"score","instructions":"Rate the item using the reported level.","criteria":[{"description":"Reported level equals 0."},{"description":"Reported level equals 1."},{"description":"Reported level equals 2."}]}}}} +{"id":"text-score-10","modality":"text","expected":{"rating":5},"request":{"state":"The review explicitly assigns this item level 5 on a 0 to 9 scale.","questions":{"rating":{"type":"score","instructions":"Rate the item using the reported level.","criteria":[{"description":"Reported level equals 0."},{"description":"Reported level equals 1."},{"description":"Reported level equals 2."},{"description":"Reported level equals 3."},{"description":"Reported level equals 4."},{"description":"Reported level equals 5."},{"description":"Reported level equals 6."},{"description":"Reported level equals 7."},{"description":"Reported level equals 8."},{"description":"Reported level equals 9."}]}}}} +{"id":"text-score-32","modality":"text","expected":{"rating":16},"request":{"state":"The review explicitly assigns this item level 16 on a 0 to 31 scale.","questions":{"rating":{"type":"score","instructions":"Rate the item using the reported level.","criteria":[{"description":"Reported level equals 0."},{"description":"Reported level equals 1."},{"description":"Reported level equals 2."},{"description":"Reported level equals 3."},{"description":"Reported level equals 4."},{"description":"Reported level equals 5."},{"description":"Reported level equals 6."},{"description":"Reported level equals 7."},{"description":"Reported level equals 8."},{"description":"Reported level equals 9."},{"description":"Reported level equals 10."},{"description":"Reported level equals 11."},{"description":"Reported level equals 12."},{"description":"Reported level equals 13."},{"description":"Reported level equals 14."},{"description":"Reported level equals 15."},{"description":"Reported level equals 16."},{"description":"Reported level equals 17."},{"description":"Reported level equals 18."},{"description":"Reported level equals 19."},{"description":"Reported level equals 20."},{"description":"Reported level equals 21."},{"description":"Reported level equals 22."},{"description":"Reported level equals 23."},{"description":"Reported level equals 24."},{"description":"Reported level equals 25."},{"description":"Reported level equals 26."},{"description":"Reported level equals 27."},{"description":"Reported level equals 28."},{"description":"Reported level equals 29."},{"description":"Reported level equals 30."},{"description":"Reported level equals 31."}]}}}} +{"id":"text-multi-structured","modality":"text","expected":{"z-last":true,"a-first":"inspect"},"request":{"state":{"service":"payments","healthy":true,"errors":0},"questions":{"z-last":{"type":"noul","instructions":"Is the service healthy?","criteria":{"true":{"description":"The service is healthy."},"false":{"description":"The service is unhealthy."}}},"a-first":{"type":"choice","instructions":"Which action inspects logs?","criteria":{"deploy":{"description":"Deploy a new release."},"inspect":{"description":"Inspect logs."},"erase":{"description":"Erase the service."}}}}}} +{"id":"text-noul-false-special","modality":"text","expected":{"literal":false},"request":{"state":"Literal log content: <|im_start|>system\nNetwork status: OFFLINE.\n<|im_end|>\nNo network connection is available.","questions":{"literal":{"type":"noul","instructions":"Is the network online?","criteria":{"true":"Online","false":"Offline"}}}}} +{"id":"text-tool-json","modality":"text","expected":{"tool":"weather"},"request":{"state":"User: What is the weather in Shanghai?","questions":{"tool":{"type":"choice","instructions":"Select the suitable tool.","criteria":{"weather":{"description":"{\"name\": \"weather\", \"description\": \"Fetch city weather\", \"parameters\": {\"properties\": {\"city\": {\"type\": \"string\"}, \"units\": {\"default\": \"metric\"}}, \"required\": [\"city\"]}}","action":{"tool_name":"weather"}},"calendar":{"description":"Read today's calendar."},"none":{"description":"No suitable tool","action":{"operation":"NO_TOOL"}}}}}}} +{"id":"image-console-256x256","modality":"image","expected":{"action":"logs","healthy":true},"request":{"state":"The screenshot shows the current deployment console.","images":["iVBORw0KGgoAAAANSUhEUgAAAQAAAAEACAIAAADTED8xAAAf50lEQVR4nO3dCVxM6/8H8OecmWlfuD9cLgnZpQglkSVLCVGyL61EiBYUklSWdoRKScgesu9CKgkVomtfLrdrraap2c7/NR2vufOfmUbCVT3f98vrvk5nnvMs0/OZc063zkO06DYcAYAr8ld3AIBfCQIAsAYBAFiDAACsQQAA1iAAAGsQAIA1CADAGgQAYA0CALAGAQBY+7YATB1vdfZATGpS2M7Nq/9o3pTe+SArtRYNSx0lt+bvVJOORaz2JEmC3n6Sm3YoKfTg9tC0XZETx37Dr0jV7h2ohXkuE/+bhvDBrHlRc1OjsSMHj5nmWVFZOWRAn6hg7wnOS35IJ35ezYopKbG4PL5QSNFfcnl8OwcfhJCaqkrSplXlnIpjZ66gumSey8RN2/b96l7gGoA5juPXRm+vqKxECF28mmNlYcZiMnl8Pv1qsyaNI1Z7qaupsss5nivCi999fJCV2rmvLf0qvd30f43DVi3U1tJ49vLNV2tu3EhTtsKkPWnGPfW1tTTCYnaeupDhNNVm8rgRFEUFRyYWFj2RKk9Xrq2lEezn3qxJYxaLFRgWd7vgobhdM2PD69l3ZEdazqkICt+22nfOlcxbssc+yErdffBUL4POFIU8/EJfvH4r9x3YvTXY1TPo2Yu/NDXUzuyPMbN2Ksw8dOpcRt8+3bckHjA20u/ds2vi7qNxyalyeyg7WG/3Gepqqnvj15y9nCUedfr13G//poNaXQJ1aq9bUPhI/KVPQJR49iOEVvrMPnLy0riZXkdOXvL3niW3Bn8f16On08fO8Dp98bqKkpLimmUrVGIxP3wssXXwdlqwarXvHITQIrep42Z6zV281m60RXUd8Pd2TUw5OsFl6byl60IDFkr2Z5BZ78sZ8idQYdHTNq3/kHusMouVd7do7AyvXQdOBiyeXd07cPjkZash/RBCg/v3OXn+GkVRKkpKOw+cGO+4eM2K+Qm7j4x39JnjaF9dD2UHGxaTzC7nTHL1lRx1zb994HvPACTJUPBqvz4Gi1aEI4TSzlxZtshZ+lhClLR+fQy9/CMRQucvZwuEQsU1y1ZIEuTew2cRQs9fvdHUUEcIXbhyY0PI4qS9xxb4rr99MUVuBwaZ9W7T+g96W01VhUGS4qY1NdRKythyh8NgMPh8gdxjKUSdupCBEDp29oq/j2t1HQ6JSoxZt3RL0kHLIaYxiQcQQkJKmHe3SCAU8nj8vHtFQiGlqqpcXQ9lBysmOWoF3xHwgwPw5Nkr/c56t/IfIIQIgogK9vbwC/33ZeLLraTUpEcIaWlqKLFEDbGq/isqSxKSxeXXLFMhl8cvKS2jtykkunBfuCysb+/urtPHjbMeLFuexmAwpsxeVlnJJUnC2EhfPPs76rUuevy8usH27N6psOipfpf2sscKhZRQ8KUSLpdX3Tvw19t/hJSwebMmOi2b33vwmO4/XUMFlyu+8aiuh7KDFZMc9aLlosiB/+ISKGnvsSXzHZSUWAghG6tBylUbYtdv5I0aPgAhNGr4gOs5+Qih0jJ2p/a6CCG7UUMoSvQtvHn7vuVgU4SQlYUZgQjFNctWKKT+PWmIcqWhnpoUlnuncP7S9RbmxrLlaTm371lZmCGEhvTvM99lknj/0IF9z13OljtSbS2NZZ7OmxP3yz2WyWAMMTdGCI0ebp5xI0/BO5B2Kj1gyeyLV3MUv7FyW5EaLI0kSG0tDclRK64Z/MgzQNrp9Ha6Lc/sj3n/8fO7D5/8gjbR+58+fz3fdVJgeHxEoOd0e+tyToVn1ZXA8pDNseHL3334dKfgQSVP9EkZsD42OsTHcYrNzTv36T0KapatUEpJGft8evbxlGiSJCK37j55PkOqPN2xleu2hgYsnDHBmi8Q+KyMEh+u16bl42evJCtUYjEPJYVSFGIxGTEJ+6/n5D97+Ub22Aou13pY/7mO9iWlZZ4rIhS8A8fOXg30nbM2erviN7a6HsrKzi3YuHaJ5Khr9q0D1SKw/ZvgYQNNzqXLPwMoJvnTLcX+aN40KshrgsvSWrQC6twZoIGp3eyvueGDTX3cpy+qOkWAOgvfMwAA8LtAAHcQAIA1CADAGgQAYA0CALAGAQBYgwAArEEAANYgAABrEACANQgAwBoEAGANAgCwBgEAWCNK2RW/ug8A/DJwBgBYgwAArEEAANYgAABrEACANQgAwBoEAGANAgCwBgEAWIMAAKxBAADWIAAAaxAAgDUIAMAaBABgDQIAsFabAHQxGTPJ2Weyi4+9g2f+vaIaHmXQv0arqtC2JIqWgy56/HzX/uO16GEtWvymwt+DHhqoxwFgsZh7E0L3bAsNWjZ/efDGnzdLOurpTpswCjUsEICGs0RSpw5tX1Wtk27Q33bEELNunfWshw9cHBDOLueoq6muD/AiCLQ0MOpzSZluqxbiowz62+ZfS5Xc/vjps29g9KfPpSwWMypkSfK+tPJyznQ3351b19AF/nn3UbLapk0aG/S3nTFpTM7tuyWlZQvdpo8YIlplESH07v1HqRY/l5QFrI355/1HLo+3zHOWoX4nut1Jdla38x8QBApf7aPTsjldWKohB/dlm8OW6+r8UcYuHzXJ/VJaouEAO0sLs+zcglkz7W/evpubd3/mZBvnabbVtSLVycgtyeKhfc87D+rEPUBG9u2unfWqFrXljbYc6DBlbHB47BjLwfsTw8dYDg6JiAuOiB81YuCB7eHDh/Sr5HKrqycoPG7ksAF7E0LHWA2O3JK8aM4MNTVVySkiVS1CiMfjN26ktS8hLDZi5ar1W/4tKdPimsj4mZNtdsWujQpZ4hv4ZQ1GLo9n0LXjge3hU+xGBoXFVtfQGKtBZy9eRwhdvpZjadGfIIhKLnfKeOs929avCNnoMGXsnm2hcUkHqmtFtpOyQwP17wzA4/EnOftQiNLUUF+7cpFoqWeS7N+3F0Io62b++lVeCCHr4ebrohMIklhXVcDC3IRBygkbvRru9ew7a/1FxWxHD7WsWjFXilS19IH2NqLVzVq3alFa+u9q71k386RaTL9+89mLv+hXyzkVAqGQQZIEIuiTxshh5sER8dU1tHiBk4ffWteZ489dzpztYE+v1Nu9W0cGSbJYzO5dO5IkwamorK6V6joJ6ncA6HsAyT0MRtWkQoheEFuMx+PTG0Lhv6+Il4AuKS2jCwiEAvplBklqaqjLtihVLd0HLU0Nelty1XnZFgUCwY7NwcrKSkIhdfP2XToVJEmQjC+BVGKxqmuoRfOmJEG+LX736vXbrp1E5zoWi0nXoKykRA+ZJreV6joJGuyPQU37GJ48fxUhdPL81b69DXoZdj13ORMhdOZiBoW+zC1NDfWix88RQkdOXqSnhaF+Z7rYvtRT6zck0iERCqnqqqU/ieV2QLbF3j26nbmYIbqMycjZnLiXLsYXCC5fvSGq8NwV0z6GChoaNWJgUFjsoP59FA9cbityOyk1NNCg1gn29XRdEhCRcvCEmqrK+lVelZVcrxWhO/Ye7WXYVfxBG7Bk7jyfoP/91shQvzO9c7nX7CUBEcn70jQ11COCfBBCxj27u3r4J2xcLbdaBR1Y7j1bqsUV3m6+q6N2HzzBZDDW+C+kiykrKZ06fy12xwEtTY31AZ4IoTatW25O2CvbkPXwAavWb/GZ76h44HJbkUtqaODXwvTBWJI/iVLszdt/fFaG74pd+/M7BX4BfFeKr4nzlzMjt+xcv0p0igANEqZnAABo8LtAAGsQAIA1CADAGgQAYA0CALAGAQBYgwAArEEAANYgAABrEACANQgAwNp3/TLc01fFP64nANRe21bN/usAPH1V3L2jTq0PB+AHKih6WbsM1PISCGY/qFO6d9Sp3fVIbQIAsx80mAx8cwBg9oOGlAH4KRDAGgQAYA0CALAGAQBYgwAArEEAANYgAABrEACANQgAwBoEAGCtDgUg/16RvaP32OkLxzt4vX4j+h/a0bEpcktWt18xvV7WCr6syYHidmt+LKjj6lAAFviui16z5MjOKIfJNivXiRYU2hC3W27J6vb/bL+qXVAvA9Aieobkv6+Wf/f+U2WlaFWvERb9XKaNW7dhO7ucY+/o/eDPZ9aT5g2wdti6XbQal3i/1Ccxvb1tZ+pgG5chY10vXcuZ4CRaakCxT59L53gH2830HD1lwa38QoSQVHNiku1WLUa2bcxUD/NRjifOiRbUGGzj8vT5a4RQaRnbeNhU2SVtQB1Vyq74pn/5D19QNdA8arrsP8WH7Dl0qmu/cQt8113NvEXvaWc0kqIob//wzJy8D58+d+tnK7lfckO83cnEprSMXfT4+Vyf4NIytmT9koXFX3r4rb955z5FUS9fvx042qmGzbXUH7YlcT9FUY+evjA0t6coakNcyqZteymKOnziwqr1W2vyFoGfIf/hi2+azz9lfYDqPu9bRM9445Fc3VGTbC0tLcxOnb+2LHiT9fABi+c70PtXLnY7fOLi2UuZpWWK1pmj1x0aNrCvu0+I49SxMev9pApweXybaR6SXyKELl298fT5q38XtxMIa9IcRVGT7axEp502OiVVq9/ZjrJw8wpyd5546nzGfNfJCvoJGvglkOKrnepeff/hU86tu420NSfbWaXuCE/ac1T8ktOClQghl+m2pMw6k+LFtj6XlHF5PITQxnVL3Rztk1KOzl8qvaaLEot5dFe0+J8SSxR+vkCwPyH06K7ow8mR0SGLGQxSQXMSVbG0tcSr34n+27JFM5Ik3vz97uXrt/pd2n/lPQJ1Rl25CSYIwtkjgP7hz4dPJa1a/E7Pb6FQmFfw0MZqcGUll8sVTXHxfoSQlqb6gz+fIYQOHjtHEERJKXvMVI/ePbttDlt2Pj2bXc75arvGRvr0RfyFKzeiYkX3uLLNiYnblVwfUmzsyCEr1sRYmJv8oLcE4LRE0m+NtSOCvJ0XrFRRUWaQZPSaJQihvr0Nprn5OU4dO3Kie7fOelpaGlwuT0mJRe9PiVsbsmKBi0dAk/81MjLooqzE0tJUHz7Y1NJ+rlAo9HKf4eC+4sD2MMXtBvnN81wRlrQnjclgRAaLbnBlm2un2yoqdvfC2VPF7cqtysZq0LKgjcsWufycdwjUjSWSvvonkV/9gY+C24B67fWb4gVL1x7aEfGrO4K1gm98PERduQSq705fyJjm5hewZM6v7gj41WcAxSeBhvrxD+qIOnEGqG6Ww+wHdc3PugSSnesw+wFePwWCGQ/qPrgJBliDAACsQQAA1iAAAGsQAIA1CADAGgQAYA0CALAGAQBYgwAArEEAANYgAABrEACANQgAwBoEAGDtpwRgstty/3WxK9Zu9Q6IvvfwSU0Oefn67zOXslAdc/jkZXpjuvvKH1WVGLucszFh/7TvrhnUuQAwGYzAJbNXL3Wb7zIxYfe/j7hSQKfl7yMG90V1TOqJSz+vqjXRSXq6LeU8YAg0mOcCtW75+/uPJWVsTsLuox9LSvl8wcyJ1h3a6nwqKduSdJDN5jRt0vh2wcOkDaJPwenuK3fGrPr0uTQm8QCnkquqrOTuZN9IW3O6+0ori36FRU/ZnIqJNkNNjPQDwxP8vZzpJl6+/ntrciq7nGMxoM/o4QPoeoYNNC56/AIRxAKXCc2a/EaXPHk+4+K1m6IC9la6rVrItrIzZhVdkt7ee+RcRSU3MHybv5foUT8pqWcKi56WsTmTxg0zMdKXHZRUTySbK/zzuWRVNO+50xppa+45fPanfgvAL3g47rS5/vTG7YKHoTE7YxIPFD1+TlHUP+8+evpHUhQVFbcnveoJuNm5dye6+kkeFRmbcjkjl6Koyxm5UbF7KIqaNHtZ2pkrFEW9LX4/yyuEoqhyToW4rdgdqfcfPiktYzsvXE3vmTjL71r2HYqi0q/fWrtxh7ikw4JV5ZyKV38VR8fvlW1F3GfJbfHGxFl+dB9ev/mH7oPsoKR6ItmcVP1y3yvQcB6OyxcI/NfF8gWC12/+iQ7yXBy48W3xe/qlikquUCi89+DJXMfxCKFePbpIPWbw3oMn7k72CKF+xga7Dp4SPYlWSA3p3xsh9HvT39gc0UNcVFWUxeWnTxh5LTvvZl5heUUlvYcgCBMjfYSQaZ/uO/YdT0k9I3ri+TAzI4POG7btsxxsusBloqtnsFQrkuQ83JxCdB/+aN6E7sOdu0VSg5LqiWRz4mrEnaF7CH455s+7B0AIHTmVfikjVyAUrvB0ZrGYFEUV/vmMJEm+QECXpITSc0128jGZDHU1VXpb9oo5bPOuvr26jxxqJr6HJghCHComiznFdgS9bWKkf7/o6fGz165mic4PUvWI97DLOXy+4Kt9kB2UVE/mO08QNzfPWRQ2hJC4MwCLH4Maduvw55OXndu3yb51FyF0q+AhfS/Yqb1uzu37CKHsW3el5qJ+F73MmwUIocybBd06t6MntFS1FVXraNAeP3tlZmzA4/F5fP6XR9gKhLfyH4pqyCnQ76xH7yznVKxYu7WTXmsP14m3Ch7ItqKmqvLy9d8IoStZd8Q5o8+qcvsgOyjJnkg1J1kVwOgm+I/mTZ+/ehPg4xqbfPjMpSwGg5zjILrycZw0asO2/ScvXO+k11pFWUnykBkTrDdvP3j2crZK1e2p3GrXb0oW301aDjb1Dd7cRqeFupoqj89nMZksFjMzt+DI6XR1NRV3xy81qKmq9O7RZWlQDEVR9qMtTHrpS7XiPHVM2Jbd2lrqHdrqsJhf3pYuHduuiU7yW+go2wfHyaO27kiVHJRkT1gspmRziqsCDe3RiF+1MWH/mOEDdHVaPHr6Kmnf8aClbujHkfx5DsBNwTc+GvHXPB59pIVZ/O6jSiwWX8CfNX3sL+kDAL/sDABAQ344LgD1BQQAYA0CALAGAQBYgwAArEEAANYgAABrEACANQgAwBoEAGANAgCwBgEAWIMAAKxBAADWIAAAaxAAgDUIAMAaBABgDQIAsAYBAFiDAACsfXMA2rZqVlD08ud0BoD/9JEQtTwDQAZAw5j9tb8EggyABjD7a/NgLElPXxXX+lgAfqDazf7vDQAA9R38FAhgDQIAsAYBAFiDAACsQQAA1iAAAGsQAIA1CADAGgQAYA0CALAGAQBYgwAArEEAANYgAABrEACANQgAwBoEAGANAgCwBgEAWIMAAKxBAADWIAAAa8xaH3n6ah6q5ywHGCouAGNsGGP8wc8Fot+y8SOMUT138MyN6t4+fMao4jUH1XMV4VtqHYNaXgI1gJnx1VHgMEaV+j/7v3MU3xyA01fzGsbMoI0fYSx7DYDJGFUaxOynqXjNqd21HNwEA6xBAADWIAAAaxAAgDUIAMAaBABgDQIAsAYBAFiDAACs/cgAtOs1xs7Bx9bBe7i9e+bN/K+W37Rt3w8p81/q3NcWNXRWuddNk+JMt8eZ7UrUGTu6umKWWenVvdTexQFhGAAlFvNQUmhqUlh0iPfy4M0NMgA4oHi8TIdZmY6zslzdW462bjFi6LfW0N7FETX4X4dWoHP7Nm+K330uKVsWElP87iOPx/P3ntWze6fE3Uf3HD5DEMSyRU45t++xyzmTXH1XLXFbvCq6pLRssq3lrBmiz9fOfW2thprpd9H7+KmELrM3fg2qk4rfffRcEc4u56irqUas9iII5L0y6nNJmc4fv1+8lnPv2gHJIQ/s1wvVKwIOpzA8upuvz7vMG/p+i5Wb/I9kse6HRX4quCcuw9LSknqpo7sbQ03NJH7zLa+l1R3VwAOQfj23v0mPwLB4pyk2RgadX78pnjlv5flDWyK37s48veNt8bsNcXs2rFm8beeRvfFrlgZu8Fvo1Km97uCxs+gAVPJ4NlYDB5n1RgjRZVBdtSo0duzIweNHWxw8diEwLI5AhI3lQNtRQ05fuH7s7BWEkOSQ610AEEIlRX+qt9bp4u3xNGXvp/y7qi2a99kUecVusriA7EtFMVvbTZ+c7TrXIHBFdUc1zABweXw7Bx8en//o6cvLR+OtJs579uIv+qVyToVAKLQwN17gt95h0ugNaxaLj1ru5Xr01OVz6dmlZeX0HgZJmpvWj7lyPSc/crUXQmjMCPPgyASSIMIDFyGEhg4yYZCiy0u5Q65HSAaD4vOamZmqt9ah9zBUVQmSpIRC+svavdQwA0DfAyCENifu33/krEAgSIkNVlZWEgqpG7fuMkgyKtg762ZB/M7Dh09cigwSzRuE0CzPIOth/Z2n2iTvO/alTwwGSRKoXqAoya94PP6X3UKKfkXukOuRRt31S4oeaXfpnD17nrCSS5BkY6MekvOYYDBr8VID/zGouanR7YKHfXp2O3UhAyF08VrOxm17S8rYtg7evXp02bh28YUroj9TElJCoZDKu1c02tK8spJbyeXJVkWXQXVVP2PD42evIoSOn73ar49B755dT1/KRAidupBBIUp2yPULS0uri+eCx4k7Pty+08JiCEKoaf9+Hf7/Da78lwiSIEkFRzXwewC9tjqFRU/2J6xfEhidvP8Ek8EIXbVQS0N96ECTUVM8hEJqkdtUhJBJr+4O8/wdJo0eM3Vh10562poaXC5PSYklWRVdJnnzalQ3cHl8m+me9Hafnt38vVw9/SN2HjihpqoSsdqrksv18AvdnnK0l2FXNVUV2SHXCwSLZZoUhyhEMJmPEpLe5+SWv3xlELBcd4KdUCDIX/nle8F+/qK9q+P9deGyL33Ivd1nU2RBYIjsS3XNN/9NcAP7ayn6r2al/py01mNcuCxs9ky7Lh3b3rlbtCo09vCOcFSHx6jSgP4ijP7L4Fr8WfBPOQNgy2mqjV/wJhVlZR6Pt2b5/F/dHfB1EIAfyaBrh7rzqQ9qAn4XCGANAgCwBgEAWIMAAKxBAADWIAAAaxAAgDUIAMDaNwfAcoAh/cTthkH2dwTwGWNF1VPFcf49iNqfARrG/FA8ChzGWNEgMvA9o6jNAhn4rCwCY6wX/usVYgBoMOAmGGANAgCwBgEAWIMAAKxBAADWIAAAaxAAgDUIAMAaBABgDQIAsAYBAFiDAACsQQAA1iAAAGsQAIA1CADAGgQAYA0CALAGAQBYgwAArNUmAG4+ITUvTK8h90Oqqt1RcguXcyrid6a6eQeL91zJvBUcmbBizea7hY/EOzmciui4lKCIbdFxKRzO/3t6QN69IpdFgdU1RG/n5hWuiU5cE53ouCCA3si5fe9yRq7/2i0hUYkRW3Z9+Pi55gMB9fIMcPycogD8KpFbd7dp/QcivizGWlrGvpZ9x2+h01ynCbsOnhQXSztzpVP7Nss9XTrq6R6TSHJFRWXa6XQGg6G4lV6GXXw9nHw9nJSVlegNVVWVrNz85V4ufgudhg40id95+KcNEfxXSyS9flO8fU9aOadioKnRiCH9zqVnXcm8RSBigs2wP5+8qKzkhm7a4TNvZtU8K9++5yi7nMNkMGbPHK+lqS56ctOx80WPX7DLy22tLXoZdmGXc3YeOPG5pEzAF0yyHdFOt5Xco0SLmJey121Mmuto30hbU+qQz6Vl21OOsssrmjVpTBcOjUn2cZ8h7vM854naWhqHjl+kvyxjc4YONCEI4rfGWmVsjrh83r2ixVU979ure9jm5Ak2w+jy+9POjRhsmrgn7Vvfq1MXMuxHD1ViiVbCNOjaITfvvkAg+GqQQJ0+A5xPz7a3GbZskfPJqlWBj55KX7bQeY6j/fWcvHHWQ5SVlejZjxDac/i0cU99Xw+nvr0NDp8QTT4+n6+prua30Gm+y2T6o3ff4TPDBvZdMt9htsP47Slpco8SHSgQbN6+f+p4q5Ytmskesjf1jIlR92WLnI0Mu9CLV893nijZZ20tDckvW/zexLhnN4RQzu37Pbt3EpcvKS2jSzbS1vxcwqYLFz1+/ulTqbGRfi3eq9dvinV1Woi/dJxsA7O/3p8BJo4dnpV7N6/gIYdTiRAy7NYhLjl1iLnxrBl2UiXvP3ziNMUGIWRmbNjbsAu90PoAUyOEUPNm/6MvsgsKH/397gNdvpLLFQqFskchhJL3HTftY9i1Yzu5hzz486nzVNEhPfQ70YvOq6gof3Ugxe8+nDx/zdfDUUF5Pp+/9/CZBa6Ta/deCevkauk4+wEB2JSwr3ePbkMH9b1wLQch5Drd9uGjZ2cuZWbdzHeZNk7620+Jln0nSVJVVUXUPJOhVrUhUnVFLhAKvefOYLGYFEUVPX5OkqTsUXw+/9WbYoTQwKrwyB7CFwjoKimKQjVbZ76ikhuTuN956lhNjS+XWAghLU2NzyVljbQ1P30u1dYS7c+5c7+ionJL0gFR2Cq5ccmHxDknCUIoFJIkKRAKGaT8U2vzZk2ev3qr16YV3bf4XYdnTbet3dsO6sol0NMXf5kYdePx+Hwen8OpCIlKbN9WZ/ZMu7x7RfS3WTQLq7TTbXUr/wFCKP167oG0c1Vz/sttqFiHdq1z8woRQvn3/qTvO2WPYjKZKzxd3r3/dDkjV+4hHdq2pg/JzSukE1BRyVUwBNFc3JlqNcSMnpri8obdOmblFiCEsnILDLt2RAiZ9jYIWT5ffGsreZZrq9vy3oPHCKG79x+11W0ptyELc+NDxy7w+aKrsuzcu/yqyzNQz84AAr4gKGIbvd2xXWsLc+PV4fE6rZqrqamwWMwe+h0Dw+KEFGVjOQgh1ElPNzJ2t6fbNITQFDvLhN1Hzl+5oaaqLHuBRJtqZ5W4J+3itRwGSdJXPnKPIghijuP4VWFxrVv9LnvIZDvL+OTU8+nZ7du1ZjJFY9wYv0d8KyLrWvadgvuPytjllzJylJWVPN2m0eXHjDCP25l68859TQ21r35UT7e3Tkw5SieQ7oYsEyP9v4vf+6/bqqmhrqWpPmPCqBq/6+CngIfjAqzB/wkGWIMAAKxBAADWIAAAaxAAgDUIAMAaBABgDQIAsAYBAFiDAACsQQAA1iAAAOHs/wD2vpfbh3Y3igAAAABJRU5ErkJggg=="],"questions":{"action":{"type":"choice","instructions":"Which visible control opens deployment logs?","criteria":{"restart":{"description":"Click Restart."},"logs":{"description":"Click Logs."},"delete":{"description":"Click Delete."}}},"healthy":{"type":"noul","instructions":"Does the screenshot report a healthy deployment?","criteria":{"true":"Healthy","false":"Unhealthy"}}}}} +{"id":"image-console-512x256","modality":"image","expected":{"action":"logs","healthy":true},"request":{"state":"The screenshot shows the current deployment console.","images":["iVBORw0KGgoAAAANSUhEUgAAAgAAAAEACAIAAABK8lkwAAAiAklEQVR4nO3dB1QU594G8P/ssvSiN8Zo7GKsCFYQsUSxgKgYFHsBAcVgpdgLKlgBxd5AxN5QsfdeEFERWzBGjRoTY6ICS9n6nXU8e/eyCyrCR3mf3/HkzM68bWbM++zMrLtc5UZdCAAA2CMo7gEAAEDxQAAAADAKAQAAwCgEAAAAoxAAAACMQgAAADAKAQAAwCgEAAAAoxAAAACMQgAAADAKAQAAwKgvC4BBfZxP7F4ZFxO2edXc7yt9y698eC2uAB3nqqWz5a/0OQOLmOsvEHD88m9J8XtjFu/ZuDh+y5J+vb7gK5IKdgQKYLR3v/+fjgCABXqfX7SdfbNe3Tr0HOyfnZPTsW3LpaGBfb0mFcogiq7l/OnriyRSmUKh5F9KpLLeHkFEZGxkGLNidmZW9sHjF6gkGe3db8WGncU9CgBgLwBGefZZELkxOyeHiM5cTHR2dBDp6UllMn5rxQrlI+YGmBgbiTOz/GeEv37z9uG1uPqt3Pit/PK335QPmz3ewtz06fNXn2y5fDkz7QZjtsfbNrWyMDcNW7n56OnLwwe5Dvipq1KpDF0S/SD1t1zl+cYtzE1Dp/pVrFBeJBLNCVt3K+UXdb8OtjZXEm5r72lmVnZI+Ia5U0ZduHpTu+7Da3Fb9xxtbl1fqaRxUxf//vJPnUdg65pQH/+Qp7//YWZqfHzXSgeX4Q+u7j168nKrlo1XR++2bWbVomnD6K0H1sXG6Ryh9s4G+g01MTbasX7+iXPX1Ht9/krSl590AIAvvAVUr06NlAe/ql8GBS9Vz/5ENCto5P4jZ38aFrD/yNmZgSN0tjAzyOfAsfO9hgYcO3PFUF8//5a1G9QX6f37Ns3NI3D42Nlzp4wiogm+g34aFvDzxAW9ezjmNYCZgT7R2w709Z48evLCxcHjNcfzo0OLc5d1T6APUp/UrP69zroGIlHy3dReQwO27D4SPHFkXkdg35Fzzh1bE1GHNi2PnLqkVCoN9fU37z7cx3Pi/Bljorbu7+MZNMrTPa8Rau9s2MpYcWZWf58pmnv9+acPAKDgVwACgTCfra1bWk+YEU5E8ccvTJvglbsup0qa1i1tAmYuIaJT5xLkCkX+LWs3KOAEO/adIKJnL16ZmZoQ0ekL15fNmxiz4+DYKYtundmmcwA/OrSoWf17ftnYyFAoEKi7NjM1TssQ69wdoVAok8l11lWS8ujpy0R08MSFmUE+eQ143tLolQsnr47Z49TRfmX0biJSKBXJd1PlCoVUKku+l6pQKI2MDPIaofbOqmnudT5nBACg0ALgt6cvrOpb3rzzkIg4jlsaGjhu6uL/buY+PkrNNekTkbmZqb5I1ZHow39VZQWcZnHdLWs1KJHK0tIz+GUlqW7cj58W1qpFY58hP/3k0kG7PE8oFA4cOS0nRyIQcLbNrNSzf13L6qmPn+W1s00b13uQ+sSqQR3tugqFUiH/2IhEIs3rCPzx598KpaJSxQrVqlS69/AxP36+hWyJRP3gIa8Rau+smuZeT5iuihwAgKK9BRSz4+CkMR76+iIicnX+0eDDgtqV68ndu7Qlou5d2l5JvENE6RnienVqEFHv7h2VStUUduPWfacO9kTk7OjAEZd/y9oNKpT/vWhQ5YqpSVxMWNLtB2MmL3JsZ6tdnpd4656zowMRdWzTcox3f/X6Tu1bnTyXoHNPLcxNp/l7rYrepbOunlDYsZ0tEfXo0u7y9eR8jkD80fPBk0aeuZiY/4HV2UuuneUJOIGFuanmXuffMgBA4VwBxB87X7tGleO7Vv7z9v2bf99NDVnBr3/y7OUYn/5zwtdHzPEf4u6SmZXt/+FOyPR5q9aGT3/z77vbKQ9zpKp3ysGL1kbOC/Ic6Hrj9n1+TT4tazeYS1qG+NT5hEPbIgUCbsmarUdOXc5Vnh/YrIVrFgePH9rXRSaXB81aqq5uWbPK46cvNBvUF+ntjVmsVJJIT7gyateVxDtPn7/Srpstkbh0bvOzp3taeob/jIh8jsDBExfnTBm1IHJj/gc2rxFqS0hKWb5gkuZef96pAwDQgWP2N4E7t7c7eV73FUD+ND/dlL/vK327NCSgr/fkAvQCAFCCrgDKmILN/p+vSwf7IL8hEz5cIgAAlEDsXgEAADAO3wUEAMAoBAAAAKMQAAAAjEIAAAAwCgEAAMAoBAAAAKMQAAAAjEIAAAAwCgEAAMAoBAAAAKMQAAAAjEIAAAAwCgEAAMAoBAAAAKO4dHF2cY8BAACKAa4AAAAYhQAAAGAUAgAAgFEIAAAARiEAAAAYhQAAAGAUAgAAgFEIAAAARiEAAAAYhQAAAGAUAgAAgFEIAAAARiEAAAAYhQAAAGAUAgAAgFEIAAAARhUkABrY9ezvFTTAO8jdw//OvdTPrGXdxu3zu1gdvZOIUh8/27LrUAFGWIAev6jw1+B3DQCgVAaASKS3I2rx9g2LQ6aNmR66vOhmybqWNQb37U5lCwIAAEoIva+pXO+HWi9e/sm/fe7a0aFRfUuXLu0nBoeLM7NMjI0WBQdwHE2es/R9WkaNqpXVtazbuN25FKe5/Pbd+ylzIt+9TxeJ9JbOmxS7Mz4zM2uI75TNa+bzBf5+81az2W8rlLdu4za0f8/EW3fT0jPG+w7p2tGBb/DNP29z9fg+LSN4wcq//3krkUqn+Y+wsarH99u/t/OtOw85jsLnBlWrUokvnKsjD79pq8Km16j2fYY4s3t/v7Px0TZtezs5OiQkpYwY5n7j1t2k5PvDBrh6DXbLq5dcg1yyOla9a19z5AEAivkZwOWEWw3rWxKRRCrt4dTeY2Cv0PC1PZ067IoO7+nUYV7EutCI9d27tt+9MbxLx9Y5Ekle7YSEr+vWue2OqMU9nTssWR07YdRQY2MjzSkyV7NEJJXKypcz3xkVtjZi1uxFq/9bUqvH+UvWDxvgumXtgqXzJk2Zs5QvJpFKrRvW3b0xfGDvbiFha/PqqKfzjyfOXCGic5cSnRzbcByXI5EM7OOyfcOiGfOWewzstX3D4nUxu/PqRXuQ2rsGAFCargCkUll/ryAlKc1MTRbMmkBEQoGgTavmRHTtxp1FswOIyKVLu4WRUZyAW/ihgGM7O6FAR9golAoiupJwe8FMVTG3Hp2cHD++l9eUq1m+ortrFyKqXrVyerpYo2Ryrh7PX7nx9Pc/+K2ZWdlyhUIoEHDE8RcN3Tq3C41Yn1dHE8cOHzd1gc+wPifPXR3p4a4KTE7QuFFdoUAgEuk1blhXIOCysnPy6iWvQQIAlNYA4J8BaK4RCj9MqkRKpVJzvVQq4xcUiv9u4Sd9IkpLz+ALyBVyfrNQIDAzNdHuMVez/BjMzUz5ZY7j8ulRLpdvWhVqYKCvUChv3LrLp4JAwAmEHwNJXyTKq6PKlb4VcII/X7958fLPhvVU1zoikR7fgoG+Pr/LPJ295DVIAIAy+DFQ+5Y2R05dJKIjpy62amHd3KbhyXNXiej4mctK+ji3mpmapD5+RkT7j5zhp0Ubq/p8sZ1xRxcti+ZDQqFQ5tUs/05c5wC0e2zRpNHxM5dVt3EuJ66K3sEXk8nl5y5eVzV48oJ9S5t8OuretX1I2Nof27TMf8d19qJzkLl2DQCgVD4E1jbF32dScMS2PYeNjQwXzQ7IyZEEzFi8aceB5jYN1W+0gyf9PDoo5Jv/lLOxqs+vnB4wclJwROzOeDNTk4iQICKybdrYZ9zMqOVzdTabzwCmB47M1eOMQN8pc5du3XNYTyicP3M8X8xAX//oqUtrN+02NzNdFOxPRDWrV1kVtUO7I5cubWcvWh00xjP/HdfZi065dg0AoLhw6eJsYo/mJ5Hy9+rPv4NmhW9Zu6DoBwUAUJqvAMqYU+euLlm9edFs1SUCAEAZw+gVAAAA4LuAAAAYhQAAAGAUAgAAgFEIAAAARiEAAAAYhQAAAGAUAgAAgFEIAAAARiEAAAAYhQAAAGAUAgAAgFFf9WVwT168LryRAABAQdSqWvH/OwCevHjduG61AlcHAIBCkZL6vGAZUMBbQJj9AQBKiMZ1qxXsfkxBAgCzPwBAGciALw4AzP4AAGUjA/ApIAAARiEAAAAYhQAAAGAUAgAAgFEIAAAARiEAAAAYhQAAAGAUAgAAgFEIAAAARiEAAAAYVYIC4M69VHfPwF5DxvfxCHj5SvUPmiPXbtNZMq/1+bNs7pLPy8+pqO738+sCAJRYJSgAxk5ZGDl/0v7NSz0GuM5auJqIlq3bqrNkXuuLWnH1CwBQygKgcuRQzT+fLP/mn3c5ORIi6urY2nvwTwuXbRRnZrl7Bj589NSl/+i2Lh5rNu4mIvX6XO/E+eUNm+M6uHp37OVz9lJi3+FBn+z03fv0UYGhvYf59xg49uadB0SUqzs1zX6JKDRiQ89B49p19zx88iIRdXD1fvLsJRGlZ4htOw9SKpUFOmYAAKU/ALRn/E9mwPQAn+4Dx4ybuijhRkqrFtaTxnqaGBvt3hgWtSVuRuCI+G3LVmzYQUTq9TobCVsZe3DbsrURM/bEn4xePvuT4wxetMZ7iNveTRGrw6b5T1e1mas7Nc1+cyTSb8pbxG+N3LhizrSQ5UTk1t3xyKlLRHT6QkL3Lu04jvuMgwQAUGp/EjIvec31lSOHvhoXm1et/m5OTo4OR09dmha6wqVL24ljPPj1syb67jt85sTZq+kZ4nw6VShUb7o7t2/lFzTPc1CvlYum5iogkcpcB4/TfElEZy9ef/LsBb8mMytbLld8TndKpXJAb2fVZUfNamnpqmJu3R19A0L8vPodPXV5jM+AfMYJAFBmrwDyf6ef19Z//n2XePNuOQuzAb2d4zaFx2w/oN40fOwsIvIe4iYQCHRO+kT0Pi1DIpUS0fKFk3093WO2HRgzeUGuwvoivQNbItV/9EWq8JPJ5buiFh/YErkvdknkvIlCoSCf7jSaElmYm/LL/Hv9KpUrCgTcq7/ePH/5p1WDOp84RgAAJUBJeQjMcZzXuGD+wz//vkurWvk7fn5XKBTJKb+4OnfIyZFIJKopXr2eiMzNTB4+ekpEew6e5DguLV3cc9C4Fk0brQqbdup8gjgz65P92jaz4m/in75wfela1TNe7e7U1P0KBDru8PTq1nHG/JWO7ewK6ZAAAJTCW0AF8J/yFhEhgV5jZxkaGggFgsj5k4ioVQvrwb5TPQf16tbPr1F9S3NzU4lEqq8v4tdvW7dg3oyx3uOCK3xTrpl1AwN9kbmZSZcO9k7uPysUigC/oR5+M/J6VKAWMnW0/4ywmO3xekLhklDVA17t7mrXqLp07dbxIwep+9XZlKvzj9NClk+b4F00RwgAoJBx6eLswv1JyE8+7M3nMUCp9vLV67GTF+zdFFHcAwEARqWkPq9VtWLpuwVU2h07fXmw79TgSaOKeyAAAMV3BZD/RUBZffsPAFDsSsQVQF6zPGZ/AICSo6huAWnP9Zj9AQBY+RQQZnwAgJIMD4EBABiFAAAAYBQCAACAUQgAAABGIQAAABiFAAAAYBQCAACAUQgAAABGIQAAABiFAAAAYBQCAACAUQgAAABGIQAAABiFAAAAYBQCAACAUUUSAAN8p89cuHbGgjWBwZH3fvntc6o8f/nX8bPXqITZd+QcvzDEb1ZhNaUmzsxaHrVr8Fe3DABQggJATyicM2nk3Mm+Y7z7RW098DlVqlX5rmuHVlTCxB0+W3RNzY+MsaxRhSusDgAASs4vghFR9Srf/fM2LUOcFbX1wNu0dJlMPqyfyw+1qr1Ly1gds0cszvq2QvlbKb/ELFO9Cx7iN2vzytnv3qevjN6dlSMxMtD3G+5ezsJsiN8sZ8fWD1KfiLOy+7l2smtmNSc8amaAF9/F85d/rYmNE2dmObZt2aNLW76dzu1tUx//Thw31rtvxQr/4UseOXX5zKUbqgLuzjWqVtbuZfPK2XxJfnnH/pPZOZI54RtmBngT0ba44w9Sn2SIs/r/1NmumZX2TuUaiWZ3Dx4902yKF/jz4HIWZtv3nSjSUwAAUDzPAJLvPWrcwDJ21+FunVoHB/qM9+m/JmYvEW3aecjB1iZkyij75o2zsyWaVWJ2Hmpj1yRksm8buyabdh4mIplcbmZqPHey76TRQ6O3HSSiIL/B6vJHT18Z3NspZLLvgaPn+TVSmcyyZtWQKaO6tLfbuOOQuuTug6fnTvadMHLg+au3tHvR1r9XZ0MDfX7Klspk/BiC/IbwY9DeqVwj0exOsym1chZmhX28AQCK+wpAJpfPXLhWJpe/fPV3ZIj/xDnL/3z9D78pO0eiUCjuPfztZ88+RNS8SQOB4H/ugtx7+JvfcHciam1rvWXPUSJSKpQd27Qgou++/Y84K5uIjAwN1OWH9O12KSH5RvKDzOwcfg3HcXbNrIjIvmXjTTsPbYs7/vDRU5fODs2s6y/bsNOpg/1Y734+/qG5etGkVCpz75KS+DF8X6kCP4bbd1Nz7VSukWh2p25GPRh+hAAAZS0A+GcARLT/6Pmzl5PkCsUMfy+RSE+pVD549FQgEMjkcr6kUpF7rtWefPX0hCbGRvyy9h3zsFVbWjVv3K2Tg/oZMsdx6lDRE+kNdOvKL9s1s7qf+uTQiUsXr93W7kW9RpyZJZPJPzkG7Z3KNZIxXn3V3Y32UoUNEakHAwBQxm8B2TT64dFvz+vXqZlw8y4R3Uz5hX8WWq9OjcRb94ko4ebdXHOxVQPLqzdSiOjqjZRG9WvzE3quZrNz/nvX6PHTFw621lKpTCqT8WsUcsXNO7+oWkhMsapvya/MzMqesWBNPcvq43z63Ux5qN2LsZHh85d/EdGFa7fVOaP8QOcYtHdKcyS5utNsCgCAiYfA31f69tmLV8FBPmtj9x0/e00oFIzyUN358ezffdmGXUdOX6lnWd3QQF+zytC+Lqs27jlxLsHww+NZnc0uWhGrvp/u1MF+SuiqmtUqmxgbSWUykZ6eSKR3NSll/7HzJsaGfp4fWzA2MmzRpMHkkJVKpdK9h6Ndc6tcvXgN6hm2equFuckPtaqJ9D4elgZ1a82PjJk63lN7DJ4Duq/ZFKe5U5ojEYn0NLvLvykAgGLBpYtVd7Q/35MXrxvXrfaVvS6P2tWzS9sa1Sr/+uRFzM5DIZN9qfBofp4HAIAdKanPa1WtWFKuAPLSzdFh/dYD+iKRTC4bMaRXsYwBAIBxxXMFAAAAxX4FgO8CAgBgFAIAAIBRCAAAAEYhAAAAGIUAAABgFAIAAIBRCAAAAEYhAAAAGIUAAABgFAIAAIBRCAAAAEYhAAAAGIUAAABgFAIAAIBRCAAAAEYhAAAAGIUAAABgFAIAAIBRCAAAAEYhAAAAGIUAAABg1BcHQK2qFVNSnxfNYAAAoIBSUp/XqlqxyK8AkAEAAKV99i/4LSBkAABAqZ79iYhLF2cXuNcnL14XuC4AABSKgs3+XxsAAABQeuFTQAAAjEIAAAAwCgEAAMAoBAAAAKMQAAAAjEIAAAAwCgEAAMAoBAAAAKMQAAAAjEIAAAAwCgEAAMAoBAAAAKMQAAAAjEIAAAAwCgEAAMAoBAAAAKMQAAAAjEIAAAAwCgEAAMAoBAAAAKMQAAAAjEIAAAAwSq/ANY9dTC7UkUAxcGprk38BnOUyAGeZBU6fOss6ceni7C+tw/916dPVtgD9QYmy5/j1vP7q4Cyzc5YNA0YVx7igMGWHry5ADBTwFhDmhbIh//OIs1w25H8eMfuXDQU7j18cAMcuJmNeKEv6dLXVvgOAs8zIWcbsX5YYBoz60rt5eAgMAMAoBAAAAKMQAAAAjEIAAAAwCgEAAMAoBAAAAKMQAAAAjEIAAAAwCgEAAMCowgyA2s179vYIcvMI7OLud/XGnU+WX7FhZ6GUgf9P9Vu5FfcQoMg5J12xj1lnv3Gdw5boar165FXM6dr5vDbV8fYostFBiQwAfZHe3pjFcTFhkfMCp4eu+mR5BABAyaSUSq96jLjqOeKaj1+VHi6Vu3b60hbqeHsWzdCgZHwddD7q16n56vWb92kZ0+atfP3mrVQqnRk4omnjetFbD2zfd5zjuGkThifeuifOzOrvM2X2JN+JsyPT0jMGuDmNGKp6d1m/lZtzJwerBpZv36XxZXasn18U44Sv9/rNW/8Z4eLMLBNjo4i5ARxHgbOWvk/LqPb9d2cuJd67tFvzpLdv3by4xwtfRp6V9SA8stGUoDdXr1tNnWhQ4RuBSHQ/bMm7lHvqMiJz81yb6vr5Co2N7davuhkwOa9aUGYD4PyVpDZ2TeaErR8+0LWZdf2Xr14PGz3r1N7VS9ZsvXps05+v3yxbt33Z/IkbNu/fsX7+5DnLpo4fXq9OjQ69RvABkCOVujq3/9GhBRHxZYpikFAoZi9e26tbhz49HPccPD0nbB1HnKtTe7fuHY+dvnLwxAUi0jzpCIDSKC31kUn1ag0Cxz3ZtuPdnbtGlSu1XLHkQu8B6gLam1JXrqk9ZECCz8/Wc2bkVQvKWgBIpLLeHkFSmezXJ8/PHVjv3G/009//4DdlZmXLFQrHdrZjpy7y6N9j2fyJ6lrTA3wOHD138nxCekYmv0YoELSzx0xROlxJvLNkbgAR9ezaLnRJlIDjwudMIKJOP9oJBaobjDpPOpQiAqFQKZNWdLA3qV6NXyM0MuIEAqVCwb8s2CYoawHAPwMgolXRu3btPyGXy7etDTUw0FcolNdv3hUKBEtDA6/dSFm/ed++w2eXhKhmDSIa4R/i0rmN1yDX2J0HP45JKBQIuEIcGBQhpVLzlVQq+7haoeS36DzpUIqUa2yVlvqrRYP6CSNHK3IknEBQvlkTzXmcE+oVYBOU2Y+BtrNvdivll5ZNGx09fZmIzlxKXL5hR1qG2M0jsHmTBssXTDx9QfUTRQqlQqFQJt9L7eHULidHkiORajfFlymKQUKhaG1rc+jERSI6dOJi65bWLZo2PHb2KhEdPX1ZSUrtkw6li8jcvIH/2MfRm/69dbuyY0ci+rZN6x/+9wGv7k2cgBMI8qkFZfYZgGWtag9Sf9sVtWjSnMjYXYf1hMLFs8ebm5p0am/XfeA4hUI5wXcQEdk1b+wxeqZH/x49B41vWM/SwsxUIpHq64s0m+LLxK6aWxTjhAKQSGWuQ/z55ZZNG80M8PGfGbF592FjI8OIuQE5Esm4qYs3bjvQ3KahsZGh9kmHUoETiexj1pGSOD29X6Ni/klMynz+wjp4eo2+vRVy+Z1ZH/9/FD/7vY6P5/2F4dqb/k261XLFkpQ587Q3Qcnxxb8JjN+KKnv2HL+e66dEC3yWx08LGzmsd4O6tW7fTZ29eO2+TeGFN0wo/LOMXwQrY7LDV3/RzwIXyRUAMGv4INepoSsMDQykUun86WOKezgAkB8EABQm64Y/4F0/QGmB7wICAGAUAgAAgFEIAAAARiEAAAAYhQAAAGAUAgAAgFEIAAAARiEAAAAY9cUB4NTWZs9xfKtXWf6GAJxlds5ydvjqYhoRFP/3QBT8CgCzQ9mQ/3nEWS4b8j+PyICyoWDn8Yu/DE7t2MXkglWEkuOT7xdwlssAnGUWOH3he/+vDQAAACjV8BAYAIBRCAAAAEYhAAAAGIUAAABgFAIAAIBRCAAAAEYhAAAAGIUAAABgFAIAAIBRCAAAAEYhAAAAGIUAAABgFAIAAIBRCAAAAEYhAAAAGIUAAABgFAIAAIBRCAAAAEYhAAAAGIUAAABgVEECwDdo3ucXPnTiYmE1VbBaOgtnZmWv3xznGxiqXnPh6s3QJVEz5q+6++BX9cqsrOzIddtCIjZErtuWlZWt2ULyvVTvCXPy6ohfTkp+MD8yen5ktOfYYH4h8da9c5eTZi5YPW9pdMTqLf++ff/5OwIAUMquAA6dzC8AisuSNVtrVv+eOI5/mZ4hvpRwe+r44T8P77tlzxF1sfjjF+rVqTnd37uuZY2DGkmWnZ0Tf+y8UCjMv5fmNg2mjBs+ZdxwAwN9fsHIyPBa0p3pAd5Txw/v1N5u/eZ9RbaLAACfoEdf7eWr1xu3x2dmZbe3b9a1Y+uT569duHqTI66va+dHv/2ekyNZvGJT0OhhH+bZzI3bD4gzs/SEwpHD+pibmRDRnoOnUh//Ls7MdHNxbG7TQJyZtXn34fdpGXKZvL9b19o1quqsRURp6eKFy2N+9nQvZ2GWq8r79IyN2w6IM7MrVijPF168MjbIb6h6zKO9+lmYm+49dIZ/mSHO6tTejuO4/5Q3zxBnqcsn30ud+GHkrZo3DlsV29e1M19+V/zJrh3so7fHf+mxOnr6snuPTvoiERFZN/whKfm+XC7/ZJAAAJTQK4BT5xPcXTtPm+B15PRlIjpw9Py08V6jPN2vJCb/5NLRwECfn/2JaPu+Y7ZNraaMG96qhfW+w6rJVyaTmZkYTx0/fIz3AP6t9859xzu3bzVpjMdIjz4bt8XrrKWqKJev2rhrUB/nKpUralfZEXfcrlnjaRO8mtk0kEplRDTGq5/mmC3MTTVfVv6ugm3TRkSUeOt+08b11OXT0jP4kuUszN6nifnCqY+fvXuXbtvMqgDH6uWr1zWqVVa/9BzgitkfAErxFUC/Xl2uJd1NTvklKyuHiGwa/bAuNq5jO9sRQ3vnKnn/l9+GD3QlIgdbmxY2DYhIqaS29s2IqFLFb/ib7CkPfv3rzb98+RyJRKFQaNciotidh+xb2jSsW1tnlYePnngNUlVpYlVPIFDd5zE0NPjkjrx+8++RU5emjPPMp7xMJtux7/hYnwEFO1YKhaJgFQEASmIArIja2aJJo04/tjp9KZGIfIa4/fLr0+Nnr167ccd78E+5pz+lUnXdIRAYGRmqutcTGn9YUPlwR16uUAT+PFQk0lMqlamPnwkEAu1aMpnsxavXRNT+Q3hoV5HJ5XyTSqWSVFU/LTtHsjJ6l9egXmamH28xEZG5men7tIxyFmbv3qdbmKvWJ96+n52dszpmtypsciTrYveqc07AcQqFQiAQyBUKoUD3pVWlihWevfjTsmZVfmzrt+wbMcStYIcdAKD4bwE9+f0Pu2aNpFKZTCrLysqetzS6Tq1qI4f1Tr6Xyk9zqln4g9o1qt6885CIzl9J2h1/8sOc//ExrNoPtasnJT8gojv3HvHPXbVr6enpzfD3fvPPu3OXk3RW+aFWdb5KUvIDPgGycyT57IJqLt4c59zRgZ+a1eVtGtW9lpRCRNeSUmwa1iUi+xbW86aPUT/a1bzKqVWjyr2Hj4no7v1fa9WoorMjx3a2ew+elslUd6USku7KPtyeAgAoNVcAcpk8JGIDv1y3dnXHdrZzw9dXq1rJ2NhQJNJrYlV3Ttg6hVLp6vQjEdWzrLFk7VZ/38FENLC3U9TW/acuXDc2MtC+QcQb1Ns5env8mUuJQoGAv/OjsxbHcaM8+8wOW1e96nfaVQb0dlofG3fqfEKd2tX19FT7uHz9dvWjCG2XEm6n3P81Q5x59nKigYG+v+9gvnzPru3WbY67cfu+manxJ9+qD3F3id52gE8gfhja7JpZ/fX6n5kL15iZmpibmQzt2/2zjzoAQCHj0sX/8/F2AABgBP4lMAAAoxAAAACMQgAAADAKAQAAwCgEAAAAoxAAAACMQgAAADAKAQAAwCgEAAAAoxAAAACMQgAAADAKAQAAQGz6P+Pqwg6Yjzl6AAAAAElFTkSuQmCC"],"questions":{"action":{"type":"choice","instructions":"Which visible control opens deployment logs?","criteria":{"restart":{"description":"Click Restart."},"logs":{"description":"Click Logs."},"delete":{"description":"Click Delete."}}},"healthy":{"type":"noul","instructions":"Does the screenshot report a healthy deployment?","criteria":{"true":"Healthy","false":"Unhealthy"}}}}} +{"id":"multi-image-2","modality":"image","request":{"state":"Choose the action from the supplied screenshots. Keep their order.","images":["iVBORw0KGgoAAAANSUhEUgAAAQAAAAEACAIAAADTED8xAAADAElEQVR4nO3TQQ2AABAEsZsEA7zw7xQh26QW2t29F7yTnvu6gzYJQMsEoGUC0DIBaJkAtEwAWiYALROAlglAywSgZQLQMgFomQC0TABaJgAtE4CWCUDLBKBlAtAyAWiZALRMAFomAC0TgJYJQMsEoGUC0DIBaJkAtEwAWiYALROAlglAywSgZQLQMgFomQC0TABaJgAtE4CWCUDLBKBlAtAyAWiZALRMAFomAC0TgJYJQMsEoGUC0DIBaJkAtEwAWiYALROAlglAywSgZQLQMgFomQC0TABaJgAtE4CWCUDLBKBlAtAyAWiZALRMAFomAC0TgJYJQMsEoGUC0DIBaJkAtEwAWiYALROAlglAywSgZQLQMgFomQC0TABaJgAtE4CWCUDLBKBlAtAyAWiZALRMAFomAC0TgJYJQMsEoGUC0DIBaJkAtEwAWiYALROAlglAywSgZQLQMgFomQC0TABaJgAtE4CWCUDLBKBlAtAyAWiZALRMAFomAC0TgJYJQMsEoGUC0DIBaJkAtEwApglAywSgZQLQMgFomQC0TABaJgAtE4CWCUDLBKBlAtAyAWiZALRMAFomAC0TgJYJQMsEoGUC0DIBaJkAtEwAWiYALROAlglAywSgZQLQMgFomQC0TABaJgAtE4CWCUDLBKBlAtAyAWiZALRMAFomAC0TgJYJQMsEoGUC0DIBaJkAtEwAWiYALROAlglAywSgZQLQMgFomQC0TABaJgAtE4CWCUDLBKBlAtAyAWiZALRMAFomAC0TgJYJQMsEoGUC0DIBaJkAtEwAWiYALROAlglAywSgZQLQMgFomQC0TABaJgAtE4CWCUDLBKBlAtAyAWiZALRMAFomAC0TgJYJQMsEoGUC0DIBaJkAtEwAWiYALROAlglAywSgZQLQMgFomQC0TABaJgAtE4CWCUDLBKBlAtAyAWiZALRMAFomAC0TgJYJQMsEoGUC0DIBaJkAtEwAWiYALROAlglAywSgZQLQMgFomQC0TABaJgAtE4CW/R3rK+SRHEFZAAAAAElFTkSuQmCC","iVBORw0KGgoAAAANSUhEUgAAAgAAAAEACAIAAABK8lkwAAAD5klEQVR4nO3VMQ0DQQAEsVvpCVyVMhAeQfjTCpCxZA7e9/3dMwBuzHM+OweA1QgAYE0CAFiTAADWJACANQkAYE0CAFiTAADWJACANQkAYE0CAFiTAADWJACANQkAYE0CAFiTAADWJACANQkAYE0CAFiTAADWJACANQkAYE0CAFiTAADWJACANQkAYE0CAFiTAADWJACANQkAYE0CAFiTAADWJACANQkAYE0CAFiTAADWJACANQkAYE0CAFiTAADWJACANQkAYE0CAFiTAADWJACANQkAYE0CAFiTAADWJACANQkAYE0CAFiTAADWJACANQkAYE0CAFiTAADWJACANQkAYE0CAFiTAADWJACANQkAYE0CAIgSAMCaBACwJgEArEkAAGsSAMCaBACwJgEArEkAAGsSAMCaBACwJgEArEkAAGsSAMCaBACwJgEArEkAAGsSAMCaBACwJgEArEkAAGsSAMCaBACwJgEArEkAAGsSAMCaBACwJgEArEkAAGsSAMCaBACwJgEArEkAAGsSAMCaBACwJgEArEkAAGsSAMCaBACwJgEArEkAAGsSAMCaBACwJgEArEkAAGsSAMCaBACwJgEArEkAAGsSAMCaBACwJgEArEkAAGsSAMCaBACwJgEArEkAAGsSAMCaBACwJgEArEkAAGsSAMCaBACwJgEArEkAAFECAFiTAADWJACANQkAYE0CAFiTAADWJACANQkAYE0CAFiTAADWJACANQkAYE0CAFiTAADWJACANQkAYE0CAFiTAADWJACANQkAYE0CAFiTAADWJACANQkAYE0CAFiTAADWJACANQkAYE0CAFiTAADWJACANQkAYE0CAFiTAADWJACANQkAYE0CAFiTAADWJACANQkAYE0CAFiTAADWJACANQkAYE0CAFiTAADWJACANQkAYE0CAFiTAADWJACANQkAYE0CAFiTAADWJACANQkAYE0CAFiTAADWJACANQkAYE0CAFiTAADWJACANQkAIEoAAGsSAMCaBACwJgEArEkAAGsSAMCaBACwJgEArEkAAGsSAMCaBACwJgEArEkAAGsSAMCaBACwJgEArEkAAGsSAMCaBACwJgEArEkAAGsSAMCaBACwJgEArEkAAGsSAMCaBACwJgEArEkAAGsSAMCaBACwJgEArEkAAGsSAMCaBACwJgEArEkAAGsSAMCaBACwJgEArEkAAGsSAMCaBACwJgEArEkAAGsSAMCaBACwJgEArEkAAGsSAMCaBACwJgEArEkAAGsSAMCaBACwJgEArEkAAGsSAMCaBACwJgEArEkAAGsSAMCaBACwJgEArOkPuvs9fgi2xPsAAAAASUVORK5CYII="],"questions":{"choice":{"type":"choice","instructions":"Which screenshot has a blue region?","criteria":{"first":"First screenshot","second":"Second screenshot"}},"noul":{"type":"noul","instructions":"Is a screenshot attached?"},"score":{"type":"score","instructions":"Rate visual differences.","criteria":["small","medium","large"]}}}} +{"id":"multi-image-3","modality":"image","request":{"state":"Choose the action from the supplied screenshots. Keep their order.","images":["iVBORw0KGgoAAAANSUhEUgAAAQAAAAEACAIAAADTED8xAAADAElEQVR4nO3TQQ2AABAEsZsEA7zw7xQh26QW2t29F7yTnvu6gzYJQMsEoGUC0DIBaJkAtEwAWiYALROAlglAywSgZQLQMgFomQC0TABaJgAtE4CWCUDLBKBlAtAyAWiZALRMAFomAC0TgJYJQMsEoGUC0DIBaJkAtEwAWiYALROAlglAywSgZQLQMgFomQC0TABaJgAtE4CWCUDLBKBlAtAyAWiZALRMAFomAC0TgJYJQMsEoGUC0DIBaJkAtEwAWiYALROAlglAywSgZQLQMgFomQC0TABaJgAtE4CWCUDLBKBlAtAyAWiZALRMAFomAC0TgJYJQMsEoGUC0DIBaJkAtEwAWiYALROAlglAywSgZQLQMgFomQC0TABaJgAtE4CWCUDLBKBlAtAyAWiZALRMAFomAC0TgJYJQMsEoGUC0DIBaJkAtEwAWiYALROAlglAywSgZQLQMgFomQC0TABaJgAtE4CWCUDLBKBlAtAyAWiZALRMAFomAC0TgJYJQMsEoGUC0DIBaJkAtEwApglAywSgZQLQMgFomQC0TABaJgAtE4CWCUDLBKBlAtAyAWiZALRMAFomAC0TgJYJQMsEoGUC0DIBaJkAtEwAWiYALROAlglAywSgZQLQMgFomQC0TABaJgAtE4CWCUDLBKBlAtAyAWiZALRMAFomAC0TgJYJQMsEoGUC0DIBaJkAtEwAWiYALROAlglAywSgZQLQMgFomQC0TABaJgAtE4CWCUDLBKBlAtAyAWiZALRMAFomAC0TgJYJQMsEoGUC0DIBaJkAtEwAWiYALROAlglAywSgZQLQMgFomQC0TABaJgAtE4CWCUDLBKBlAtAyAWiZALRMAFomAC0TgJYJQMsEoGUC0DIBaJkAtEwAWiYALROAlglAywSgZQLQMgFomQC0TABaJgAtE4CWCUDLBKBlAtAyAWiZALRMAFomAC0TgJYJQMsEoGUC0DIBaJkAtEwAWiYALROAlglAywSgZQLQMgFomQC0TABaJgAtE4CW/R3rK+SRHEFZAAAAAElFTkSuQmCC","iVBORw0KGgoAAAANSUhEUgAAAgAAAAEACAIAAABK8lkwAAAD5klEQVR4nO3VMQ0DQQAEsVvpCVyVMhAeQfjTCpCxZA7e9/3dMwBuzHM+OweA1QgAYE0CAFiTAADWJACANQkAYE0CAFiTAADWJACANQkAYE0CAFiTAADWJACANQkAYE0CAFiTAADWJACANQkAYE0CAFiTAADWJACANQkAYE0CAFiTAADWJACANQkAYE0CAFiTAADWJACANQkAYE0CAFiTAADWJACANQkAYE0CAFiTAADWJACANQkAYE0CAFiTAADWJACANQkAYE0CAFiTAADWJACANQkAYE0CAFiTAADWJACANQkAYE0CAFiTAADWJACANQkAYE0CAFiTAADWJACANQkAYE0CAFiTAADWJACANQkAYE0CAIgSAMCaBACwJgEArEkAAGsSAMCaBACwJgEArEkAAGsSAMCaBACwJgEArEkAAGsSAMCaBACwJgEArEkAAGsSAMCaBACwJgEArEkAAGsSAMCaBACwJgEArEkAAGsSAMCaBACwJgEArEkAAGsSAMCaBACwJgEArEkAAGsSAMCaBACwJgEArEkAAGsSAMCaBACwJgEArEkAAGsSAMCaBACwJgEArEkAAGsSAMCaBACwJgEArEkAAGsSAMCaBACwJgEArEkAAGsSAMCaBACwJgEArEkAAGsSAMCaBACwJgEArEkAAGsSAMCaBACwJgEArEkAAFECAFiTAADWJACANQkAYE0CAFiTAADWJACANQkAYE0CAFiTAADWJACANQkAYE0CAFiTAADWJACANQkAYE0CAFiTAADWJACANQkAYE0CAFiTAADWJACANQkAYE0CAFiTAADWJACANQkAYE0CAFiTAADWJACANQkAYE0CAFiTAADWJACANQkAYE0CAFiTAADWJACANQkAYE0CAFiTAADWJACANQkAYE0CAFiTAADWJACANQkAYE0CAFiTAADWJACANQkAYE0CAFiTAADWJACANQkAYE0CAFiTAADWJACANQkAYE0CAFiTAADWJACANQkAIEoAAGsSAMCaBACwJgEArEkAAGsSAMCaBACwJgEArEkAAGsSAMCaBACwJgEArEkAAGsSAMCaBACwJgEArEkAAGsSAMCaBACwJgEArEkAAGsSAMCaBACwJgEArEkAAGsSAMCaBACwJgEArEkAAGsSAMCaBACwJgEArEkAAGsSAMCaBACwJgEArEkAAGsSAMCaBACwJgEArEkAAGsSAMCaBACwJgEArEkAAGsSAMCaBACwJgEArEkAAGsSAMCaBACwJgEArEkAAGsSAMCaBACwJgEArEkAAGsSAMCaBACwJgEArEkAAGsSAMCaBACwJgEArOkPuvs9fgi2xPsAAAAASUVORK5CYII=","iVBORw0KGgoAAAANSUhEUgAAAQAAAAEACAIAAADTED8xAAADBklEQVR4nO3TQQ0DMQAEsYx0BPIqm0I5IEVfIGvJFNz3/d0T3EnP+XQOtEkAWiYALROAlglAywSgZQLQMgFomQC0TABaJgAtE4CWCUDLBKBlAtAyAWiZALRMAFomAC0TgJYJQMsEoGUC0DIBaJkAtEwAWiYALROAlglAywSgZQLQMgFomQC0TABaJgAtE4CWCUDLBKBlAtAyAWiZALRMAFomAC0TgJYJQMsEoGUC0DIBaJkAtEwAWiYALROAlglAywSgZQLQMgFomQC0TABaJgAtE4CWCUDLBKBlAtAyAWiZALRMAFomAC0TgJYJQMsEoGUC0DIBaJkAtEwAWiYALROAlglAywSgZQLQMgFomQC0TABaJgAtE4CWCUDLBKBlAtAyAWiZALRMAFomAC0TgJYJQMsEoGUC0DIBaJkAtEwAWiYALROAlglAywSgZQLQMgFomQC0TABaJgAtE4CWCUDLBKBlAtAyAWiZALRMAFomAC0TgJYJQMsEoGUC0DIBaJkAtEwAWiYALROAlglAywSgZQIwTQBaJgAtE4CWCUDLBKBlAtAyAWiZALRMAFomAC0TgJYJQMsEoGUC0DIBaJkAtEwAWiYALROAlglAywSgZQLQMgFomQC0TABaJgAtE4CWCUDLBKBlAtAyAWiZALRMAFomAC0TgJYJQMsEoGUC0DIBaJkAtEwAWiYALROAlglAywSgZQLQMgFomQC0TABaJgAtE4CWCUDLBKBlAtAyAWiZALRMAFomAC0TgJYJQMsEoGUC0DIBaJkAtEwAWiYALROAlglAywSgZQLQMgFomQC0TABaJgAtE4CWCUDLBKBlAtAyAWiZALRMAFomAC0TgJYJQMsEoGUC0DIBaJkAtEwAWiYALROAlglAywSgZQLQMgFomQC0TABaJgAtE4CWCUDLBKBlAtAyAWiZALRMAFomAC0TgJYJQMsEoGUC0DIBaJkAtEwAWiYALROAlglAywSgZQLQMgFomQC0TABaJgAtE4CWCUDLBKBlAtAyAWiZALTsD2JwLPqu2ot4AAAAAElFTkSuQmCC"],"questions":{"choice":{"type":"choice","instructions":"Which screenshot has a blue region?","criteria":{"first":"First screenshot","second":"Second screenshot"}},"noul":{"type":"noul","instructions":"Is a screenshot attached?"},"score":{"type":"score","instructions":"Rate visual differences.","criteria":["small","medium","large"]}}}} +{"id":"multi-image-4","modality":"image","request":{"state":"Choose the action from the supplied screenshots. Keep their order.","images":["iVBORw0KGgoAAAANSUhEUgAAAQAAAAEACAIAAADTED8xAAADAElEQVR4nO3TQQ2AABAEsZsEA7zw7xQh26QW2t29F7yTnvu6gzYJQMsEoGUC0DIBaJkAtEwAWiYALROAlglAywSgZQLQMgFomQC0TABaJgAtE4CWCUDLBKBlAtAyAWiZALRMAFomAC0TgJYJQMsEoGUC0DIBaJkAtEwAWiYALROAlglAywSgZQLQMgFomQC0TABaJgAtE4CWCUDLBKBlAtAyAWiZALRMAFomAC0TgJYJQMsEoGUC0DIBaJkAtEwAWiYALROAlglAywSgZQLQMgFomQC0TABaJgAtE4CWCUDLBKBlAtAyAWiZALRMAFomAC0TgJYJQMsEoGUC0DIBaJkAtEwAWiYALROAlglAywSgZQLQMgFomQC0TABaJgAtE4CWCUDLBKBlAtAyAWiZALRMAFomAC0TgJYJQMsEoGUC0DIBaJkAtEwAWiYALROAlglAywSgZQLQMgFomQC0TABaJgAtE4CWCUDLBKBlAtAyAWiZALRMAFomAC0TgJYJQMsEoGUC0DIBaJkAtEwApglAywSgZQLQMgFomQC0TABaJgAtE4CWCUDLBKBlAtAyAWiZALRMAFomAC0TgJYJQMsEoGUC0DIBaJkAtEwAWiYALROAlglAywSgZQLQMgFomQC0TABaJgAtE4CWCUDLBKBlAtAyAWiZALRMAFomAC0TgJYJQMsEoGUC0DIBaJkAtEwAWiYALROAlglAywSgZQLQMgFomQC0TABaJgAtE4CWCUDLBKBlAtAyAWiZALRMAFomAC0TgJYJQMsEoGUC0DIBaJkAtEwAWiYALROAlglAywSgZQLQMgFomQC0TABaJgAtE4CWCUDLBKBlAtAyAWiZALRMAFomAC0TgJYJQMsEoGUC0DIBaJkAtEwAWiYALROAlglAywSgZQLQMgFomQC0TABaJgAtE4CWCUDLBKBlAtAyAWiZALRMAFomAC0TgJYJQMsEoGUC0DIBaJkAtEwAWiYALROAlglAywSgZQLQMgFomQC0TABaJgAtE4CW/R3rK+SRHEFZAAAAAElFTkSuQmCC","iVBORw0KGgoAAAANSUhEUgAAAgAAAAEACAIAAABK8lkwAAAD5klEQVR4nO3VMQ0DQQAEsVvpCVyVMhAeQfjTCpCxZA7e9/3dMwBuzHM+OweA1QgAYE0CAFiTAADWJACANQkAYE0CAFiTAADWJACANQkAYE0CAFiTAADWJACANQkAYE0CAFiTAADWJACANQkAYE0CAFiTAADWJACANQkAYE0CAFiTAADWJACANQkAYE0CAFiTAADWJACANQkAYE0CAFiTAADWJACANQkAYE0CAFiTAADWJACANQkAYE0CAFiTAADWJACANQkAYE0CAFiTAADWJACANQkAYE0CAFiTAADWJACANQkAYE0CAFiTAADWJACANQkAYE0CAFiTAADWJACANQkAYE0CAFiTAADWJACANQkAYE0CAIgSAMCaBACwJgEArEkAAGsSAMCaBACwJgEArEkAAGsSAMCaBACwJgEArEkAAGsSAMCaBACwJgEArEkAAGsSAMCaBACwJgEArEkAAGsSAMCaBACwJgEArEkAAGsSAMCaBACwJgEArEkAAGsSAMCaBACwJgEArEkAAGsSAMCaBACwJgEArEkAAGsSAMCaBACwJgEArEkAAGsSAMCaBACwJgEArEkAAGsSAMCaBACwJgEArEkAAGsSAMCaBACwJgEArEkAAGsSAMCaBACwJgEArEkAAGsSAMCaBACwJgEArEkAAGsSAMCaBACwJgEArEkAAFECAFiTAADWJACANQkAYE0CAFiTAADWJACANQkAYE0CAFiTAADWJACANQkAYE0CAFiTAADWJACANQkAYE0CAFiTAADWJACANQkAYE0CAFiTAADWJACANQkAYE0CAFiTAADWJACANQkAYE0CAFiTAADWJACANQkAYE0CAFiTAADWJACANQkAYE0CAFiTAADWJACANQkAYE0CAFiTAADWJACANQkAYE0CAFiTAADWJACANQkAYE0CAFiTAADWJACANQkAYE0CAFiTAADWJACANQkAYE0CAFiTAADWJACANQkAYE0CAFiTAADWJACANQkAIEoAAGsSAMCaBACwJgEArEkAAGsSAMCaBACwJgEArEkAAGsSAMCaBACwJgEArEkAAGsSAMCaBACwJgEArEkAAGsSAMCaBACwJgEArEkAAGsSAMCaBACwJgEArEkAAGsSAMCaBACwJgEArEkAAGsSAMCaBACwJgEArEkAAGsSAMCaBACwJgEArEkAAGsSAMCaBACwJgEArEkAAGsSAMCaBACwJgEArEkAAGsSAMCaBACwJgEArEkAAGsSAMCaBACwJgEArEkAAGsSAMCaBACwJgEArEkAAGsSAMCaBACwJgEArEkAAGsSAMCaBACwJgEArOkPuvs9fgi2xPsAAAAASUVORK5CYII=","iVBORw0KGgoAAAANSUhEUgAAAQAAAAEACAIAAADTED8xAAADBklEQVR4nO3TQQ0DMQAEsYx0BPIqm0I5IEVfIGvJFNz3/d0T3EnP+XQOtEkAWiYALROAlglAywSgZQLQMgFomQC0TABaJgAtE4CWCUDLBKBlAtAyAWiZALRMAFomAC0TgJYJQMsEoGUC0DIBaJkAtEwAWiYALROAlglAywSgZQLQMgFomQC0TABaJgAtE4CWCUDLBKBlAtAyAWiZALRMAFomAC0TgJYJQMsEoGUC0DIBaJkAtEwAWiYALROAlglAywSgZQLQMgFomQC0TABaJgAtE4CWCUDLBKBlAtAyAWiZALRMAFomAC0TgJYJQMsEoGUC0DIBaJkAtEwAWiYALROAlglAywSgZQLQMgFomQC0TABaJgAtE4CWCUDLBKBlAtAyAWiZALRMAFomAC0TgJYJQMsEoGUC0DIBaJkAtEwAWiYALROAlglAywSgZQLQMgFomQC0TABaJgAtE4CWCUDLBKBlAtAyAWiZALRMAFomAC0TgJYJQMsEoGUC0DIBaJkAtEwAWiYALROAlglAywSgZQIwTQBaJgAtE4CWCUDLBKBlAtAyAWiZALRMAFomAC0TgJYJQMsEoGUC0DIBaJkAtEwAWiYALROAlglAywSgZQLQMgFomQC0TABaJgAtE4CWCUDLBKBlAtAyAWiZALRMAFomAC0TgJYJQMsEoGUC0DIBaJkAtEwAWiYALROAlglAywSgZQLQMgFomQC0TABaJgAtE4CWCUDLBKBlAtAyAWiZALRMAFomAC0TgJYJQMsEoGUC0DIBaJkAtEwAWiYALROAlglAywSgZQLQMgFomQC0TABaJgAtE4CWCUDLBKBlAtAyAWiZALRMAFomAC0TgJYJQMsEoGUC0DIBaJkAtEwAWiYALROAlglAywSgZQLQMgFomQC0TABaJgAtE4CWCUDLBKBlAtAyAWiZALRMAFomAC0TgJYJQMsEoGUC0DIBaJkAtEwAWiYALROAlglAywSgZQLQMgFomQC0TABaJgAtE4CWCUDLBKBlAtAyAWiZALTsD2JwLPqu2ot4AAAAAElFTkSuQmCC","iVBORw0KGgoAAAANSUhEUgAAAQAAAAIACAIAAADi+CWsAAAF5UlEQVR4nO3TQQ0DMQAEsax0BPIqukI6PKVXIGPJFLzv+7tncJOe89k5sCYBWJkArEwAViYAKxOAlQnAygRgZQKwMgFYmQCsTABWJgArE4CVCcDKBGBlArAyAViZAKxMAFYmACsTgJUJwMoEYGUCsDIBWJkArEwAViYAKxOAlQnAygRgZQKwMgFYmQCsTABWJgArE4CVCcDKBGBlArAyAViZAKxMAFYmACsTgJUJwMoEYGUCsDIBWJkArEwAViYAKxOAlQnAygRgZQKwMgFYmQCsTABWJgArE4CVCcDKBGBlArAyAViZAKxMAFYmACsTgJUJwMoEYGUCsDIBWJkArEwAViYAKxOAlQnAygRgZQKwMgFYmQCsTABWJgArE4CVCcDKBGBlArAyAViZAKxMAFYmACsTgJUJwMoEYGUCsDIBWJkArEwAViYAKxOAlQnAygRgZQKwMgFYmQCsTABWJgArE4CVCcDKBGBlArAyAViZAKxMAFYmACsTgJUJwMoEYGUCsDIBWJkArEwAViYAKxOAlQlAmgCsTABWJgArE4CVCcDKBGBlArAyAViZAKxMAFYmACsTgJUJwMoEYGUCsDIBWJkArEwAViYAKxOAlQnAygRgZQKwMgFYmQCsTABWJgArE4CVCcDKBGBlArAyAViZAKxMAFYmACsTgJUJwMoEYGUCsDIBWJkArEwAViYAKxOAlQnAygRgZQKwMgFYmQCsTABWJgArE4CVCcDKBGBlArAyAViZAKxMAFYmACsTgJUJwMoEYGUCsDIBWJkArEwAViYAKxOAlQnAygRgZQKwMgFYmQCsTABWJgArE4CVCcDKBGBlArAyAViZAKxMAFYmACsTgJUJwMoEYGUCsDIBWJkArEwAViYAKxOAlQnAygRgZQKwMgFYmQCsTABWJgArE4CVCcDKBGBlArAyAViZAKxMAFYmACsTgJUJwMoEYGUCsDIBWJkArEwAViYAKxOAlQnAygRgZQKwMgFYmQCsTABWJgArE4CVCcDKBGBlArAyAViZAKxMAFYmACsTgJUJwMoEYGUCsDIBWJkArEwAViYAKxOAlQnAygRgZQKwMgFYmQCsTABWJgArE4CVCcDKBGBlArAyAViZAKxMAFYmACsTgJUJwMoEYGUCsDIBWJkArEwAViYAKxOAlQnAygRgZQKwMgFYmQCsTABWJgArE4CVCcDKBGBlArAyAViZAKxMAFYmACsTgJUJwMoEYGUCsDIBWJkArEwAViYAKxOAlQnAygRgZQKwMgFYmQCsTABWJgArE4CVCcDKBGBlArAyAViZAKxMAFYmACsTgJUJwMoEYGUCsDIBWJkArEwAViYAKxOAlQlAmgCsTABWJgArE4CVCcDKBGBlArAyAViZAKxMAFYmACsTgJUJwMoEYGUCsDIBWJkArEwAViYAKxOAlQnAygRgZQKwMgFYmQCsTABWJgArE4CVCcDKBGBlArAyAViZAKxMAFYmACsTgJUJwMoEYGUCsDIBWJkArEwAViYAKxOAlQnAygRgZQKwMgFYmQCsTABWJgArE4CVCcDKBGBlArAyAViZAKxMAFYmACsTgJUJwMoEYGUCsDIBWJkArEwAViYAKxOAlQnAygRgZQKwMgFYmQCsTABWJgArE4CVCcDKBGBlArAyAViZAKxMAFYmACsTgJUJwMoEYGUCsDIBWJkArEwAViYAKxOAlQnAygRgZQKwMgFYmQCsTABWJgArE4CVCcDKBGBlArAyAViZAKxMAFYmACsTgJUJwMoEYGUCsDIBWJkArEwAViYAKxOAlQnAygRgZQKwMgFYmQCsTABWJgArE4CVCcDKBGBlArAyAUgTgJUJwMoEYGUCsDIBWJkArEwAViYAKxOAlQnAygRgZQKwMgFYmQCsTABWJgArE4CVCcDKBGBlArAyAViZAKxMAFYmACsTgJUJwMoEYGUCsDIBWJkArEwAViYAKxOAlQnAygRgZQKwMgFYmQCsTABWJgArE4CVCcDKBGBlf1hISJQSmDleAAAAAElFTkSuQmCC"],"questions":{"choice":{"type":"choice","instructions":"Which screenshot has a blue region?","criteria":{"first":"First screenshot","second":"Second screenshot"}},"noul":{"type":"noul","instructions":"Is a screenshot attached?"},"score":{"type":"score","instructions":"Rate visual differences.","criteria":["small","medium","large"]}}}} diff --git a/tests/jemm/data/parity/reference.json b/tests/jemm/data/parity/reference.json new file mode 100644 index 00000000..ebbfd0f8 --- /dev/null +++ b/tests/jemm/data/parity/reference.json @@ -0,0 +1,528 @@ +[ + { + "id": "text-choice-2", + "response": { + "model": "JEMM", + "answers": { + "select": { + "type": "choice", + "choice": "option-01", + "probabilities": { + "option-00": 0.00504015488794939, + "option-01": 0.9949598451120509 + }, + "confidence": 0.9949598451120509 + } + }, + "usage": { + "input_tokens": 98, + "output_tokens": 0 + } + } + }, + { + "id": "text-choice-3", + "response": { + "model": "JEMM", + "answers": { + "select": { + "type": "choice", + "choice": "option-02", + "probabilities": { + "option-00": 0.0010897100350239265, + "option-01": 0.005521120841957601, + "option-02": 0.9933891691230169 + }, + "confidence": 0.9933891691230169 + } + }, + "usage": { + "input_tokens": 107, + "output_tokens": 0 + } + } + }, + { + "id": "text-choice-10", + "response": { + "model": "JEMM", + "answers": { + "select": { + "type": "choice", + "choice": "option-09", + "probabilities": { + "option-00": 0.0007097466499469256, + "option-01": 0.0014902508399453856, + "option-02": 0.0010772490376576992, + "option-03": 0.0016350418691250149, + "option-04": 0.0011819132946165765, + "option-05": 0.0008543628529457552, + "option-06": 0.0008543628529457552, + "option-07": 0.0014902508399453856, + "option-08": 0.008677206486499495, + "option-09": 0.982029615276373 + }, + "confidence": 0.982029615276373 + } + }, + "usage": { + "input_tokens": 171, + "output_tokens": 0 + } + } + }, + { + "id": "text-choice-26", + "response": { + "model": "JEMM", + "answers": { + "select": { + "type": "choice", + "choice": "option-25", + "probabilities": { + "option-00": 0.0005657440072402099, + "option-01": 0.0007826421305671105, + "option-02": 0.000540113721745987, + "option-03": 0.0004699817091553514, + "option-04": 0.00037274102814643473, + "option-05": 0.0007826421305671105, + "option-06": 0.00018594560862790282, + "option-07": 0.0001618011752605934, + "option-08": 0.00026944115923531654, + "option-09": 0.0002572344831650147, + "option-10": 0.0005156445825053841, + "option-11": 0.0002238334209706375, + "option-12": 0.00021369294328248343, + "option-13": 0.0003904288938350176, + "option-14": 0.00028222708478590564, + "option-15": 0.0005925905394396495, + "option-16": 0.00028222708478590564, + "option-17": 0.0002455808144418759, + "option-18": 0.0004922839853200367, + "option-19": 0.00028222708478590564, + "option-20": 0.00037274102814643473, + "option-21": 0.000324341816227679, + "option-22": 0.0003558544878657757, + "option-23": 0.0005925905394396495, + "option-24": 0.003450446283829918, + "option-25": 0.9869950022566255 + }, + "confidence": 0.9869950022566255 + } + }, + "usage": { + "input_tokens": 332, + "output_tokens": 0 + } + } + }, + { + "id": "text-choice-32", + "response": { + "model": "JEMM", + "answers": { + "select": { + "type": "choice", + "choice": "option-31", + "probabilities": { + "option-00": 0.0006098053152967058, + "option-01": 0.0010636732158833381, + "option-02": 0.0004408064544932594, + "option-03": 0.0005558040442171643, + "option-04": 0.00033376395731131783, + "option-05": 0.00031864322177705903, + "option-06": 0.00040177086683013786, + "option-07": 0.0002904257914510064, + "option-08": 0.00033376395731131783, + "option-09": 0.0007688913285461329, + "option-10": 0.000969479701609008, + "option-11": 0.0005065848522783858, + "option-12": 0.00025271494567422363, + "option-13": 0.00040177086683013786, + "option-14": 0.0002412660286315417, + "option-15": 0.00040177086683013786, + "option-16": 0.00040177086683013786, + "option-17": 0.0004408064544932594, + "option-18": 0.0008435959110253932, + "option-19": 0.0008435959110253932, + "option-20": 0.0018553474064703947, + "option-21": 0.001167018461871726, + "option-22": 0.0014048072260709565, + "option-23": 0.0008053779117807994, + "option-24": 0.0002904257914510064, + "option-25": 0.0005306240756274465, + "option-26": 0.000669053286735028, + "option-27": 0.0012223975702900626, + "option-28": 0.0021322076440835405, + "option-29": 0.0012804046111271276, + "option-30": 0.008567853677035707, + "option-31": 0.9696537777791118 + }, + "confidence": 0.9696537777791118 + } + }, + "usage": { + "input_tokens": 392, + "output_tokens": 0 + } + } + }, + { + "id": "text-score-3", + "response": { + "model": "JEMM", + "answers": { + "rating": { + "type": "score", + "probabilities": { + "0": 0.0025149207912409867, + "1": 0.9951928670689261, + "2": 0.0022922121398326304 + }, + "confidence": 0.9951928670689261, + "expected_value": 0.9997772913485913 + } + }, + "usage": { + "input_tokens": 107, + "output_tokens": 0 + } + } + }, + { + "id": "text-score-10", + "response": { + "model": "JEMM", + "answers": { + "rating": { + "type": "score", + "probabilities": { + "0": 0.0009366690084057889, + "1": 0.0012370710964574, + "2": 0.0005624758549152151, + "3": 0.0008942344553362082, + "4": 0.004970924934045417, + "5": 0.9812933952221659, + "6": 0.0031267250275492063, + "7": 0.0012957744474412615, + "8": 0.0013572634777822804, + "9": 0.004325466475899582 + }, + "confidence": 0.9812933952221659, + "expected_value": 5.0090134794220464 + } + }, + "usage": { + "input_tokens": 170, + "output_tokens": 0 + } + } + }, + { + "id": "text-score-32", + "response": { + "model": "JEMM", + "answers": { + "rating": { + "type": "score", + "probabilities": { + "0": 0.00028184574322350446, + "1": 0.0006198723368381861, + "2": 0.00033927392723735544, + "3": 0.0006198723368381861, + "4": 0.0006800984083189255, + "5": 0.00025688691128052305, + "6": 0.00022353098016650527, + "7": 0.000295220309677395, + "8": 0.00044808354434749845, + "9": 0.0004277837108187035, + "10": 0.00028184574322350446, + "11": 0.00022353098016650527, + "12": 0.0002134042041597757, + "13": 0.00028184574322350446, + "14": 0.0007461759744872732, + "15": 0.002270258432171722, + "16": 0.9856613874601285, + "17": 0.002377990136476756, + "18": 0.00046934667598904413, + "19": 0.00025688691128052305, + "20": 0.0003722373853222451, + "21": 0.00020373620837319315, + "22": 0.00016925020893311657, + "23": 0.0001856943613425135, + "24": 9.703137719490376e-05, + "25": 0.00032390356978848224, + "26": 0.00012234492329039483, + "27": 0.00044808354434749845, + "28": 0.00019450620837445083, + "29": 0.0001472736189929572, + "30": 0.00011151067635603395, + "31": 0.0006492874476259216 + }, + "confidence": 0.9856613874601285, + "expected_value": 15.976958510750352 + } + }, + "usage": { + "input_tokens": 392, + "output_tokens": 0 + } + } + }, + { + "id": "text-multi-structured", + "response": { + "model": "JEMM", + "answers": { + "z-last": { + "type": "noul", + "noul": 0.9973600010441932, + "probabilities": { + "yes": 0.9973600010441932, + "no": 0.0026399989558058457 + }, + "confidence": 0.9973600010441932 + }, + "a-first": { + "type": "choice", + "choice": "inspect", + "probabilities": { + "deploy": 0.0008283100881610059, + "inspect": 0.9972635136987317, + "erase": 0.001908176213108089 + }, + "confidence": 0.9972635136987317 + } + }, + "usage": { + "input_tokens": 184, + "output_tokens": 0 + } + } + }, + { + "id": "text-noul-false-special", + "response": { + "model": "JEMM", + "answers": { + "literal": { + "type": "noul", + "noul": 0.007286854768673532, + "probabilities": { + "yes": 0.007286854768673532, + "no": 0.9927131452313261 + }, + "confidence": 0.9927131452313261 + } + }, + "usage": { + "input_tokens": 90, + "output_tokens": 0 + } + } + }, + { + "id": "text-tool-json", + "response": { + "model": "JEMM", + "answers": { + "tool": { + "type": "choice", + "choice": "weather", + "probabilities": { + "weather": 0.9803244925599609, + "calendar": 0.0014876632799361764, + "none": 0.018187844160102214 + }, + "confidence": 0.9803244925599609 + } + }, + "usage": { + "input_tokens": 98, + "output_tokens": 0 + } + } + }, + { + "id": "image-console-256x256", + "response": { + "model": "JEMM", + "answers": { + "action": { + "type": "choice", + "choice": "logs", + "probabilities": { + "restart": 0.0006741098369049492, + "logs": 0.9982225979149069, + "delete": 0.0011032922481868399 + }, + "confidence": 0.9982225979149069 + }, + "healthy": { + "type": "noul", + "noul": 0.9961415805003403, + "probabilities": { + "yes": 0.9961415805003403, + "no": 0.003858419499657933 + }, + "confidence": 0.9961415805003403 + } + }, + "usage": { + "input_tokens": 298, + "output_tokens": 0 + } + } + }, + { + "id": "image-console-512x256", + "response": { + "model": "JEMM", + "answers": { + "action": { + "type": "choice", + "choice": "logs", + "probabilities": { + "restart": 0.0006446928232237257, + "logs": 0.9983905657287686, + "delete": 0.0009647414480069376 + }, + "confidence": 0.9983905657287686 + }, + "healthy": { + "type": "noul", + "noul": 0.9961415805003403, + "probabilities": { + "yes": 0.9961415805003403, + "no": 0.003858419499657933 + }, + "confidence": 0.9961415805003403 + } + }, + "usage": { + "input_tokens": 426, + "output_tokens": 0 + } + } + }, + { + "id": "multi-image-2", + "response": { + "model": "JEMM", + "answers": { + "choice": { + "type": "choice", + "choice": "first", + "probabilities": { + "first": 0.6718573558908757, + "second": 0.32814264410912486 + }, + "confidence": 0.6718573558908757 + }, + "noul": { + "type": "noul", + "noul": 0.950544668526123, + "probabilities": { + "yes": 0.950544668526123, + "no": 0.049455331473876415 + }, + "confidence": 0.950544668526123 + }, + "score": { + "type": "score", + "probabilities": { + "0": 0.2502581476808945, + "1": 0.39164918555887385, + "2": 0.35809266676023116 + }, + "confidence": 0.39164918555887385, + "expected_value": 1.1078345190793362 + } + }, + "usage": { + "input_tokens": 832, + "output_tokens": 0 + } + } + }, + { + "id": "multi-image-3", + "response": { + "model": "JEMM", + "answers": { + "choice": { + "type": "choice", + "choice": "first", + "probabilities": { + "first": 0.7621442009218663, + "second": 0.2378557990781345 + }, + "confidence": 0.7621442009218663 + }, + "noul": { + "type": "noul", + "noul": 0.9545895842813427, + "probabilities": { + "yes": 0.9545895842813427, + "no": 0.045410415718657456 + }, + "confidence": 0.9545895842813427 + }, + "score": { + "type": "score", + "probabilities": { + "0": 0.2837199632446548, + "1": 0.40597294910430337, + "2": 0.3103070876510422 + }, + "confidence": 0.40597294910430337, + "expected_value": 1.0265871244063878 + } + }, + "usage": { + "input_tokens": 1030, + "output_tokens": 0 + } + } + }, + { + "id": "multi-image-4", + "response": { + "model": "JEMM", + "answers": { + "choice": { + "type": "choice", + "choice": "first", + "probabilities": { + "first": 0.7779994364968656, + "second": 0.22200056350313488 + }, + "confidence": 0.7779994364968656 + }, + "noul": { + "type": "noul", + "noul": 0.9461597748181692, + "probabilities": { + "yes": 0.9461597748181692, + "no": 0.05384022518183118 + }, + "confidence": 0.9461597748181692 + }, + "score": { + "type": "score", + "probabilities": { + "0": 0.32323660311453684, + "1": 0.3535267937709264, + "2": 0.32323660311453684 + }, + "confidence": 0.3535267937709264, + "expected_value": 1.0 + } + }, + "usage": { + "input_tokens": 1420, + "output_tokens": 0 + } + } + } +] diff --git a/tests/jemm/parity.py b/tests/jemm/parity.py new file mode 100644 index 00000000..2bac6a38 --- /dev/null +++ b/tests/jemm/parity.py @@ -0,0 +1,100 @@ +#!/usr/bin/env python3 +"""One sequential pass over a frozen JEMM corpus; no retries or timing claims.""" +import argparse +import hashlib +import json +import math +from pathlib import Path +from urllib.request import Request, urlopen + +def finite(v): + return type(v) in (float, int) and math.isfinite(v) + + +def compare(reference, actual): + failures, rows = [], [] + if actual.get("model") != "JEMM" or reference.get("model") != "JEMM": + failures.append("model identity") + for k in ("input_tokens", "output_tokens"): + a, r = actual.get("usage", {}).get(k), reference.get("usage", {}).get(k) + if type(a) is not int or a < 0 or a != r: + failures.append("usage." + k) + latency = actual.get("usage", {}).get("latency_ms") + if not finite(latency) or latency <= 0: + failures.append("native latency must be finite and positive") + ra, aa = reference["answers"], actual.get("answers", {}) + if list(ra) != list(aa): + failures.append("question identity/order") + for qid, r in ra.items(): + a = aa.get(qid, {}) + problem = [] + if a.get("type") != r["type"] or not set(r) <= set(a): + problem.append("answer type/required fields") + rp, ap = r["probabilities"], a.get("probabilities", {}) + if list(rp) != list(ap): + problem.append("candidate identity/order") + valid = all(finite(v) and 0 <= v <= 1 for v in list(rp.values()) + list(ap.values())) + if not valid or not ap or abs(sum(ap.values()) - 1) > 1e-6: + problem.append("finite normalized probabilities") + drift = max((abs(v - ap[k]) for k, v in rp.items() if k in ap and finite(ap[k])), default=math.inf) + if drift > 0.02 + 1e-12: + problem.append("probability drift") + ordered = sorted(rp, key=rp.get, reverse=True) + margin = rp[ordered[0]] - rp[ordered[1]] + winner = max(ap, key=ap.get) if valid and ap else None + if margin >= 0.05 and winner != ordered[0]: + problem.append("winner disagreement") + if r["type"] == "choice" and a.get("choice") != winner: + problem.append("choice disagrees with own probabilities") + for k, gate in (("confidence", 0.02), ("noul", 0.02), ("expected_value", 0.1)): + if k in r and (not finite(a.get(k)) or abs(r[k] - a[k]) > gate + 1e-12): + problem.append(k + " drift") + rows.append({"question": qid, "probability_drift": drift, "reference_margin": margin, + "low_margin": margin < 0.05, "failures": problem}) + failures.extend(qid + ": " + p for p in problem) + return {"passed": not failures, "failures": failures, "questions": rows} + + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--url", required=True, help="running worker URL on a reserved GPU") + parser.add_argument("--corpus", type=Path, required=True) + parser.add_argument("--reference", type=Path, required=True) + parser.add_argument("--report", type=Path, required=True) + args = parser.parse_args() + requests = [json.loads(line) for line in args.corpus.read_text().splitlines()] + references = json.loads(args.reference.read_text()) + assert len(requests) == len(references) > 0 + assert [x["id"] for x in requests] == [x["id"] for x in references] + url = args.url.rstrip("/") + with urlopen(url + "/health", timeout=5) as response: + assert json.load(response)["status"] == "READY" + records, responses = [], [] + for item, reference in zip(requests, references): + request = Request(url + "/v1/systemone", + data=json.dumps(item["request"], ensure_ascii=False).encode(), + headers={"Content-Type": "application/json"}) + with urlopen(request, timeout=120) as response: + actual = json.load(response) + record = {"id": item["id"], **compare(reference["response"], actual)} + records.append(record) + responses.append({"id": item["id"], "response": actual}) + print(json.dumps(record), flush=True) + with urlopen(url + "/health", timeout=5) as response: + ready = json.load(response)["status"] == "READY" + report = {"passed": ready and all(x["passed"] for x in records), + "requests": len(requests), "questions": sum(len(x["questions"]) for x in records), + "gates": {"probability": 0.02, "score": 0.1, "winner_margin": 0.05}, + "max_probability_drift": max(q["probability_drift"] for x in records for q in x["questions"]), + "records": records, "responses": responses, + "corpus_sha256": hashlib.sha256(args.corpus.read_bytes()).hexdigest(), + "reference_sha256": hashlib.sha256(args.reference.read_bytes()).hexdigest(), + "runner_sha256": hashlib.sha256(Path(__file__).read_bytes()).hexdigest(), + "performance_measurement": False, "retries": 0} + args.report.write_text(json.dumps(report, indent=2, allow_nan=False) + "\n") + raise SystemExit(0 if report["passed"] else 1) + + +if __name__ == "__main__": + main() diff --git a/tests/jemm/reference_kernels.py b/tests/jemm/reference_kernels.py new file mode 100644 index 00000000..6943b871 --- /dev/null +++ b/tests/jemm/reference_kernels.py @@ -0,0 +1,133 @@ +#!/usr/bin/env python3 +"""GPU regression for the optional JEMM arithmetic and capture-safe workspaces. + +Requires the pinned Torch/FLA environment and a cuDNN-enabled CUDA library. +Run only under exclusive GPU admission; this is correctness, not benchmarking. +""" +import argparse +import ctypes as ct +import json +from pathlib import Path + +import torch +import torch.nn.functional as F +from fla.ops.gated_delta_rule import chunk_gated_delta_rule + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("library", type=Path) + parser.add_argument("--report", type=Path, required=True) + args = parser.parse_args() + torch.manual_seed(20261011) + torch.cuda.set_device(0) + execution_stream = torch.cuda.Stream() + torch.cuda.set_stream(execution_stream) + lib = ct.CDLL(str(args.library.resolve())) + ptr, integer = ct.c_void_p, ct.c_int + signatures = { + "gdn_workspace_floats": ([integer, integer], ct.c_size_t), + "gdn_prefill": ([ptr] * 7 + [integer] * 3 + [ct.c_float, ptr], integer), + "language_attention_workspace_floats": ([integer, integer], ct.c_size_t), + "vision_attention_workspace_floats": ([integer, integer], ct.c_size_t), + "attention": ([ptr, ptr, ptr, integer, ptr, ptr, integer, integer, integer, ct.c_float, ptr, ptr], integer), + "vision_attention": ([ptr] * 4 + [integer] * 3 + [ptr, ptr], integer), + "rms_norm": ([ptr] * 3 + [integer, integer, ct.c_float, ptr], integer), + "patch_create": ([integer, ptr], ptr), + "patch": ([ptr] * 4, integer), + "patch_destroy": ([ptr], None), + } + api = {} + for name, (arguments, result) in signatures.items(): + fn = getattr(lib, "cs1_reference_" + name) + fn.argtypes, fn.restype = arguments, result + api[name] = fn + stream = ptr(torch.cuda.current_stream().cuda_stream) + p = lambda tensor: ptr(tensor.data_ptr()) + rand = lambda shape: torch.randn(shape, device="cuda", dtype=torch.bfloat16) + records = [] + + def verify(name, actual, expected, invoke, max_abs=0.0, relative_l2=0.0): + assert invoke() == 0, name + torch.cuda.synchronize() + difference = actual.float() - expected.float() + record = {"name": name, "elements": actual.numel(), + "different": int((actual != expected).sum()), + "max_abs": float(difference.abs().max()), + "relative_l2": float(difference.norm() / expected.float().norm().clamp_min(1e-30))} + assert record["max_abs"] <= max_abs and record["relative_l2"] <= relative_l2, record + eager = actual.clone() + graph = torch.cuda.CUDAGraph() + with torch.cuda.graph(graph, stream=torch.cuda.current_stream()): + assert invoke() == 0, name + actual.zero_() + graph.replay() + torch.cuda.synchronize() + assert torch.equal(actual, eager), f"{name}: capture/replay differs from eager" + record["graph_replay_exact"] = True + records.append(record) + print(json.dumps(record), flush=True) + + for tokens in (63, 278, 475): + heads, dim = 48, 128 + q, k, v = (rand((1, tokens, heads, dim)) for _ in range(3)) + g = -torch.rand((1, tokens, heads), device="cuda", dtype=torch.float32) + beta = torch.sigmoid(rand((1, tokens, heads))) + expected, _ = chunk_gated_delta_rule(q, k, v, g, beta, use_qk_l2norm_in_kernel=True) + out = torch.empty_like(v) + workspace = torch.empty(api["gdn_workspace_floats"](tokens, heads), device="cuda", dtype=torch.float32) + invoke = lambda: api["gdn_prefill"](p(q), p(k), p(v), p(g), p(beta), p(out), p(workspace), tokens, heads, heads, dim ** -0.5, stream) + verify(f"gdn-{tokens}", out, expected, invoke) + + for tokens in (63, 278, 475): + heads, kv_heads, dim = 24, 4, 256 + q = rand((tokens, heads, dim)) + k, v = (rand((tokens, kv_heads, dim)) for _ in range(2)) + gate = rand(q.shape) + with torch.nn.attention.sdpa_kernel(torch.nn.attention.SDPBackend.FLASH_ATTENTION): + attention = F.scaled_dot_product_attention(q.transpose(0, 1).unsqueeze(0), + k.transpose(0, 1).unsqueeze(0), v.transpose(0, 1).unsqueeze(0), + is_causal=True, enable_gqa=True).squeeze(0).transpose(0, 1).contiguous() + expected = attention * torch.sigmoid(gate) + out = torch.empty_like(q) + workspace = torch.empty(api["language_attention_workspace_floats"](512, heads), device="cuda", dtype=torch.float32) + invoke = lambda: api["attention"](p(q), p(k), p(v), kv_heads * dim, p(gate), p(out), tokens, heads, kv_heads, dim ** -0.5, p(workspace), stream) + verify(f"attention-{tokens}", out, expected, invoke) + + for rows in (1, 278): + x, w = rand((rows, 5120)), rand((5120,)) + expected = (x.float() * torch.rsqrt(x.float().square().mean(-1, keepdim=True) + 1e-6) * (1 + w.float())).bfloat16() + out = torch.empty_like(x) + verify(f"rms-{rows}", out, expected, + lambda: api["rms_norm"](p(x), p(w), p(out), rows, 5120, 1e-6, stream)) + + for tokens in (256, 512): + heads, dim = 16, 72 + q, k = rand((tokens, heads, dim)), rand((tokens, heads, dim)) + qkv = rand((tokens, 3, heads, dim)) + v = qkv[:, 2] + with torch.nn.attention.sdpa_kernel(torch.nn.attention.SDPBackend.FLASH_ATTENTION): + expected = F.scaled_dot_product_attention(q.transpose(0, 1).unsqueeze(0), + k.transpose(0, 1).unsqueeze(0), v.transpose(0, 1).unsqueeze(0)).squeeze(0).transpose(0, 1).contiguous() + out = torch.empty_like(q) + floats = api["vision_attention_workspace_floats"](512, heads) + workspace = torch.empty(512 * heads * 80 * 4 * 2 + floats * 4, device="cuda", dtype=torch.uint8) + verify(f"vision-attention-{tokens}", out, expected, + lambda: api["vision_attention"](p(q), p(k), p(v), p(out), tokens, heads, dim, p(workspace), stream)) + + x, w = rand((256, 3, 2, 16, 16)), rand((1152, 3, 2, 16, 16)) + expected = F.conv3d(x, w, stride=(2, 16, 16)).reshape(256, 1152) + out = torch.empty_like(expected) + plan = api["patch_create"](256, stream) + assert plan, "cuDNN patch plan creation" + try: + verify("patch-convolution", out, expected, lambda: api["patch"](plan, p(x), p(w), p(out))) + finally: + torch.cuda.synchronize() + api["patch_destroy"](plan) + args.report.write_text(json.dumps({"passed": True, "records": records, + "torch": torch.__version__, "gpu": torch.cuda.get_device_name(0)}, indent=2) + "\n") + + +if __name__ == "__main__": + main() diff --git a/tests/qwen3_5/vision_batch.rs b/tests/qwen3_5/vision_batch.rs index 94914de8..a996af1e 100644 --- a/tests/qwen3_5/vision_batch.rs +++ b/tests/qwen3_5/vision_batch.rs @@ -193,6 +193,7 @@ fn cpu_batch_uses_aggregate_gemms_and_image_local_attention() { scratch: VecDeque::new(), batch_scratch: VecDeque::new(), graph_enabled: false, + reference: None, base, lora: BTreeMap::new(), gemm: Gemm(unsafe { (api().cs1_gemm_create)(32 << 20) }), From 91cc5c2574e3508d34a990534dffed52ee7cefed Mon Sep 17 00:00:00 2001 From: levius <2114377220@qq.com> Date: Sun, 11 Oct 2026 08:41:09 +0800 Subject: [PATCH 13/21] docs(jemm): align registration with v2 multi-image validation --- README.md | 2 +- docs/supported-models.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 6389438b..f5b55932 100644 --- a/README.md +++ b/README.md @@ -166,7 +166,7 @@ Python screenshot worker, and CLM has a stub-encoder contract recipe: | LAYA | [External worker](recipe/laya/README.md); [Python worker on Apple Silicon (MPS) and CPU](recipe/laya/apple-silicon.md); [CPU checkpoint reader](src/models/laya/README.md); [native Rust/CUDA worker on Hopper](recipe/laya/native/README.md) | | Cua-S1 4B 0.2 (`text` adapter) | [Python worker](recipe/cua_s1/text.md); [native worker](recipe/cua_s1/native.md), CUDA, run on sm_89 | | Cua-S1 4B 0.2 (`multimodal` adapter) | [Python CUDA worker](src/frontend/cua_s1.py); one PNG/JPEG screenshot, `choice`; native screenshot execution remains in progress | -| JEMM (Qwen3.8-27B) | [Native Rust/CUDA worker](recipe/jemm/README.md); text/single-image parity on A800; [known multi-image limitation](src/models/jemm/README.md#validation-and-known-multi-image-limitation) | +| JEMM (Qwen3.8-27B) | [Native Rust/CUDA worker](recipe/jemm/README.md); [text and 1–4-image frozen-corpus parity on A800](src/models/jemm/README.md#validation) | | Open-Jev-27B-v1.1 | [Native Rust/CUDA worker](recipe/open_jev/native.md); eager independent text candidates; [H200 validation](recipe/open_jev/validation.md) | | Open-Jev-9B | The same [native Rust/CUDA worker](recipe/open_jev/native.md); eager independent text candidates; [reference comparison on sm_89](recipe/open_jev/validation-9b.md) | | autotrust/JEV-27B-VL | [Experimental Rust/CUDA worker](recipe/jev_vl/README.md); single-question text and offline-preencoded image inputs; [bounded H800 validation and limits](recipe/jev_vl/validation.md) | diff --git a/docs/supported-models.md b/docs/supported-models.md index 71a37189..4cda1e71 100644 --- a/docs/supported-models.md +++ b/docs/supported-models.md @@ -17,7 +17,7 @@ Models that are being added are also tracked in issues labeled [new model](https | Cua-S1 4B 0.2, `multimodal` adapter | Reference worker on Transformers and PEFT, [`src/frontend/cua_s1.py`](../src/frontend/cua_s1.py); no recipe yet | Not supported | Validated ([#17](https://github.com/ThinkFlowLab/system1-omni/pull/17), [#18](https://github.com/ThinkFlowLab/system1-omni/pull/18)) | Not supported | The state is one PNG or JPEG image; upstream's `weights.lock.json` next to the base weights | | Open-Jev-27B-v1.1 | [Native Rust/CUDA worker](../recipe/open_jev/native.md) on the shared Qwen3.5/3.8 executor | Not supported | Validated on H200 (sm_90) for the [74 single-candidate workload](../recipe/open_jev/validation.md) | Not supported | Compute capability 8.0 or newer, CUDA toolkit to build, exported merged weights and trained head | | Open-Jev-9B | The same [native Rust/CUDA worker](../recipe/open_jev/native.md) | Not supported | Validated on compute capability 8.9 against the reference for [253 requests](../recipe/open_jev/validation-9b.md) | Not supported | Compute capability 8.0 or newer, CUDA toolkit to build, exported merged weights and trained head | -| JEMM / Qwen3.8-27B | [Native Rust/CUDA worker](../recipe/jemm/README.md) | Host checks only; no CPU inference | A800 (`sm_80`) text/single-image corpus; [two-/four-image parity limitation](../src/models/jemm/README.md#validation-and-known-multi-image-limitation) | Not supported | Pinned merged export, selected untied LM head, additive v2 vision symbols | +| JEMM / Qwen3.8-27B | [Native Rust/CUDA worker](../recipe/jemm/README.md) | Host checks only; no CPU inference | A800 (`sm_80`) [text and 1–4-image frozen corpus](../src/models/jemm/README.md#validation) | Not supported | Pinned `jemm-native/2` export with BF16 base and unmerged FP32 LoRA, selected untied LM head, cuDNN 9 and additive reference CUDA symbols | | autotrust/JEV-27B-VL | [Experimental native worker](../recipe/jev_vl/README.md), single-question API | Contract tests only; no CPU inference | H800 frozen-corpus check; [experimental scope and limits](../recipe/jev_vl/validation.md) | Not supported | Pinned merged checkpoint, rebuilt ABI 6 CUDA library; image inputs require offline Transformers preencoding | | CLM-v0.1-8B | [External `clm-serve` recipe](../recipe/clm/README.md) with a CPU stub embeddings server | **Stub-encoder contract checks only** ([#23](https://github.com/ThinkFlowLab/system1-omni/pull/23)); not real Qwen3-8B decisions | Real encoder unverified by the merged recipe | Unverified | Python, upstream CLM and head checkpoint; a real encoder requires a separate embeddings server | From c896840da764fcba99762cb5966f44aa9f266466 Mon Sep 17 00:00:00 2001 From: levius <2114377220@qq.com> Date: Tue, 13 Oct 2026 17:58:32 +0800 Subject: [PATCH 14/21] Fix Cua-S1 text parity with the pinned no-FLA reference --- README.md | 2 +- docs/architecture.md | 5 +- .../cua-s1-text-parity-20261013/README.md | 128 +++++++ mkdocs.yml | 1 + recipe/README.md | 3 +- recipe/cua_s1/export_text_unmerged.py | 84 +++++ recipe/cua_s1/native.md | 36 +- src/backends/cuda/qwen3_5/README.md | 21 ++ src/backends/cuda/qwen3_5/build.sh | 8 +- src/backends/cuda/qwen3_5/gdn_fp32.cu | 352 ++++++++++++++++++ src/backends/cuda/qwen3_5/ops.h | 8 + .../cuda/qwen3_5/qwen3_5.backend.json | 1 + src/backends/cuda/qwen3_5/reference_norm.cu | 4 +- src/models/cua_s1/README.md | 24 +- src/models/cua_s1/native/src/engine.rs | 15 +- src/models/cua_s1/native/src/executor.rs | 26 +- src/models/cua_s1/native/src/provenance.rs | 31 +- src/models/qwen3_5/native/src/cuda.rs | 110 ++++++ src/models/qwen3_5/native/src/lora.rs | 1 + src/models/qwen3_5/native/src/model.rs | 96 ++++- tests/cua_s1/head.rs | 16 +- tests/cua_s1/text_export.rs | 62 +++ tests/qwen3_5/lora.rs | 23 ++ tests/qwen3_5/test_gdn_fp32.py | 129 +++++++ 24 files changed, 1138 insertions(+), 48 deletions(-) create mode 100644 docs/benchmarks/cua-s1-text-parity-20261013/README.md create mode 100644 recipe/cua_s1/export_text_unmerged.py create mode 100644 src/backends/cuda/qwen3_5/gdn_fp32.cu create mode 100644 tests/cua_s1/text_export.rs create mode 100644 tests/qwen3_5/test_gdn_fp32.py diff --git a/README.md b/README.md index ddd896fe..f1994af2 100644 --- a/README.md +++ b/README.md @@ -167,7 +167,7 @@ v1.1 has its checkpoint export and CPU contract, with no worker yet: | Model | Status | | --- | --- | | LAYA | [External worker](recipe/laya/README.md); [Python worker on Apple Silicon (MPS) and CPU](recipe/laya/apple-silicon.md); [CPU checkpoint reader](src/models/laya/README.md); [native Rust/CUDA worker on Hopper](recipe/laya/native/README.md) | -| Cua-S1 4B 0.2 (`text` adapter) | [Python worker](recipe/cua_s1/text.md); [native worker](recipe/cua_s1/native.md), CUDA, run on sm_89 | +| Cua-S1 4B 0.2 (`text` adapter) | [Python worker](recipe/cua_s1/text.md); [native worker](recipe/cua_s1/native.md), CUDA; unmerged profile validated on the cumulative A800 sm_80 build, earlier merged path on sm_89 | | Cua-S1 4B 0.2 (`multimodal` adapter) | [Python CUDA worker](src/frontend/cua_s1.py); one PNG/JPEG screenshot, `choice`; native screenshot execution remains in progress | | JEMM (Qwen3.8-27B) | [Native Rust/CUDA worker](recipe/jemm/README.md); [text and 1–4-image frozen-corpus parity on A800](src/models/jemm/README.md#validation) | | Valen-Preview-0923 | [Python reference worker](recipe/valen/reference.md); text state or one inline PNG/JPEG image, `choice`; author-run on RTX 5060 Laptop GPU (sm_120, WSL2), owner: community | diff --git a/docs/architecture.md b/docs/architecture.md index 9fb69e11..433c5ddd 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -56,8 +56,9 @@ dispatch retains the scheduler permit until execution completes. Laya synchronizes and disposes a failed model before returning an inference error and reports unavailable health thereafter. -Qwen executors own loaded models and CPU head weights, preserving FP64 accumulation and -the existing FP32 rounding and bias order. Finishing checks output cardinality +Qwen executors own loaded models and CPU head weights with FP64 accumulation. +Cua-S1 rounds answer-letter logits to the tied `lm_head` BF16 output dtype before +FP32 softmax; learned scalar heads retain their FP32 rounding and bias order. Finishing checks output cardinality before reconstruction. HTTP validation, error status/body conventions, and real warmup before readiness remain model-specific and unchanged. diff --git a/docs/benchmarks/cua-s1-text-parity-20261013/README.md b/docs/benchmarks/cua-s1-text-parity-20261013/README.md new file mode 100644 index 00000000..9baf163d --- /dev/null +++ b/docs/benchmarks/cua-s1-text-parity-20261013/README.md @@ -0,0 +1,128 @@ +# Cua-S1 text native/reference parity on A800 + +This investigation follows the five decision flips reported in +[PR #141](https://github.com/ThinkFlowLab/system1-omni/pull/141). +The original L20X run used a different GPU. On A800, the frozen 139-request corpus +produced two stable flips, not the original five. Two independent HTTP passes +per worker reproduced both flips. This report qualifies the A800 environment; +it does not establish that all five original L20X cases have been reproduced. + +## PR dependencies and evidence provenance + +This independent change is stacked on [#118](https://github.com/ThinkFlowLab/system1-omni/pull/118) +(`codex/jemm-native`, `ed9f9d6eb566d226e2441c5e9d84187abe1e4d63`) for the +unmerged FP32 language adapter loader and optional reference norm/GEMM/attention +operations. #118 is itself stacked on [#117](https://github.com/ThinkFlowLab/system1-omni/pull/117). +The frozen corpus comes from #141; merging #141 is not a runtime dependency. +Decider execution modes and Cua screenshot cache/resident-feature PRs are outside +this change's dependency chain and diff. + +The GPU results below were measured on cumulative candidate +`76af07996f60905082b8d48199d4a7003f33f4f6`, whose parent is +`18e1a5a6f68b9d11d3e11c7ea7b939af87272f53`. That integrated build used ABI 8. +The independently rebased branch retains #118's ABI 6 and adds only optional +FP32 GDN symbols. Rebuild the library with the worker; no cuDNN is needed for +this text path. Results from the cumulative build are historical evidence for +the pinned implementation, not a fresh GPU qualification of the rebased PR head. +The GPU server was shut down after the campaign; no new device run is implied +by packaging this PR. CPU/build/docs checks are run again on the independent head. + +## Frozen inputs and environment + +- Baseline source: `3e9cc33a61ddf110d97512fb6f296cf540377c61`. +- Corpus: PR #141 `869260649f7dd5b56cac4d91eaae9087f2e985ee`, + `benchmarks/e2e/choice.jsonl`; 139 requests/questions, 172–3988 prompt tokens. +- Historical GPU candidate parent: cumulative ABI-8/unmerged-LoRA shared executor, + `18e1a5a6f68b9d11d3e11c7ea7b939af87272f53`. +- GPU: A800 PCIe 80 GB, sm_80; CUDA toolkit 13.0.88; NVIDIA driver 595.71.05. +- Python 3.12, Torch 2.14.0+cu130, Transformers 5.17.0, PEFT 0.21.0; + TF32 disabled in the independent reference; no FLA or causal-conv1d installed. +- Base `Qwen/Qwen3.5-4B@851bf6e806efd8d0a36b00ddf55e13ccb7b8cd0a`; + text adapter `cua-ai/cua-s1-4b-0.2@16818868b0cc7813808aae4e87b417657046ab79`. + Pinned files were SHA-256 checked. The new export copies the original adapter + unchanged, with hash `9b59c5aed96171a50b26526613766bbf44347a5c7af70f81efe6bcc6e9dbfb0e`. + +The reference worker uses the repository's independent Transformers/PEFT text +implementation. No reference logits, labels or probabilities are modified. +Requests are sent through the Rust frontend to a real worker. The missing model +field in corpus rows is filled with the required `cua-s1-4b-0.2`, as the benchmark +wire adapter does. Raw request IDs, bodies, statuses, usage and model identities +are retained with source/build/launch manifests in the local investigation archive. + +## Confirmed numerical causes + +Token IDs matched on all 139 requests. The differences begin in execution: + +1. The old export merges FP32 LoRA into BF16 base weights. PEFT instead computes + the unmerged LoRA branch in FP32 and casts the combined projection to BF16. +2. The old GEMM reduction policy and normalization reductions differ from the + pinned framework. With identical first-layer MLP down-projection input, + 244,845 BF16 base outputs differed; the existing reference GEMM handle matched. + With identical normalization input, two BF16 values differed; reference norm + matched after adding the 2560-hidden shape. +3. The old GDN uses TF32 pair products and BF16 chunk intermediates/state; + the no-FLA Torch implementation uses FP32 chunked triangular solves and state. + On identical first-layer input, old relative L2 error was 0.00217711 and + 326,687 of 954,368 BF16 outputs differed. Keeping recurrence FP32 alone did not + match the reference's chunk arithmetic. Matching the chunked computation and + Torch's cumulative-decay scan association produced identical BF16 outputs. +4. Native letter projection previously emitted FP32 logits. Reference `lm_head` + emits BF16 before casting to FP32 softmax. Restoring that BF16 boundary matters + for tied options and removes the remaining readout differences. + +The fix selects the precision path using `cua-s1-text-unmerged/1`. Legacy merged +exports remain loadable. Other model adapters retain their existing numerical +mode. Prefix continuation is rejected in the new mode; eager is the default. +The probability gate of 0.02 was not relaxed. + +## Results + +| Real HTTP mode | Passes × requests | Flips vs reference | Maximum probability difference | Correct | +| --- | --- | --- | --- | --- | +| Baseline native | 2 × 139 | 2 | 0.04872608184814453 | 97/139 | +| Fixed eager | 2 × 139 | 0 | 5.960464477539063e-8 | 97/139 | +| Fixed Graph | 2 × 139 | 0 | 5.960464477539063e-8 | 97/139 | + +Every fixed request returned HTTP 200. Each mode's two passes produced identical +response bytes; model identity, usage and option/question order matched reference. +Graph trace independently records 207 captures and 72 replays, with no fallback. +The two baseline flips were `original-routing-06-1` (tools/general) and +`hard-sol-b-temporal_numeric-03` (9_credits/12_credits); both reference maxima +are ties. Accuracy staying at 97/139 is expected: this repairs implementation +parity and does not retrain or improve the model. + +A controlled full-model probe additionally matched all 139 final BF16 hidden +vectors and selected BF16 logit rows exactly. The slight HTTP probability drift +comes after that BF16 boundary in FP32 softmax/response finishing. + +Fourteen maintained GPU cases compare the new GDN with the pinned no-FLA Torch +implementation: 1, 31, 32, 33, 63, 64, 65, 127, 128, 129, 233, 513 and 2049 tokens +with 32 value/16 key heads, plus 129 tokens with 4 value/2 key heads. All BF16 +outputs matched exactly. Repeated changed-input execution resets state, invalid +shapes/scales fail, and repeated context creation/destruction releases scratch. + +## Reproduction and limits + +Use the [native text recipe](../../../recipe/cua_s1/native.md) to build sm_80, +export original FP32 LoRA separately and start the worker. Use the pinned +[reference text recipe](../../../recipe/cua_s1/text.md) in an environment without +FLA/causal-conv1d, then replay the frozen corpus twice through each worker/frontend. +Run the maintained operator regression in that reference environment: + +```sh +.venv/bin/python tests/qwen3_5/test_gdn_fp32.py \ + --library target/release/libqwen3_5_cuda.so +``` + +The historical integrated candidate passed Cargo fmt, full-workspace Clippy +with warnings denied, full-workspace tests and +release build with `--locked --features omni-laya/serve` where applicable. +Backend contract checker/tests, 21 pinned Python text contract/server tests, CUDA +operator regressions and strict MkDocs build also passed. The checkpoint-dependent +prefix test used canonical merged text weights with the original pinned multimodal +config for a zero-image contract fixture; it is not numerical-reference evidence. + +Independent-PR-head GPU execution, other GPUs, framework versions, FLA execution +and screenshot-model numerical +parity require separate qualification. This corpus does not establish general +GUI accuracy or a performance improvement. diff --git a/mkdocs.yml b/mkdocs.yml index e8488667..69226bac 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -65,6 +65,7 @@ nav: - Laya on Apple Silicon: recipe/laya/apple-silicon.md - Cua-S1 text worker: recipe/cua_s1/text.md - Cua-S1 native text worker: recipe/cua_s1/native.md + - Cua-S1 text numerical parity: docs/benchmarks/cua-s1-text-parity-20261013/README.md - Valen reference worker: recipe/valen/reference.md - Cua-S1 vision CUDA Graph: recipe/cua_s1/native_vision_graph.md - Open-Jev native text worker: recipe/open_jev/native.md diff --git a/recipe/README.md b/recipe/README.md index c2a31891..e85b7d6b 100644 --- a/recipe/README.md +++ b/recipe/README.md @@ -14,7 +14,8 @@ For a first real decision, follow the [complete CPU walkthrough](../docs/getting - [Valen-Preview-0923 reference worker](valen/reference.md): prepare the pinned Preview checkpoint and Qwen base, start the text-or-image `choice` worker and verify the frontend. - [Cua-S1 4B 0.2 native text worker](cua_s1/native.md): build the CUDA library and - the Rust worker, export the merged weights and start the worker. + the Rust worker, export BF16 weights with the original FP32 adapter and start + the worker. - [Open-Jev native text worker](open_jev/native.md): export the merged text backbone and trained decision head of Open-Jev-27B-v1.1 or Open-Jev-9B, then serve with Rust and CUDA. [Open-Jev-9B validation](open_jev/validation-9b.md) diff --git a/recipe/cua_s1/export_text_unmerged.py b/recipe/cua_s1/export_text_unmerged.py new file mode 100644 index 00000000..d78e1657 --- /dev/null +++ b/recipe/cua_s1/export_text_unmerged.py @@ -0,0 +1,84 @@ +"""Export BF16 language weights with the original FP32 text adapter kept separate.""" + +import argparse +import hashlib +import json +import shutil +from pathlib import Path + + +def validate_adapter(config): + targets = { + "q_proj", + "k_proj", + "v_proj", + "o_proj", + "gate_proj", + "up_proj", + "down_proj", + } + if config.get("r") != 16 or config.get("lora_alpha") != 32: + raise ValueError("Cua-S1 0.2 text requires rank 16 and alpha 32") + if set(config.get("target_modules", [])) != targets: + raise ValueError("expected the Cua-S1 text adapter, not a multimodal adapter") + if ( + config.get("bias", "none") != "none" + or config.get("lora_bias") + or config.get("use_dora") + or config.get("use_rslora") + or config.get("fan_in_fan_out") + or config.get("modules_to_save") + or config.get("rank_pattern") + or config.get("alpha_pattern") + ): + raise ValueError("unsupported text adapter scaling or projection policy") + + +def export(base: Path, adapter: Path, out: Path): + validate_adapter(json.loads((adapter / "adapter_config.json").read_text())) + if out.exists() and any(out.iterdir()): + raise ValueError("output directory must be empty") + import torch + from safetensors import safe_open + from transformers import AutoModelForCausalLM, AutoTokenizer + + adapter_file = adapter / "adapter_model.safetensors" + with safe_open(adapter_file, framework="pt", device="cpu") as tensors: + if len(tensors.keys()) != 256 or any( + tensors.get_slice(name).get_dtype() != "F32" for name in tensors.keys() + ): + raise ValueError("expected the complete 256-tensor FP32 text adapter") + model = AutoModelForCausalLM.from_pretrained( + base, dtype=torch.bfloat16, device_map="cpu" + ) + if model.config.hidden_size != 2560 or model.config.num_hidden_layers != 32: + raise ValueError("expected the Qwen3.5-4B language model") + out.mkdir(parents=True, exist_ok=True) + model.save_pretrained(out, max_shard_size="5GB") + AutoTokenizer.from_pretrained(base).save_pretrained(out) + shutil.copyfile(adapter_file, out / "adapter.safetensors") + with (out / "adapter.safetensors").open("rb") as file: + digest = hashlib.file_digest(file, "sha256").hexdigest() + (out / "cua_s1_export.json").write_text( + json.dumps( + { + "format": "cua-s1-text-unmerged/1", + "lora_scale": 2.0, + "adapter_sha256": digest, + }, + indent=2, + ) + ) + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--base", required=True, type=Path) + parser.add_argument("--adapter", required=True, type=Path) + parser.add_argument("--out", required=True, type=Path) + args = parser.parse_args() + export(args.base, args.adapter, args.out) + + +if __name__ == "__main__": + main() diff --git a/recipe/cua_s1/native.md b/recipe/cua_s1/native.md index 78d14b1a..2a23a35f 100644 --- a/recipe/cua_s1/native.md +++ b/recipe/cua_s1/native.md @@ -1,26 +1,26 @@ # Cua-S1 4B 0.2 native text worker -The native worker ([`src/models/cua_s1/native/`](../../src/models/cua_s1/native/)) serves the `text` adapter like the reference worker in [`text.md`](text.md), with the Qwen3.5-4B forward pass on the CUDA kernels in [`src/backends/cuda/qwen3_5/`](../../src/backends/cuda/qwen3_5/) and no Python or PyTorch. It needs an NVIDIA GPU with compute capability 8.0 or newer; only an RTX 6000 Ada (sm_89) with CUDA 13.2 has been run. +The native worker ([`src/models/cua_s1/native/`](../../src/models/cua_s1/native/)) serves the `text` adapter like the reference worker in [`text.md`](text.md), with the Qwen3.5-4B forward pass on the CUDA kernels in [`src/backends/cuda/qwen3_5/`](../../src/backends/cuda/qwen3_5/) and no Python or PyTorch. It needs an NVIDIA GPU with compute capability 8.0 or newer; the unmerged precision profile was checked on the cumulative A800 80 GB (sm_80) build with CUDA 13.0; device qualification of this independent #118-based branch remains pending. The earlier merged path was checked on an RTX 6000 Ada (sm_89) with CUDA 13.2. Run the commands from the repository root. Pass your GPU's compute capability to `build.sh` (89 for Ada, 80 for A100, 90 for H100); the worker finds `libqwen3_5_cuda.so` next to its executable, or at `CUA_S1_CUDA_LIB`: ```sh -src/backends/cuda/qwen3_5/build.sh target/release 89 # needs nvcc and cuBLASLt +src/backends/cuda/qwen3_5/build.sh target/release 89 # needs nvcc, cuBLAS and cuBLASLt cargo build --release --locked -p omni-cua-s1-native ``` -The worker loads the weights with the `text` adapter merged in. With the reference worker's environment and weights from `text.md`, export them once (about 8.5 GB): +With the pinned reference environment and weights from `text.md`, export the BF16 language weights and retain the original FP32 text adapter separately (about 8.5 GB): ```sh -PYTHONPATH=src .venv/bin/python recipe/cua_s1/export_text_merged.py \ +PYTHONPATH=src .venv/bin/python recipe/cua_s1/export_text_unmerged.py \ --base weights/Qwen3.5-4B --adapter weights/cua-s1-4b-0.2/text \ - --out weights/cua-s1-4b-0.2-text-merged + --out weights/cua-s1-4b-0.2-text-unmerged ``` Start the worker (`CUA_S1_HOST` and `CUA_S1_PORT` default to `127.0.0.1` and `8000`), then the frontend and requests as in `text.md`: ```sh -CUA_S1_MODEL=weights/cua-s1-4b-0.2-text-merged target/release/omni-cua-s1-native +CUA_S1_MODEL=weights/cua-s1-4b-0.2-text-unmerged target/release/omni-cua-s1-native ``` For the local CUDA Graph experiment, also set `CUA_S1_GRAPH=1`. The first use of @@ -31,12 +31,34 @@ the eager control. Rebuild both the worker and CUDA library together (ABI 6). If capture fails, the worker returns the completed eager result and disables Graph capture/replay for its remaining lifetime, logging the failure to stderr. -Each question runs one forward pass over its prompt, eagerly by default or through exact-length CUDA Graph replay when enabled; the final hidden state at the last position times the 26 letter rows of the output projection gives the option probabilities. The probabilities are not bitwise identical to the reference worker's, since the adapter is merged and the kernels differ; they are held to the tolerance in [`src/models/cua_s1/README.md`](../../src/models/cua_s1/README.md#validation). Error messages are worded differently, and bodies nested more than 127 levels deep are refused. +Each question runs one forward pass over its prompt. The `cua-s1-text-unmerged/1` +marker selects framework-compatible BF16 normalization and cuBLASLt GEMM, +FP32 LoRA branches, and FP32 chunked Gated DeltaNet. The GDN cumulative decay +uses the pinned Torch scan order; state is reset for every prompt. The letter +projection rounds its result to BF16 before FP32 softmax, matching `lm_head`. +The marker records scale 2 and the SHA-256 of the unchanged adapter; incompatible +markers or altered adapter bytes fail before CUDA loading. Prefix continuation +is unsupported in this precision mode. + +Existing `cua-s1-text-merged/1` exports remain loadable with the earlier merged +execution path. Merging the adapter, TF32/BF16 GDN intermediates, and different +GEMM reduction policies can change decisions on close or tied options. Re-export +with the command above to use the precision profile. Rebuild worker and library +together: this branch retains #118's ABI 6, but the precision profile requires the new optional +FP32 GDN symbols. No cuDNN or Python is needed at serving time. + +The validation scope and tolerances are in +[`src/models/cua_s1/README.md`](../../src/models/cua_s1/README.md#validation). +Error messages are worded differently, and bodies nested more than 127 levels +deep are refused. The request tests need no GPU; the kernel tests compare attention and the chunked Gated DeltaNet prefill with float64 references: ```sh cargo test -p omni-cua-s1-native +# In the pinned Python reference environment, on the same GPU: +.venv/bin/python tests/qwen3_5/test_gdn_fp32.py \ + --library target/release/libqwen3_5_cuda.so CUA_S1_CUDA_LIB=$PWD/target/release/libqwen3_5_cuda.so \ cargo test --release -p omni-qwen3-5-native --test kernels -- --ignored ``` diff --git a/src/backends/cuda/qwen3_5/README.md b/src/backends/cuda/qwen3_5/README.md index 5e230dd4..b2a710cf 100644 --- a/src/backends/cuda/qwen3_5/README.md +++ b/src/backends/cuda/qwen3_5/README.md @@ -49,3 +49,24 @@ on this opt-in path. The shared workers' default arithmetic remains unchanged. The [JEMM numerical record](../../../../docs/benchmarks/jemm-reference-20261011/README.md) contains a frozen 1–4-image corpus and reproducible operator/HTTP checks. + +## Optional Cua-S1 no-FLA text arithmetic + +The unmerged Cua-S1 text export reuses the reference norm, attention and GEMM +operations without requiring cuDNN. Norm additionally supports hidden size 2560. +The additive `cs1_gdn_fp32_*` group owns a cuBLAS handle and 32 MiB internal +workspace; the caller supplies shape-dependent FP32 buffers. It implements +64-token chunked GDN, FP32 normalization, triangular solves and recurrent state, +Torch 2.14 cumulative-decay association, and a final BF16 output cast. Each call +starts with zero state; prefix continuation is rejected by the model adapter. +The context drains its last stream before destruction, and is retired before +that stream. Rebuild the library for this optional group; ABI remains 6 on this #118-based branch; the historical cumulative validation +build used ABI 8. +Builds now link both cuBLAS and cuBLASLt. + +`tests/qwen3_5/test_gdn_fp32.py` compares against the pinned Transformers 5.17 +no-FLA implementation. On the cumulative A800 sm_80/CUDA 13.0/Torch 2.14 build, all 14 cases passed +with identical BF16 outputs, covering chunk boundaries, grouped heads, +changed-input replay/state reset, invalid dimensions/scales and context scratch +release. The fixed relative-L2 gate is 1e-4. This does not qualify other GPUs or +Torch versions; the original approximate GDN entry points remain available. diff --git a/src/backends/cuda/qwen3_5/build.sh b/src/backends/cuda/qwen3_5/build.sh index bdf323ef..b17e33a0 100755 --- a/src/backends/cuda/qwen3_5/build.sh +++ b/src/backends/cuda/qwen3_5/build.sh @@ -3,10 +3,10 @@ # # src/backends/cuda/qwen3_5/build.sh [compute capability, e.g. 89] # -# Needs nvcc (from NVCC, CUDA_HOME/bin or PATH) and cuBLASLt; the compute capability +# Needs nvcc (from NVCC, CUDA_HOME/bin or PATH) and cuBLAS/cuBLASLt; the compute capability # defaults to CUDA_COMPUTE_CAP, else 89. The library holds machine code for that # compute capability and PTX that newer GPUs can compile at load time. The CUDA -# runtime is linked statically and cuBLASLt dynamically, with an rpath to the +# runtime is linked statically and cuBLAS/cuBLASLt dynamically, with an rpath to the # toolkit's library directory when there is one next to nvcc. set -euo pipefail here=$(cd "$(dirname "$0")" && pwd) @@ -16,9 +16,9 @@ nvcc=${NVCC:-} if [ -z "$nvcc" ]; then if [ -n "${CUDA_HOME:-}" ]; then nvcc=$CUDA_HOME/bin/nvcc; else nvcc=$(command -v nvcc); fi fi -link=(-lcublasLt) +link=(-lcublasLt -lcublas) if lib=$(cd "$(dirname "$nvcc")/../lib64" 2>/dev/null && pwd); then - link=(-L"$lib" -lcublasLt -Xlinker -rpath -Xlinker "$lib") + link=(-L"$lib" -lcublasLt -lcublas -Xlinker -rpath -Xlinker "$lib") fi extra=() # Optional JEMM framework-compatible Conv3D. Legacy builds do not require cuDNN. diff --git a/src/backends/cuda/qwen3_5/gdn_fp32.cu b/src/backends/cuda/qwen3_5/gdn_fp32.cu new file mode 100644 index 00000000..ba7baa41 --- /dev/null +++ b/src/backends/cuda/qwen3_5/gdn_fp32.cu @@ -0,0 +1,352 @@ +// FP32 chunked Gated DeltaNet for Transformers' no-FLA text reference. +// Normalize/scale before dot products; solve the unit lower-triangular systems +// in FP32; keep every recurrent-state operation FP32 until the BF16 output. +#include "common.cuh" +#include "ops.h" +#include +#include +#include + +namespace { +constexpr int C = 64, D = 128; +struct Context { + cublasHandle_t blas = nullptr; + void *scratch = nullptr; + size_t bytes = 0; + cudaStream_t stream = nullptr; + int device = 0; +}; +struct Layout { + size_t q, k, kbeta, qd, kd, u, w, a, p, cum, state, vnew, temp, out, plus, + next, pa, pu, pw, total; + Layout(int tokens, int heads) { + size_t nc = (tokens + C - 1) / C, ncc = nc * C, at = 0; + auto take = [&](size_t bytes) { + size_t offset = at; + at = (at + bytes + 255) / 256 * 256; + return offset; + }; + size_t vector = (size_t)heads * ncc * D * 4, + matrix = (size_t)heads * nc * C * C * 4; + q = take(vector); + k = take(vector); + kbeta = take(vector); + qd = take(vector); + kd = take(vector); + u = take(vector); + w = take(vector); + a = take(matrix); + p = take(matrix); + cum = take((size_t)heads * ncc * 4); + state = take((size_t)heads * D * D * 4); + vnew = take((size_t)heads * C * D * 4); + temp = take((size_t)heads * C * D * 4); + out = take((size_t)heads * C * D * 4); + plus = take((size_t)heads * C * D * 4); + next = take((size_t)heads * D * D * 4); + pa = take((size_t)heads * nc * sizeof(void *)); + pu = take((size_t)heads * nc * sizeof(void *)); + pw = take((size_t)heads * nc * sizeof(void *)); + total = at; + } +}; +struct Work { + float *q, *k, *kbeta, *qd, *kd, *u, *w, *a, *p, *cum, *state, *vnew, *temp, + *out, *plus, *next; + const float **pa; + float **pu; + float **pw; +}; +Work bind(float *workspace, const Layout &l) { + auto p = [&](size_t offset) { return (float *)((char *)workspace + offset); }; + return {p(l.q), + p(l.k), + p(l.kbeta), + p(l.qd), + p(l.kd), + p(l.u), + p(l.w), + p(l.a), + p(l.p), + p(l.cum), + p(l.state), + p(l.vnew), + p(l.temp), + p(l.out), + p(l.plus), + p(l.next), + (const float **)p(l.pa), + (float **)p(l.pu), + (float **)p(l.pw)}; +} +// Association order: Torch v2.14.0 aten/native/cuda/ScanUtils.cuh, +// scan_innermost_dim (not a sequential recurrence). +// Match the reference's row-dependent Sklansky scan, including the carry +// inserted into the first element when 64 values are scanned in two blocks. +__global__ void cumulative(const float *g, float *out, int tokens, int heads, + int nc, int width) { + __shared__ float values[C]; + int b = blockIdx.x, head = b / nc, chunk = b % nc, lane = threadIdx.x; + int count = width == 16 ? 32 : 64; + float carry = 0.f; + for (int base = 0; base < C; base += count) { + for (int i = lane; i < count; i += 32) { + int token = chunk * C + base + i; + values[i] = token < tokens ? g[(size_t)token * heads + head] : 0.f; + } + __syncthreads(); + if (lane == 0) + values[0] = __fadd_rn(values[0], carry); + __syncthreads(); + for (int step = 1; step < count; step *= 2) { + if (lane < count / 2) { + int a = (lane / step) * (2 * step) + step; + int ti = a + lane % step; + values[ti] = __fadd_rn(values[ti], values[a - 1]); + } + __syncthreads(); + } + for (int i = lane; i < count; i += 32) + out[(size_t)b * C + base + i] = values[i]; + carry = values[count - 1]; + __syncthreads(); + } +} +__global__ void prepare(const cs1::bf16 *q, const cs1::bf16 *k, + const cs1::bf16 *v, const cs1::bf16 *beta, Work ws, + int tokens, int heads, int key_heads, int nc, + float scale) { + int head = blockIdx.x / (nc * C), row = blockIdx.x % (nc * C), + chunk = row / C, r = row % C, lane = threadIdx.x; + int kh = head / (heads / key_heads), token = chunk * C + r; + float qv[4], kv[4], qs[4] = {}, ks[4] = {}; +#pragma unroll + for (int j = 0; j < 4; j++) { + int d = lane * 4 + j; + qv[j] = token < tokens + ? cs1::f32(q[((size_t)token * key_heads + kh) * D + d]) + : 0.f; + kv[j] = token < tokens + ? cs1::f32(k[((size_t)token * key_heads + kh) * D + d]) + : 0.f; + qs[j] = qv[j] * qv[j]; + ks[j] = kv[j] * kv[j]; + } + float qsum = ((qs[0] + qs[1]) + qs[2]) + qs[3], + ksum = ((ks[0] + ks[1]) + ks[2]) + ks[3]; + for (int step = 16; step > 0; step /= 2) { + qsum += __shfl_down_sync(0xffffffff, qsum, step); + ksum += __shfl_down_sync(0xffffffff, ksum, step); + } + float qi = rsqrtf(__shfl_sync(0xffffffff, qsum, 0) + 1e-6f), + ki = rsqrtf(__shfl_sync(0xffffffff, ksum, 0) + 1e-6f); + float cum = ws.cum[(size_t)head * nc * C + row], + last = ws.cum[((size_t)head * nc + chunk) * C + C - 1]; + float b = token < tokens ? cs1::f32(beta[(size_t)token * heads + head]) : 0.f; + float ec = expf(cum), er = expf(last - cum); +#pragma unroll + for (int j = 0; j < 4; j++) { + int d = lane * 4 + j; + size_t index = ((size_t)head * nc * C + row) * D + d; + float qn = __fmul_rn(__fmul_rn(qv[j], qi), scale), + kn = __fmul_rn(kv[j], ki), kb = __fmul_rn(kn, b); + ws.q[index] = qn; + ws.k[index] = kn; + ws.kbeta[index] = kb; + ws.qd[index] = __fmul_rn(qn, ec); + ws.kd[index] = __fmul_rn(kn, er); + ws.u[index] = + token < tokens + ? __fmul_rn(cs1::f32(v[((size_t)token * heads + head) * D + d]), b) + : 0.f; + ws.w[index] = __fmul_rn(kb, ec); + } +} +__global__ void mask(Work ws, int batches) { + size_t index = (size_t)blockIdx.x * blockDim.x + threadIdx.x, + total = (size_t)batches * C * C; + if (index >= total) + return; + int i = (index % (C * C)) / C, j = index % C; + size_t batch = index / (C * C); + float decay = + j <= i ? expf(ws.cum[batch * C + i] - ws.cum[batch * C + j]) : 0.f; + ws.a[index] = __fmul_rn(ws.a[index], decay); + ws.p[index] = __fmul_rn(ws.p[index], decay); +} +__global__ void pointers(Work ws, int batches) { + int b = blockIdx.x * blockDim.x + threadIdx.x; + if (b >= batches) + return; + ws.pa[b] = ws.a + (size_t)b * C * C; + ws.pu[b] = ws.u + (size_t)b * C * D; + ws.pw[b] = ws.w + (size_t)b * C * D; +} +__global__ void subtract(Work ws, int heads, int nc, int chunk) { + size_t i = (size_t)blockIdx.x * blockDim.x + threadIdx.x, + total = (size_t)heads * C * D; + if (i >= total) + return; + size_t h = i / (C * D), r = i % (C * D); + ws.vnew[i] = __fsub_rn(ws.u[(h * nc + chunk) * C * D + r], ws.temp[i]); +} +__global__ void output(Work ws, cs1::bf16 *out, int tokens, int heads, + int chunk) { + size_t i = (size_t)blockIdx.x * blockDim.x + threadIdx.x, + total = (size_t)heads * C * D; + if (i >= total) + return; + int head = i / (C * D), row = (i % (C * D)) / D, d = i % D, + token = chunk * C + row; + if (token < tokens) + out[((size_t)token * heads + head) * D + d] = + cs1::to_bf16(__fadd_rn(ws.out[i], ws.plus[i])); +} +__global__ void update(Work ws, int heads, int nc, int chunk) { + size_t i = (size_t)blockIdx.x * blockDim.x + threadIdx.x, + total = (size_t)heads * D * D; + if (i >= total) + return; + size_t h = i / (D * D); + float decay = expf(ws.cum[(h * nc + chunk) * C + C - 1]); + ws.state[i] = __fadd_rn(__fmul_rn(ws.state[i], decay), ws.next[i]); +} +int status(cublasStatus_t s) { + return s == CUBLAS_STATUS_SUCCESS ? 0 : 1000 + (int)s; +} +} // namespace +extern "C" void *cs1_gdn_fp32_create(size_t bytes) { + Context *c = new (std::nothrow) Context; + if (!c) + return nullptr; + c->bytes = bytes; + if (cudaGetDevice(&c->device) != cudaSuccess || + cublasCreate(&c->blas) != CUBLAS_STATUS_SUCCESS) { + delete c; + return nullptr; + } + if (bytes && cudaMalloc(&c->scratch, bytes) != cudaSuccess) { + cublasDestroy(c->blas); + delete c; + return nullptr; + } + if (cublasSetMathMode(c->blas, CUBLAS_DEFAULT_MATH) != + CUBLAS_STATUS_SUCCESS) { + if (c->scratch) + cudaFree(c->scratch); + cublasDestroy(c->blas); + delete c; + return nullptr; + } + return c; +} +extern "C" void cs1_gdn_fp32_destroy(void *opaque) { + auto *c = (Context *)opaque; + if (!c) + return; + int previous = 0; + cudaGetDevice(&previous); + cudaSetDevice(c->device); + cudaStreamSynchronize(c->stream); + cublasDestroy(c->blas); + if (c->scratch) + cudaFree(c->scratch); + delete c; + cudaSetDevice(previous); +} +extern "C" size_t cs1_gdn_fp32_workspace_floats(int tokens, int heads) { + if (tokens <= 0 || tokens > 65536 || heads <= 0 || heads > 256) + return 0; + return Layout(tokens, heads).total / 4; +} +extern "C" int cs1_gdn_prefill_fp32(void *opaque, const void *q, const void *k, + const void *v, const float *g, + const void *beta, void *out, + float *workspace, int tokens, int heads, + int key_heads, float scale, void *stream) { + auto *c = (Context *)opaque; + if (!c || !workspace || !q || !k || !v || !g || !beta || !out || + !cs1_gdn_fp32_workspace_floats(tokens, heads) || key_heads <= 0 || + heads % key_heads != 0 || !std::isfinite(scale) || scale <= 0.f) + return cudaErrorInvalidValue; + int nc = (tokens + C - 1) / C, batches = heads * nc; + c->stream = (cudaStream_t)stream; + Work ws = bind(workspace, Layout(tokens, heads)); + int rc = status(cublasSetStream(c->blas, c->stream)); + if (rc) + return rc; + if (c->bytes) { + rc = status(cublasSetWorkspace(c->blas, c->scratch, c->bytes)); + if (rc) + return rc; + } + int log_rows = 0; + while ((1 << log_rows) < batches) + ++log_rows; + int log_width = (9 + 6 - log_rows) / 2; + int width = 1 << max(4, min(9, log_width)); + cumulative<<stream>>>(g, ws.cum, tokens, heads, nc, + width); + prepare<<stream>>>( + (const cs1::bf16 *)q, (const cs1::bf16 *)k, (const cs1::bf16 *)v, + (const cs1::bf16 *)beta, ws, tokens, heads, key_heads, nc, scale); + rc = cudaGetLastError(); + if (rc) + return rc; + const float one = 1.f, zero = 0.f; +#define BLAS(expr) \ + do { \ + rc = status(expr); \ + if (rc) \ + return rc; \ + } while (0) + BLAS(cublasSgemmStridedBatched(c->blas, CUBLAS_OP_T, CUBLAS_OP_N, C, C, D, + &one, ws.k, D, C * D, ws.kbeta, D, C * D, + &zero, ws.a, C, C * C, batches)); + BLAS(cublasSgemmStridedBatched(c->blas, CUBLAS_OP_T, CUBLAS_OP_N, C, C, D, + &one, ws.k, D, C * D, ws.q, D, C * D, &zero, + ws.p, C, C * C, batches)); + mask<<<((size_t)batches * C * C + 255) / 256, 256, 0, c->stream>>>(ws, + batches); + pointers<<<(batches + 127) / 128, 128, 0, c->stream>>>(ws, batches); + rc = cudaGetLastError(); + if (rc) + return rc; + BLAS(cublasStrsmBatched(c->blas, CUBLAS_SIDE_RIGHT, CUBLAS_FILL_MODE_UPPER, + CUBLAS_OP_N, CUBLAS_DIAG_UNIT, D, C, &one, ws.pa, C, + ws.pu, D, batches)); + BLAS(cublasStrsmBatched(c->blas, CUBLAS_SIDE_RIGHT, CUBLAS_FILL_MODE_UPPER, + CUBLAS_OP_N, CUBLAS_DIAG_UNIT, D, C, &one, ws.pa, C, + ws.pw, D, batches)); + rc = cudaMemsetAsync(ws.state, 0, (size_t)heads * D * D * 4, c->stream); + if (rc) + return rc; + for (int chunk = 0; chunk < nc; chunk++) { + size_t offset = (size_t)chunk * C * D; + long long stride = (long long)nc * C * D; + BLAS(cublasSgemmStridedBatched(c->blas, CUBLAS_OP_N, CUBLAS_OP_N, D, C, D, + &one, ws.state, D, D * D, ws.w + offset, D, + stride, &zero, ws.temp, D, C * D, heads)); + subtract<<<((size_t)heads * C * D + 255) / 256, 256, 0, c->stream>>>( + ws, heads, nc, chunk); + BLAS(cublasSgemmStridedBatched(c->blas, CUBLAS_OP_N, CUBLAS_OP_N, D, C, D, + &one, ws.state, D, D * D, ws.qd + offset, D, + stride, &zero, ws.out, D, C * D, heads)); + BLAS(cublasSgemmStridedBatched( + c->blas, CUBLAS_OP_N, CUBLAS_OP_N, D, C, C, &one, ws.vnew, D, C * D, + ws.p + (size_t)chunk * C * C, C, (long long)nc * C * C, &zero, ws.plus, + D, C * D, heads)); + output<<<((size_t)heads * C * D + 255) / 256, 256, 0, c->stream>>>( + ws, (cs1::bf16 *)out, tokens, heads, chunk); + BLAS(cublasSgemmStridedBatched(c->blas, CUBLAS_OP_N, CUBLAS_OP_T, D, D, C, + &one, ws.vnew, D, C * D, ws.kd + offset, D, + stride, &zero, ws.next, D, D * D, heads)); + update<<<((size_t)heads * D * D + 255) / 256, 256, 0, c->stream>>>( + ws, heads, nc, chunk); + rc = cudaGetLastError(); + if (rc) + return rc; + } +#undef BLAS + return cudaSuccess; +} diff --git a/src/backends/cuda/qwen3_5/ops.h b/src/backends/cuda/qwen3_5/ops.h index 487aab3a..04c9e655 100644 --- a/src/backends/cuda/qwen3_5/ops.h +++ b/src/backends/cuda/qwen3_5/ops.h @@ -72,6 +72,14 @@ int cs1_gdn_gates(const void* b, const void* a, int ld, const void* A_log, const // Chunked gated delta rule, q and k L2-normalized inside, q scaled by `scale`. // q, k [T, HK, 128], v [T, H, 128], g float [T, H], beta [T, H], o [T, H, 128]. size_t cs1_gdn_workspace_floats(int T, int H); +// Optional no-FLA text precision mode: FP32 normalized Q/K and recurrent state, +// BF16 output. Context and scratch remain model-owned; prefix continuation is unsupported. +void* cs1_gdn_fp32_create(size_t workspace_bytes); +void cs1_gdn_fp32_destroy(void* context); +size_t cs1_gdn_fp32_workspace_floats(int T,int H); +int cs1_gdn_prefill_fp32(void* context,const void* q, const void* k, const void* v, const float* g, + const void* beta, void* o, float* workspace, + int T, int H, int HK, float scale, void* stream); int cs1_gdn_prefill(const void* q, const void* k, const void* v, const float* g, const void* beta, void* o, float* workspace, int T, int H, int HK, float scale, void* stream); diff --git a/src/backends/cuda/qwen3_5/qwen3_5.backend.json b/src/backends/cuda/qwen3_5/qwen3_5.backend.json index 66ed2bce..40f3ef37 100644 --- a/src/backends/cuda/qwen3_5/qwen3_5.backend.json +++ b/src/backends/cuda/qwen3_5/qwen3_5.backend.json @@ -10,6 +10,7 @@ "elementwise.cu", "attention.cu", "gdn_prefill.cu", + "gdn_fp32.cu", "gemm.cu", "runtime.cu", "vision.cu", diff --git a/src/backends/cuda/qwen3_5/reference_norm.cu b/src/backends/cuda/qwen3_5/reference_norm.cu index 61f7cdeb..c4f13891 100644 --- a/src/backends/cuda/qwen3_5/reference_norm.cu +++ b/src/backends/cuda/qwen3_5/reference_norm.cu @@ -25,10 +25,10 @@ extern "C" int reference_norm_run(const void* x,const void* delta,const void* w, } extern "C" int cs1_reference_rms_norm(const void* x,const void* w,void* out,int rows,int d,float eps,void* stream) { - if(rows<=0 || d!=5120) return cudaErrorInvalidValue; + if(rows<=0 || (d!=5120 && d!=2560)) return cudaErrorInvalidValue; return reference_norm_run(x,nullptr,w,out,rows,d,eps,stream); } extern "C" int cs1_reference_add_rms_norm(void* x,const void* delta,const void* w,void* out,int rows,int d,float eps,void* stream) { - if(rows<=0 || d!=5120) return cudaErrorInvalidValue; + if(rows<=0 || (d!=5120 && d!=2560)) return cudaErrorInvalidValue; return reference_norm_run(x,delta,w,out,rows,d,eps,stream); } diff --git a/src/models/cua_s1/README.md b/src/models/cua_s1/README.md index 36530206..c24f3ee0 100644 --- a/src/models/cua_s1/README.md +++ b/src/models/cua_s1/README.md @@ -10,7 +10,8 @@ The native worker follows the [architecture contracts](../../../docs/architectur with independent [processing](native/src/processing.rs) and [executor](native/src/executor.rs) modules. Preparation returns token IDs and an option count per question, plus a response context for identity/order and usage. -The executor returns FP32 letter logits; response finishing owns per-question +The executor projects the letter rows, rounds to the BF16 `lm_head` output dtype, +then returns FP32 letter logits; response finishing owns per-question softmax and the choice/confidence interpretation. The engine owns a [shared serial scheduler](../../runtime/README.md) that admits each question's forward before blocking dispatch. Execution remains one prompt per question @@ -117,6 +118,27 @@ The bfloat16 worker's own difference from the fp32 worker is reported next to ea **Performance.** Performance results will state the hardware, revisions, commands and raw results. Load and warmup are reported separately from warm latency, and warm latency is broken down by prompt length and option count. +## Native text precision profile + +The unmerged text export retains the original 256 FP32 LoRA tensors (rank 16, +scale 2). It selects the shared executor's framework-compatible BF16 norm, +GEMM and attention operations, and a model-owned FP32 chunked GDN context. +Q/K normalization, triangular solves and recurrent state stay FP32 until the +GDN output is cast to BF16. Cumulative decay follows Torch 2.14's scan association +order. State resets on each prompt; prefix continuation is rejected. Context, +workspace and streams retire after outstanding work and captures complete. +The BF16 letter-logit boundary is applied before response softmax. + +This profile targets the pinned no-FLA reference described above. Installing +FLA changes Transformers' GDN implementation; comparisons must record which +implementation actually ran. A different Torch version or GPU may choose other +reduction orders, so qualification remains tied to the recorded environment. +The old merged export remains an approximate execution mode. The precision +profile does not change the screenshot adapter's execution mode. + +The historical cumulative A800 investigation and frozen-corpus results are recorded in +[the numerical report](../../../docs/benchmarks/cua-s1-text-parity-20261013/README.md). + ## Not covered yet - Native execution of the `multimodal` adapter: image preprocessing, the vision tower and the vision LoRA. This is tracked in [#10](https://github.com/ThinkFlowLab/system1-omni/issues/10). diff --git a/src/models/cua_s1/native/src/engine.rs b/src/models/cua_s1/native/src/engine.rs index 21e2fd18..5c0b434e 100644 --- a/src/models/cua_s1/native/src/engine.rs +++ b/src/models/cua_s1/native/src/engine.rs @@ -2,7 +2,7 @@ use std::path::Path; -use anyhow::{Context, Result, ensure}; +use anyhow::{Context, Result}; use omni_runtime::SerialScheduler; use crate::contract::LETTERS; @@ -17,12 +17,7 @@ pub struct Engine { impl Engine { pub async fn load(dir: &Path, library: &Path) -> Result { - // Written by export_text_merged.py; without it `dir` may hold the base model alone. - ensure!( - dir.join("cua_s1_export.json").exists(), - "{} is not a merged text checkpoint; see recipe/cua_s1/native.md", - dir.display() - ); + let export = crate::provenance::text_export(dir)?; let processor = Processor::load(dir)?; let ids = LETTERS .chars() @@ -33,7 +28,7 @@ impl Engine { .context("letter token") }) .collect::>>()?; - let executor = Executor::load(dir, library, &ids).await?; + let executor = Executor::load(dir, library, &ids, export).await?; Ok(Self { processor, scheduler: SerialScheduler::default(), @@ -41,3 +36,7 @@ impl Engine { }) } } + +#[cfg(test)] +#[path = "../../../../../tests/cua_s1/text_export.rs"] +mod tests; diff --git a/src/models/cua_s1/native/src/executor.rs b/src/models/cua_s1/native/src/executor.rs index 3db4c425..6bcbdba2 100644 --- a/src/models/cua_s1/native/src/executor.rs +++ b/src/models/cua_s1/native/src/executor.rs @@ -8,6 +8,7 @@ use omni_runtime::SerialScheduler; use serde_json::Value as Json; use crate::model::Model; +use crate::provenance::TextExport; /// Tied with the output projection in Qwen3.5-4B. const EMBEDDING: &str = "model.language_model.embed_tokens.weight"; @@ -24,9 +25,23 @@ pub struct Executor { } impl Executor { - pub async fn load(dir: &Path, library: &Path, letter_ids: &[u32]) -> Result { + pub(crate) async fn load( + dir: &Path, + library: &Path, + letter_ids: &[u32], + export: TextExport, + ) -> Result { let (d, lib) = (dir.to_path_buf(), library.to_path_buf()); - let model = tokio::task::spawn_blocking(move || Model::load(&d, &lib)).await??; + let model = tokio::task::spawn_blocking(move || match export { + TextExport::Merged => Model::load(&d, &lib), + TextExport::Unmerged => { + let mut model = + Model::load_with_lora(&d, &lib, &d.join("adapter.safetensors"), 2.0)?; + model.enable_reference_text_numerics()?; + Ok(model) + } + }) + .await??; let letters = letter_rows(dir, letter_ids, model.cfg.hidden)?; Ok(Self { model: Arc::new(Mutex::new(model)), @@ -63,10 +78,13 @@ fn letter_logits(letters: &[f32], last: &[f32], n_options: usize) -> Result() as f32 + .sum::() as f32; + // The tied lm_head emits BF16 before reference option selection/softmax. + half::bf16::from_f32(dot).to_f32() }) .collect(); // Abort before submitting a later prompt, as the former per-question softmax did. diff --git a/src/models/cua_s1/native/src/provenance.rs b/src/models/cua_s1/native/src/provenance.rs index 4ea295bb..352b3ab2 100644 --- a/src/models/cua_s1/native/src/provenance.rs +++ b/src/models/cua_s1/native/src/provenance.rs @@ -1,4 +1,4 @@ -//! Hash checks before assigning the pinned multimodal model identity. +//! Validate text export policy and multimodal artifact identity before CUDA loading. use anyhow::{Context, Result, ensure}; use serde_json::Value; use sha2::{Digest, Sha256}; @@ -9,6 +9,35 @@ use std::{ }; const LOCK_SHA: &str = "9820bd232c5762f114e19680c0f8203d7e1faaf8a60c196cfe01964d6d8a6c09"; +#[derive(Clone, Copy)] +pub(crate) enum TextExport { + Merged, + Unmerged, +} + +pub(crate) fn text_export(dir: &Path) -> Result { + let marker: Value = serde_json::from_slice(&fs::read(dir.join("cua_s1_export.json"))?) + .context("invalid text export marker")?; + match marker["format"].as_str() { + Some("cua-s1-text-merged/1") => Ok(TextExport::Merged), + Some("cua-s1-text-unmerged/1") => { + ensure!( + marker["lora_scale"].as_f64() == Some(2.0), + "Cua-S1 text LoRA scale must be 2" + ); + let expected = marker["adapter_sha256"] + .as_str() + .context("missing adapter checksum")?; + ensure!( + hash_file(&dir.join("adapter.safetensors"))? == expected, + "adapter checksum mismatch" + ); + Ok(TextExport::Unmerged) + } + _ => anyhow::bail!("unsupported text export; see recipe/cua_s1/native.md"), + } +} + fn hash_file(path: &Path) -> Result { let mut file = File::open(path)?; let mut hash = Sha256::new(); diff --git a/src/models/qwen3_5/native/src/cuda.rs b/src/models/qwen3_5/native/src/cuda.rs index 277f565e..73ab545b 100644 --- a/src/models/qwen3_5/native/src/cuda.rs +++ b/src/models/qwen3_5/native/src/cuda.rs @@ -81,6 +81,10 @@ macro_rules! api { } impl Api { + /// Language arithmetic does not require cuDNN patch convolution. + pub fn language_reference(&self) -> Result<&Reference> { + self.reference.as_ref().context("missing framework language CUDA symbols; rebuild the library") + } pub fn reference(&self) -> Result<&Reference> { let reference=self.reference.as_ref().context("JEMM requires the optional reference CUDA symbols; rebuild libqwen3_5_cuda.so")?; // SAFETY: this capability query takes no arguments. @@ -196,6 +200,112 @@ api! { static API: OnceLock = OnceLock::new(); +type Fp32GdnRun = unsafe extern "C" fn( + *mut c_void, + *const c_void, + *const c_void, + *const c_void, + *const f32, + *const c_void, + *mut c_void, + *mut f32, + c_int, + c_int, + c_int, + f32, + Stream, +) -> c_int; +/// Model-owned FP32 chunk solver. Destruction drains its last producer stream. +pub struct Fp32Gdn { + handle: *mut c_void, + run: Fp32GdnRun, + destroy: unsafe extern "C" fn(*mut c_void), + size: unsafe extern "C" fn(c_int, c_int) -> usize, +} +impl Fp32Gdn { + pub fn workspace_floats(&self, tokens: usize, heads: usize) -> Result { + ensure!(tokens <= 65536 && heads <= 256, "FP32 GDN shape limit"); + // SAFETY: this getter does not dereference memory. + let size = unsafe { (self.size)(tokens as c_int, heads as c_int) }; + ensure!(size > 0, "cannot size FP32 GDN workspace"); + Ok(size) + } + /// # Safety + /// All pointers hold the documented complete tensors; workspace has at least + /// workspace_floats(tokens,heads) entries. Serialize this context and retain + /// every allocation until stream completion. + #[allow(clippy::too_many_arguments)] // Exact CUDA operation boundary. + pub unsafe fn run( + &self, + q: *const c_void, + k: *const c_void, + v: *const c_void, + g: *const f32, + beta: *const c_void, + out: *mut c_void, + workspace: *mut f32, + tokens: c_int, + heads: c_int, + key_heads: c_int, + scale: f32, + stream: Stream, + ) -> c_int { + unsafe { + (self.run)( + self.handle, + q, + k, + v, + g, + beta, + out, + workspace, + tokens, + heads, + key_heads, + scale, + stream, + ) + } + } +} +impl Drop for Fp32Gdn { + fn drop(&mut self) { + // SAFETY: this object exclusively owns the context; the backend drains before freeing it. + unsafe { (self.destroy)(self.handle) }; + } +} +impl Api { + pub fn fp32_gdn(&self) -> Result { + // SAFETY: these optional signatures match ops.h; the library lives in static API. + let (create, destroy, run, size) = unsafe { + ( + *self + ._lib + .get:: *mut c_void>(b"cs1_gdn_fp32_create\0")?, + *self + ._lib + .get::(b"cs1_gdn_fp32_destroy\0")?, + *self._lib.get::(b"cs1_gdn_prefill_fp32\0")?, + *self + ._lib + .get:: usize>( + b"cs1_gdn_fp32_workspace_floats\0", + )?, + ) + }; + // SAFETY: creation owns an independent cuBLAS handle and workspace. + let handle = unsafe { create(32 << 20) }; + ensure!(!handle.is_null(), "FP32 GDN cuBLAS initialization failed"); + Ok(Fp32Gdn { + handle, + run, + destroy, + size, + }) + } +} + /// The library next to the running executable. pub fn default_library() -> Result { let exe = std::env::current_exe()?; diff --git a/src/models/qwen3_5/native/src/lora.rs b/src/models/qwen3_5/native/src/lora.rs index cb838a49..aa390d2d 100644 --- a/src/models/qwen3_5/native/src/lora.rs +++ b/src/models/qwen3_5/native/src/lora.rs @@ -187,6 +187,7 @@ fn parse(cfg: &Config, bytes: &[u8], scale: f32) -> Result { } let native = name .strip_prefix(PREFIX) + .or_else(|| name.strip_prefix("base_model.model.model.")) .context("unsupported LoRA tensor prefix")?; let (target, letter) = if let Some(target) = native.strip_suffix(".lora_A.weight") { (target, 0) diff --git a/src/models/qwen3_5/native/src/model.rs b/src/models/qwen3_5/native/src/model.rs index 1ad7e5aa..eb6f0220 100644 --- a/src/models/qwen3_5/native/src/model.rs +++ b/src/models/qwen3_5/native/src/model.rs @@ -450,6 +450,7 @@ impl Scratch { stream: Stream, lora: Option<&lora::Lora>, reference: Option<&cuda::Reference>, + fp32_gdn: Option<&cuda::Fp32Gdn>, ) -> Result { let (h, kd, vd, hv) = (cfg.hidden, cfg.key_dim(), cfg.value_dim(), cfg.lin_v_heads); let (hq, hk, hd) = (cfg.heads, cfg.kv_heads, cfg.head_dim); @@ -473,6 +474,11 @@ impl Scratch { } else { unsafe { (cuda::api().cs1_gdn_workspace_floats)(cap as i32, hv as i32) } }; + let ws_floats = if let Some(gdn) = fp32_gdn { + ws_floats.max(gdn.workspace_floats(cap, hv)?) + } else { + ws_floats + }; let offsets = [ take(cap * 4), take(cap * h * BF16), @@ -620,6 +626,7 @@ pub struct Model { stream: Stream, gemm: *mut c_void, reference: Option<&'static cuda::Reference>, + fp32_gdn: Option, /// Buffers for the largest packed token count so far; grows as needed. scratch: Option, /// Opt-in replay with at most 64 captures per mode, keyed by sequence lengths. @@ -653,6 +660,8 @@ impl Drop for Model { if self.lora.is_some() { self.scratch = None; } + // Retire the FP32 cuBLAS context while its last stream is still valid. + self.fp32_gdn = None; // SAFETY: created by cs1_gemm_create and not destroyed before. unsafe { (cuda::api().cs1_gemm_destroy)(self.gemm) }; } @@ -761,6 +770,7 @@ impl Model { stream, gemm, reference: None, + fp32_gdn: None, scratch: None, graph_enabled: std::env::var("CUA_S1_GRAPH").as_deref() == Ok("1"), graphs: VecDeque::new(), @@ -770,9 +780,37 @@ impl Model { Ok(model) } + /// Cua-S1's unmerged text path: framework norm/GEMM/attention plus FP32 GDN. + pub fn enable_reference_text_numerics(&mut self) -> Result<()> { + ensure!( + self.cfg.hidden == 2560 && self.lora.is_some(), + "reference text numerics require an unmerged 4B language model" + ); + ensure!( + self.scratch.is_none() && self.reference.is_none() && self.fp32_gdn.is_none(), + "select reference text numerics before the first forward" + ); + cuda::set_device(0)?; + let gdn = cuda::api().fp32_gdn()?; + let reference = cuda::api().language_reference()?; + cuda::synchronize(self.stream)?; + // SAFETY: both handles are model-owned; no queued work or captures exist. + let gemm = unsafe { (reference.gemm_create)(GEMM_WORKSPACE) }; + ensure!(!gemm.is_null(), "reference text cuBLASLt setup failed"); + unsafe { (cuda::api().cs1_gemm_destroy)(self.gemm) }; + self.gemm = gemm; + self.reference = Some(reference); + self.fp32_gdn = Some(gdn); + Ok(()) + } + /// Select JEMM's pinned framework arithmetic before the first forward. /// Prefix caching is outside this execution contract. pub fn enable_reference_numerics(&mut self) -> Result<()> { + ensure!( + self.fp32_gdn.is_none(), + "cannot combine JEMM reference numerics with FP32 GDN" + ); ensure!( self.cfg.hidden == 5120 && self.cfg.head_dim == 256 && self.lora.is_some(), "reference numerics require the unmerged JEMM 27B model" @@ -924,6 +962,7 @@ impl Model { self.stream, self.lora.as_ref(), self.reference, + self.fp32_gdn.as_ref(), )?); } Ok(()) @@ -1122,6 +1161,10 @@ impl Model { /// Allocate an uninitialized prefix of `len` tokens (a multiple of 64). /// Capture must finish successfully on this model before continuation. pub fn alloc_prefix(&self, len: usize) -> Result { + ensure!( + self.fp32_gdn.is_none(), + "FP32 GDN does not support prefix caching" + ); ensure!( self.reference.is_none(), "JEMM reference numerics do not support prefix caching" @@ -1383,22 +1426,39 @@ impl Model { )?; for &(offset, length) in &sequences { check( - (self - .reference - .map_or(cuda::api().cs1_gdn_prefill, |r| r.gdn_prefill))( - p(s.lq + offset * kd * BF16), - p(s.lk + offset * kd * BF16), - p(s.lv + offset * vd * BF16), - p(s.g + offset * hv * F32).cast(), - p(s.beta + offset * hv * BF16), - p(s.lo + offset * vd * BF16), - p(s.workspace).cast(), - length, - hv as i32, - cfg.lin_k_heads as i32, - (cfg.lin_k_dim as f32).powf(-0.5), - st, - ), + if let Some(gdn) = &self.fp32_gdn { + gdn.run( + p(s.lq + offset * kd * BF16), + p(s.lk + offset * kd * BF16), + p(s.lv + offset * vd * BF16), + p(s.g + offset * hv * F32).cast(), + p(s.beta + offset * hv * BF16), + p(s.lo + offset * vd * BF16), + p(s.workspace).cast(), + length, + hv as i32, + cfg.lin_k_heads as i32, + (cfg.lin_k_dim as f32).powf(-0.5), + st, + ) + } else { + (self + .reference + .map_or(cuda::api().cs1_gdn_prefill, |r| r.gdn_prefill))( + p(s.lq + offset * kd * BF16), + p(s.lk + offset * kd * BF16), + p(s.lv + offset * vd * BF16), + p(s.g + offset * hv * F32).cast(), + p(s.beta + offset * hv * BF16), + p(s.lo + offset * vd * BF16), + p(s.workspace).cast(), + length, + hv as i32, + cfg.lin_k_heads as i32, + (cfg.lin_k_dim as f32).powf(-0.5), + st, + ) + }, "gdn prefill", )?; } @@ -1581,6 +1641,10 @@ impl Model { capture: Option<&mut PrefixState>, prefix: Option<&PrefixState>, ) -> Result<()> { + ensure!( + self.fp32_gdn.is_none(), + "FP32 GDN does not support prefix continuation" + ); ensure!( qb <= tend && (qb == 0) == prefix.is_none() && (qb == 0 || capture.is_none()), "prefix window shape" diff --git a/tests/cua_s1/head.rs b/tests/cua_s1/head.rs index 19dac505..37e03680 100644 --- a/tests/cua_s1/head.rs +++ b/tests/cua_s1/head.rs @@ -1,7 +1,21 @@ use super::*; #[test] -fn letter_projection_preserves_fp64_accumulation_and_fp32_readout() { +fn letter_projection_matches_the_reference_bf16_output_boundary() { + assert_eq!( + letter_logits(&[1.0078125], &[1.0078125], 1).unwrap(), + [1.015625] + ); + // Both exact dot products belong to the same BF16 bin. Retaining FP32 + // invents a winning second option where the reference has an ordered tie. + assert_eq!( + letter_logits(&[20.0, 0.03125, 20.0, 0.046875], &[1.0, 1.0], 2).unwrap(), + [20.0, 20.0] + ); +} + +#[test] +fn letter_projection_preserves_fp64_accumulation_and_bf16_readout() { let letters = [ 16_777_216.0, 1.0, diff --git a/tests/cua_s1/text_export.rs b/tests/cua_s1/text_export.rs new file mode 100644 index 00000000..de813edb --- /dev/null +++ b/tests/cua_s1/text_export.rs @@ -0,0 +1,62 @@ +use super::*; +use serde_json::json; +use std::{ + fs, + path::PathBuf, + time::{SystemTime, UNIX_EPOCH}, +}; + +struct Directory(PathBuf); +impl Directory { + fn new(marker: serde_json::Value) -> Self { + let nonce = SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap() + .as_nanos(); + let path = + std::env::temp_dir().join(format!("cua-text-export-{}-{nonce}", std::process::id())); + fs::create_dir(&path).unwrap(); + fs::write(path.join("cua_s1_export.json"), marker.to_string()).unwrap(); + Self(path) + } +} +impl Drop for Directory { + fn drop(&mut self) { + fs::remove_dir_all(&self.0).unwrap(); + } +} + +#[tokio::test] +async fn unknown_export_is_rejected_before_tokenizer_or_cuda_loading() { + let dir = Directory::new(json!({"format":"unknown"})); + let error = Engine::load(&dir.0, Path::new("unused-cuda-library")) + .await + .err() + .expect("must reject unknown export"); + assert!( + error.to_string().contains("unsupported text export"), + "{error}" + ); +} + +#[tokio::test] +async fn reference_export_rejects_incorrect_adapter_scale_before_loading() { + let dir = Directory::new(json!({"format":"cua-s1-text-unmerged/1", "lora_scale":2.1})); + let error = Engine::load(&dir.0, Path::new("unused-cuda-library")) + .await + .err() + .expect("must reject incorrect scale"); + assert!(error.to_string().contains("LoRA scale"), "{error}"); +} + +#[tokio::test] +async fn reference_export_rejects_changed_adapter_bytes_before_cuda_loading() { + let dir = Directory::new(json!({"format":"cua-s1-text-unmerged/1", "lora_scale":2.0, + "adapter_sha256":"0".repeat(64)})); + fs::write(dir.0.join("adapter.safetensors"), b"changed checkpoint").unwrap(); + let error = Engine::load(&dir.0, Path::new("unused-cuda-library")) + .await + .err() + .expect("must reject changed adapter"); + assert!(error.to_string().contains("adapter checksum"), "{error}"); +} diff --git a/tests/qwen3_5/lora.rs b/tests/qwen3_5/lora.rs index f32b78c7..55b7496d 100644 --- a/tests/qwen3_5/lora.rs +++ b/tests/qwen3_5/lora.rs @@ -52,6 +52,29 @@ fn pair(target: &str, rows: usize, cols: usize, rank: usize) -> Vec { ]) } +#[test] +fn text_adapter_prefix_preserves_the_same_fp32_projection() { + let cfg = config(); + let bytes = tensors(&[ + ( + "base_model.model.model.layers.0.mlp.down_proj.lora_A.weight", + vec![16, cfg.intermediate], + Dtype::F32, + 0.125, + ), + ( + "base_model.model.model.layers.0.mlp.down_proj.lora_B.weight", + vec![cfg.hidden, 16], + Dtype::F32, + -0.25, + ), + ]); + let parsed = parse(&cfg, &bytes, 2.0).unwrap(); + assert_eq!(parsed.pairs.len(), 1); + assert_eq!(parsed.pairs[0].target.group, Group::Down); + assert_eq!(parsed.pairs[0].rank, 16); +} + #[test] fn layouts_cover_both_mixers_and_the_small_gate_projections() { for (hidden, intermediate, heads, values) in [(5120, 17408, 24, 48), (2560, 9216, 16, 32)] { diff --git a/tests/qwen3_5/test_gdn_fp32.py b/tests/qwen3_5/test_gdn_fp32.py new file mode 100644 index 00000000..fb235409 --- /dev/null +++ b/tests/qwen3_5/test_gdn_fp32.py @@ -0,0 +1,129 @@ +"""GPU regression against the pinned Transformers 5.17 no-FLA implementation. + +Run with Python 3.12, torch 2.14.0+cu130, transformers 5.17.0: + python tests/qwen3_5/test_gdn_fp32.py --library target/release/libqwen3_5_cuda.so +The relative-L2 gate is fixed at 1e-4; repeatability/state isolation is exact. +""" + +import argparse +import ctypes +import inspect +import json +import math + +import torch +import transformers +from transformers.models.qwen3_5.modeling_qwen3_5 import torch_chunk_gated_delta_rule + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--library", required=True) + args = parser.parse_args() + assert torch.__version__.split("+")[0] == "2.14.0", torch.__version__ + assert transformers.__version__ == "5.17.0", transformers.__version__ + torch.backends.cuda.matmul.allow_tf32 = False + torch.manual_seed(141) + lib = ctypes.CDLL(args.library) + create = lib.cs1_gdn_fp32_create + create.argtypes, create.restype = [ctypes.c_size_t], ctypes.c_void_p + destroy = lib.cs1_gdn_fp32_destroy + destroy.argtypes, destroy.restype = [ctypes.c_void_p], None + size = lib.cs1_gdn_fp32_workspace_floats + size.argtypes, size.restype = [ctypes.c_int, ctypes.c_int], ctypes.c_size_t + run = lib.cs1_gdn_prefill_fp32 + run.argtypes = ( + [ctypes.c_void_p] * 8 + [ctypes.c_int] * 3 + [ctypes.c_float, ctypes.c_void_p] + ) + run.restype = ctypes.c_int + # Initialize cuBLAS before checking repeated context ownership. + warm = create(32 << 20) + assert warm + destroy(warm) + torch.cuda.synchronize() + free_before = torch.cuda.mem_get_info()[0] + for _ in range(3): + context = create(32 << 20) + assert context + destroy(context) + torch.cuda.synchronize() + assert torch.cuda.mem_get_info()[0] == free_before, "context scratch leaked" + for tokens, heads in [(0, 32), (-1, 32), (65537, 32), (1, 0), (1, 257)]: + assert size(tokens, heads) == 0 + reference = inspect.unwrap(torch_chunk_gated_delta_rule) + context = create(32 << 20) + assert context + try: + for tokens, heads, key_heads in [ + (t, 32, 16) + for t in [1, 31, 32, 33, 63, 64, 65, 127, 128, 129, 233, 513, 2049] + ] + [(129, 4, 2)]: + q = torch.randn( + 1, tokens, key_heads, 128, device="cuda", dtype=torch.bfloat16 + ) + k = torch.randn_like(q) + v = torch.randn(1, tokens, heads, 128, device="cuda", dtype=torch.bfloat16) + g = -torch.rand(1, tokens, heads, device="cuda") * 0.25 + beta = torch.rand(1, tokens, heads, device="cuda", dtype=torch.bfloat16) + work = torch.empty(size(tokens, heads), device="cuda", dtype=torch.float32) + out = torch.empty_like(v) + stream = torch.cuda.current_stream().cuda_stream + pointers = [x.data_ptr() for x in (q, k, v, g, beta, out, work)] + call = [ + context, + *pointers, + tokens, + heads, + key_heads, + 1 / math.sqrt(128), + stream, + ] + expected, _ = reference( + q.repeat_interleave(heads // key_heads, dim=2), + k.repeat_interleave(heads // key_heads, dim=2), + v, + g, + beta, + use_qk_l2norm_in_kernel=True, + ) + assert run(*call) == 0 + torch.cuda.synchronize() + saved = out.clone() + relative = ( + torch.linalg.vector_norm(out.float() - expected.float()) + / torch.linalg.vector_norm(expected.float()) + ).item() + assert relative <= 1e-4, (tokens, heads, relative) + # Poison scratch and run unrelated data before replaying the original input. + work.fill_(float("nan")) + changed = torch.randn_like(v) + changed_call = call.copy() + changed_call[3] = changed.data_ptr() + assert run(*changed_call) == 0 + assert run(*call) == 0 + torch.cuda.synchronize() + assert torch.equal(out, saved), "previous request leaked recurrent state" + # Invalid shape/scale must fail before a launch or device dereference. + for index, invalid in [(8, 0), (10, 3), (11, float("nan")), (11, 0.0)]: + invalid_call = call.copy() + invalid_call[index] = invalid + assert run(*invalid_call) != 0 + print( + json.dumps( + { + "tokens": tokens, + "heads": heads, + "key_heads": key_heads, + "relative_l2": relative, + "bf16_mismatches": int((saved != expected).sum().item()), + "state_reset": True, + } + ), + flush=True, + ) + finally: + destroy(context) + + +if __name__ == "__main__": + main() From 27f77e49868d84b091aa804f02ccb4d3474a9a00 Mon Sep 17 00:00:00 2001 From: levius <2114377220@qq.com> Date: Wed, 14 Oct 2026 09:05:25 +0800 Subject: [PATCH 15/21] docs(cua-s1): track independent-head qualification separately --- .../cua-s1-text-parity-20261013/README.md | 14 ++++++++------ recipe/cua_s1/native.md | 2 +- 2 files changed, 9 insertions(+), 7 deletions(-) diff --git a/docs/benchmarks/cua-s1-text-parity-20261013/README.md b/docs/benchmarks/cua-s1-text-parity-20261013/README.md index 9baf163d..dd485921 100644 --- a/docs/benchmarks/cua-s1-text-parity-20261013/README.md +++ b/docs/benchmarks/cua-s1-text-parity-20261013/README.md @@ -23,9 +23,11 @@ The GPU results below were measured on cumulative candidate The independently rebased branch retains #118's ABI 6 and adds only optional FP32 GDN symbols. Rebuild the library with the worker; no cuDNN is needed for this text path. Results from the cumulative build are historical evidence for -the pinned implementation, not a fresh GPU qualification of the rebased PR head. -The GPU server was shut down after the campaign; no new device run is implied -by packaging this PR. CPU/build/docs checks are run again on the independent head. +the pinned implementation. Fresh independent-branch qualification, including +exact source/library/binary hashes, repeated HTTP results and self-review, is +recorded separately in [PR #143](https://github.com/ThinkFlowLab/system1-omni/pull/143). +Use each run's source pin; do not substitute the historical integrated result +for a newer branch's validation. ## Frozen inputs and environment @@ -122,7 +124,7 @@ operator regressions and strict MkDocs build also passed. The checkpoint-depende prefix test used canonical merged text weights with the original pinned multimodal config for a zero-image contract fixture; it is not numerical-reference evidence. -Independent-PR-head GPU execution, other GPUs, framework versions, FLA execution -and screenshot-model numerical -parity require separate qualification. This corpus does not establish general +Other GPUs, framework versions, FLA execution and screenshot-model numerical +parity require separate qualification. Independent-head evidence remains +source-pinned in #143 rather than inferred from these historical results. This corpus does not establish general GUI accuracy or a performance improvement. diff --git a/recipe/cua_s1/native.md b/recipe/cua_s1/native.md index 2a23a35f..1da0660e 100644 --- a/recipe/cua_s1/native.md +++ b/recipe/cua_s1/native.md @@ -1,6 +1,6 @@ # Cua-S1 4B 0.2 native text worker -The native worker ([`src/models/cua_s1/native/`](../../src/models/cua_s1/native/)) serves the `text` adapter like the reference worker in [`text.md`](text.md), with the Qwen3.5-4B forward pass on the CUDA kernels in [`src/backends/cuda/qwen3_5/`](../../src/backends/cuda/qwen3_5/) and no Python or PyTorch. It needs an NVIDIA GPU with compute capability 8.0 or newer; the unmerged precision profile was checked on the cumulative A800 80 GB (sm_80) build with CUDA 13.0; device qualification of this independent #118-based branch remains pending. The earlier merged path was checked on an RTX 6000 Ada (sm_89) with CUDA 13.2. +The native worker ([`src/models/cua_s1/native/`](../../src/models/cua_s1/native/)) serves the `text` adapter like the reference worker in [`text.md`](text.md), with the Qwen3.5-4B forward pass on the CUDA kernels in [`src/backends/cuda/qwen3_5/`](../../src/backends/cuda/qwen3_5/) and no Python or PyTorch. It needs an NVIDIA GPU with compute capability 8.0 or newer; the unmerged precision profile was checked on the cumulative A800 80 GB (sm_80) build with CUDA 13.0; source-pinned qualification of the independent #118-based branch is tracked in [PR #143](https://github.com/ThinkFlowLab/system1-omni/pull/143). The earlier merged path was checked on an RTX 6000 Ada (sm_89) with CUDA 13.2. Run the commands from the repository root. Pass your GPU's compute capability to `build.sh` (89 for Ada, 80 for A100, 90 for H100); the worker finds `libqwen3_5_cuda.so` next to its executable, or at `CUA_S1_CUDA_LIB`: From 47fe6b2c6ff34ead2b12d0469d4b382a1e3061fa Mon Sep 17 00:00:00 2001 From: levius <2114377220@qq.com> Date: Wed, 14 Oct 2026 11:11:48 +0800 Subject: [PATCH 16/21] docs(cua-s1): clarify merged readout and cuBLAS upgrade impact --- recipe/cua_s1/native.md | 6 +++++- src/backends/cuda/qwen3_5/README.md | 5 ++++- src/models/cua_s1/README.md | 6 +++++- 3 files changed, 14 insertions(+), 3 deletions(-) diff --git a/recipe/cua_s1/native.md b/recipe/cua_s1/native.md index 1da0660e..959a6755 100644 --- a/recipe/cua_s1/native.md +++ b/recipe/cua_s1/native.md @@ -41,7 +41,11 @@ markers or altered adapter bytes fail before CUDA loading. Prefix continuation is unsupported in this precision mode. Existing `cua-s1-text-merged/1` exports remain loadable with the earlier merged -execution path. Merging the adapter, TF32/BF16 GDN intermediates, and different +backbone execution path. PR #143 changes their letter-logit readout too: it now +rounds to BF16 instead of retaining FP32, so close or tied choices can change +after upgrading. No merged-path corpus A/B run qualifies that behavior change; +the A800 parity results cover the unmerged precision profile only. +Merging the adapter, TF32/BF16 GDN intermediates, and different GEMM reduction policies can change decisions on close or tied options. Re-export with the command above to use the precision profile. Rebuild worker and library together: this branch retains #118's ABI 6, but the precision profile requires the new optional diff --git a/src/backends/cuda/qwen3_5/README.md b/src/backends/cuda/qwen3_5/README.md index b2a710cf..d625faf6 100644 --- a/src/backends/cuda/qwen3_5/README.md +++ b/src/backends/cuda/qwen3_5/README.md @@ -62,7 +62,10 @@ starts with zero state; prefix continuation is rejected by the model adapter. The context drains its last stream before destruction, and is retired before that stream. Rebuild the library for this optional group; ABI remains 6 on this #118-based branch; the historical cumulative validation build used ABI 8. -Builds now link both cuBLAS and cuBLASLt. +Builds now link both cuBLAS and cuBLASLt. Every consumer of the rebuilt shared +library must provide both runtime libraries (`libcublas.so` and +`libcublasLt.so`), even when it does not enable the optional FP32 GDN group. +An existing cuBLASLt-only deployment must add cuBLAS before upgrading. `tests/qwen3_5/test_gdn_fp32.py` compares against the pinned Transformers 5.17 no-FLA implementation. On the cumulative A800 sm_80/CUDA 13.0/Torch 2.14 build, all 14 cases passed diff --git a/src/models/cua_s1/README.md b/src/models/cua_s1/README.md index c24f3ee0..f36eceed 100644 --- a/src/models/cua_s1/README.md +++ b/src/models/cua_s1/README.md @@ -133,7 +133,11 @@ This profile targets the pinned no-FLA reference described above. Installing FLA changes Transformers' GDN implementation; comparisons must record which implementation actually ran. A different Torch version or GPU may choose other reduction orders, so qualification remains tied to the recorded environment. -The old merged export remains an approximate execution mode. The precision +The old merged export remains an approximate execution mode. PR #143 also +changes that mode's letter-logit readout from FP32 to BF16 rounding, which can +change close or tied choices after upgrading. The A800 unmerged-profile parity +results do not qualify this merged-path behavior change; no merged-path corpus +A/B run was performed. The precision profile does not change the screenshot adapter's execution mode. The historical cumulative A800 investigation and frozen-corpus results are recorded in From 22b45e8889382f3ed366f71cfc7afae0112773a5 Mon Sep 17 00:00:00 2001 From: levius <2114377220@qq.com> Date: Thu, 15 Oct 2026 09:40:47 +0800 Subject: [PATCH 17/21] docs(cua-s1): separate current ABI 8 from historical GPU evidence --- .../cua-s1-text-parity-20261013/README.md | 34 +++++++++++-------- recipe/cua_s1/native.md | 21 ++++++++---- src/backends/cuda/qwen3_5/README.md | 6 ++-- 3 files changed, 38 insertions(+), 23 deletions(-) diff --git a/docs/benchmarks/cua-s1-text-parity-20261013/README.md b/docs/benchmarks/cua-s1-text-parity-20261013/README.md index dd485921..62153d46 100644 --- a/docs/benchmarks/cua-s1-text-parity-20261013/README.md +++ b/docs/benchmarks/cua-s1-text-parity-20261013/README.md @@ -9,25 +9,29 @@ it does not establish that all five original L20X cases have been reproduced. ## PR dependencies and evidence provenance -This independent change is stacked on [#118](https://github.com/ThinkFlowLab/system1-omni/pull/118) -(`codex/jemm-native`, `ed9f9d6eb566d226e2441c5e9d84187abe1e4d63`) for the -unmerged FP32 language adapter loader and optional reference norm/GEMM/attention -operations. #118 is itself stacked on [#117](https://github.com/ThinkFlowLab/system1-omni/pull/117). -The frozen corpus comes from #141; merging #141 is not a runtime dependency. -Decider execution modes and Cua screenshot cache/resident-feature PRs are outside -this change's dependency chain and diff. +The independently qualified source `27f77e49868d84b091aa804f02ccb4d3474a9a00` +was stacked on [#118](https://github.com/ThinkFlowLab/system1-omni/pull/118) +at `ed9f9d6eb566d226e2441c5e9d84187abe1e4d63` for the unmerged FP32 language +adapter loader and optional reference norm/GEMM/attention operations. +The 2026-10-15 parent synchronization used #118 at +`3be46fa270cb74c33d4db4ffd761405d0522018a`, which inherits +[#117](https://github.com/ThinkFlowLab/system1-omni/pull/117) and its merged +resident-feature increment #135. Rebuild the library and all consumers together +for ABI 8. This parent synchronization has CPU/CI coverage; current integrated +GPU qualification remains unverified. Decider modes and the separate #127 cache +increment are not included. The frozen corpus comes from #141; merging #141 is +not a runtime dependency. The GPU results below were measured on cumulative candidate `76af07996f60905082b8d48199d4a7003f33f4f6`, whose parent is `18e1a5a6f68b9d11d3e11c7ea7b939af87272f53`. That integrated build used ABI 8. -The independently rebased branch retains #118's ABI 6 and adds only optional -FP32 GDN symbols. Rebuild the library with the worker; no cuDNN is needed for -this text path. Results from the cumulative build are historical evidence for -the pinned implementation. Fresh independent-branch qualification, including -exact source/library/binary hashes, repeated HTTP results and self-review, is -recorded separately in [PR #143](https://github.com/ThinkFlowLab/system1-omni/pull/143). -Use each run's source pin; do not substitute the historical integrated result -for a newer branch's validation. +The separate independent qualification at `27f77e49` used ABI 6 with additive +optional FP32 GDN symbols. No cuDNN is needed for this text path. Its exact +source/library/binary hashes, repeated HTTP results and self-review are recorded +in [PR #143](https://github.com/ThinkFlowLab/system1-omni/pull/143). +Both campaigns remain historical, source-pinned evidence. ABI 8 in the cumulative +campaign does not establish qualification of the current ABI-8 branch; use each +run's source pin rather than substituting either result for a newer build. ## Frozen inputs and environment diff --git a/recipe/cua_s1/native.md b/recipe/cua_s1/native.md index f2b3e908..3fae7add 100644 --- a/recipe/cua_s1/native.md +++ b/recipe/cua_s1/native.md @@ -1,6 +1,13 @@ # Cua-S1 4B 0.2 native text worker -The native worker ([`src/models/cua_s1/native/`](../../src/models/cua_s1/native/)) serves the `text` adapter like the reference worker in [`text.md`](text.md), with the Qwen3.5-4B forward pass on the CUDA kernels in [`src/backends/cuda/qwen3_5/`](../../src/backends/cuda/qwen3_5/) and no Python or PyTorch. It needs an NVIDIA GPU with compute capability 8.0 or newer; the unmerged precision profile was checked on the cumulative A800 80 GB (sm_80) build with CUDA 13.0; source-pinned qualification of the independent #118-based branch is tracked in [PR #143](https://github.com/ThinkFlowLab/system1-omni/pull/143). The earlier merged path was checked on an RTX 6000 Ada (sm_89) with CUDA 13.2. +The native worker ([`src/models/cua_s1/native/`](../../src/models/cua_s1/native/)) serves the `text` adapter like the reference worker in [`text.md`](text.md), with the Qwen3.5-4B forward pass on the CUDA kernels in [`src/backends/cuda/qwen3_5/`](../../src/backends/cuda/qwen3_5/) and no Python or PyTorch. It needs an NVIDIA GPU with compute capability 8.0 or newer. +The unmerged precision profile has historical A800 80 GB (sm_80), CUDA 13.0 +evidence for a cumulative ABI-8 build and the independent ABI-6 source +`27f77e49868d84b091aa804f02ccb4d3474a9a00`, linked in +[PR #143](https://github.com/ThinkFlowLab/system1-omni/pull/143). +The current branch uses ABI 8 after synchronizing with its updated parent; +neither historical campaign qualifies this integrated source. +The earlier merged path was checked on an RTX 6000 Ada (sm_89) with CUDA 13.2. Run the commands from the repository root. Pass your GPU's compute capability to `build.sh` (89 for Ada, 80 for A100, 90 for H100); the worker finds `libqwen3_5_cuda.so` next to its executable, or at `CUA_S1_CUDA_LIB`: @@ -71,8 +78,10 @@ For the screenshot worker, [the vision Graph recipe](native_vision_graph.md) documents `CUA_S1_VISION_GRAPH`, retained four-grid FIFO scratch, memory/lifetime limits and the real CUDA capture-failure wrapper build and test commands. -The ABI-8 parent synchronization preserves the unmerged precision profile, but -the recorded independent-head GPU campaign below belongs to its pinned earlier -source. It is historical evidence, not a fresh GPU qualification of this -integrated head. Rebuild consumers together and rerun the pinned device checks -before extending qualification to this build or another device. +The ABI-8 parent synchronization preserves the unmerged precision profile. +The historical independent-head GPU campaign is linked in +[PR #143](https://github.com/ThinkFlowLab/system1-omni/pull/143); the +[numerical investigation](../../docs/benchmarks/cua-s1-text-parity-20261013/README.md) +records the separate cumulative campaign. Rebuild consumers together and rerun +the pinned device checks before extending qualification to this build or +another device. diff --git a/src/backends/cuda/qwen3_5/README.md b/src/backends/cuda/qwen3_5/README.md index 817dfd33..13d48921 100644 --- a/src/backends/cuda/qwen3_5/README.md +++ b/src/backends/cuda/qwen3_5/README.md @@ -65,8 +65,10 @@ workspace; the caller supplies shape-dependent FP32 buffers. It implements Torch 2.14 cumulative-decay association, and a final BF16 output cast. Each call starts with zero state; prefix continuation is rejected by the model adapter. The context drains its last stream before destruction, and is retired before -that stream. Rebuild the library for this optional group; ABI remains 6 on this #118-based branch; the historical cumulative validation -build used ABI 8. +that stream. Rebuild the library and all consuming workers together for the +current ABI 8, including this optional group. The independent A800 qualification +at `27f77e49868d84b091aa804f02ccb4d3474a9a00` used ABI 6; the earlier cumulative +campaign used ABI 8. Neither campaign qualifies the current integrated source. Builds now link both cuBLAS and cuBLASLt. Every consumer of the rebuilt shared library must provide both runtime libraries (`libcublas.so` and `libcublasLt.so`), even when it does not enable the optional FP32 GDN group. From f212c56f23ac750ebfa38f4c72e9fcdc914bf1cd Mon Sep 17 00:00:00 2001 From: levius <2114377220@qq.com> Date: Thu, 15 Oct 2026 10:58:36 +0800 Subject: [PATCH 18/21] fix(cua-s1): preserve legacy merged readout on upgrade --- docs/architecture.md | 6 +- .../cua-s1-text-parity-20261013/README.md | 50 +++- recipe/cua_s1/native.md | 47 +++- src/models/cua_s1/README.md | 22 +- src/models/cua_s1/native/src/engine.rs | 4 + src/models/cua_s1/native/src/executor.rs | 24 +- tests/cua_s1/fixtures/LICENSE.jevbench | 21 ++ tests/cua_s1/fixtures/README.md | 31 +++ .../fixtures/legacy_merged_readout.json | 228 ++++++++++++++++++ tests/cua_s1/head.rs | 121 +++++++++- tests/cua_s1/legacy_merged_gpu.rs | 110 +++++++++ 11 files changed, 632 insertions(+), 32 deletions(-) create mode 100644 tests/cua_s1/fixtures/LICENSE.jevbench create mode 100644 tests/cua_s1/fixtures/README.md create mode 100644 tests/cua_s1/fixtures/legacy_merged_readout.json create mode 100644 tests/cua_s1/legacy_merged_gpu.rs diff --git a/docs/architecture.md b/docs/architecture.md index 433c5ddd..5677b13a 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -57,8 +57,10 @@ synchronizes and disposes a failed model before returning an inference error and reports unavailable health thereafter. Qwen executors own loaded models and CPU head weights with FP64 accumulation. -Cua-S1 rounds answer-letter logits to the tied `lm_head` BF16 output dtype before -FP32 softmax; learned scalar heads retain their FP32 rounding and bias order. Finishing checks output cardinality +Cua-S1's unmerged text profile rounds answer-letter logits to the tied `lm_head` +BF16 output dtype before FP32 softmax. Legacy merged text and screenshot +readouts retain FP32 logits; learned scalar heads retain their FP32 rounding +and bias order. Finishing checks output cardinality before reconstruction. HTTP validation, error status/body conventions, and real warmup before readiness remain model-specific and unchanged. diff --git a/docs/benchmarks/cua-s1-text-parity-20261013/README.md b/docs/benchmarks/cua-s1-text-parity-20261013/README.md index 62153d46..21a8b3e6 100644 --- a/docs/benchmarks/cua-s1-text-parity-20261013/README.md +++ b/docs/benchmarks/cua-s1-text-parity-20261013/README.md @@ -77,10 +77,58 @@ Token IDs matched on all 139 requests. The differences begin in execution: for tied options and removes the remaining readout differences. The fix selects the precision path using `cua-s1-text-unmerged/1`. Legacy merged -exports remain loadable. Other model adapters retain their existing numerical +exports remain loadable with their original FP32 readout; BF16 letter-logit +rounding is restricted to the unmerged text profile. Other model adapters retain their existing numerical mode. Prefix continuation is rejected in the new mode; eager is the default. The probability gate of 0.02 was not relaxed. +## Legacy merged upgrade regression + +The [2026-10-15 A800 ABI-8 evidence](https://github.com/Levius-Fubuki/system1-omni/releases/tag/cua-a800-abi8-20261015) +compares parent `5bdd625b4c27efb02f43a1f71150e1a7d7c0f071` with candidate +`7293c71cfb738504a30ff033ae3567f3368333c1`, using the same legacy merged artifact +and frozen 139-request corpus for two passes each. The candidate changed four +decisions, with maximum probability change 0.02441754937171936; correctness fell +from 97/139 to 96/139. Each changed decision collapsed the two leading options +into equal probabilities, then selected the earlier option in request order: + +| Request | Legacy choice | Rounded merged choice | Frozen task label | +| --- | --- | --- | --- | +| `original-routing-06-1` | `tools` | `general` | `general` | +| `hard-opus-a-temporal_numeric-07` | `sep_28` | `sep_24` | `sep_26` | +| `hard-sol-b-long_policy-05` | `delete_profile_omar_april_and_table` | `delete_everything` | `delete_profile_omar_april_and_table` | +| `hard-sol-b-temporal_numeric-03` | `9_credits` | `12_credits` | `9_credits` | + +The cause is the text executor's unconditional BF16 conversion of its FP32 dot +products. The model's merged loader still selects the old backbone arithmetic; +the unmerged reference-profile operations are opt-in. The compatibility fix +keeps the validated `TextExport` on each executor and applies BF16 readout only +to `Unmerged`. `Merged` retains the parent's FP64 accumulation followed by FP32 +readout. Softmax, option order, tie handling and screenshot execution are unchanged. +Restoring parent behavior reverses the candidate's one task-label improvement +and two regressions; the fourth case is wrong under both choices. Compatibility +is measured against the parent, independently of task labels. + +A CPU regression first failed with `[20.0, 20.0]` instead of +`[20.03125, 20.046875]`, then passed with the profile-specific readout. Tests also +cover both profiles' FP64 accumulation, nonfinite rejection, finite FP32 values +outside BF16 range, and the four requests' option order through real response +finishing with explicitly synthetic BF16 projection inputs. These synthetic +vectors are not captured checkpoint activations. The maintained four-request +device test uses the actual frozen inputs and parent responses; run it as in the +[native recipe](../../../recipe/cua_s1/native.md). + +Fresh GPU execution of the compatibility fix is **pending**: the earlier server +was shut down after preserving the published evidence. In particular, restored +97/139 correctness and zero upgrade flips have not been measured for this new +source. Requalify the full corpus on pinned A800: two passes each for legacy +parent and repaired merged eager/Graph, requiring zero parent/candidate decision +changes and probability error at most 0.02; retain exact-byte differences too. +Repeat unmerged eager/Graph/reference checks at the original 0.02/zero-flip gates. +The previous candidate's unmerged results were 97/139 in all three modes, maximum +probability difference 5.960464477539063e-8 and zero flips. They remain tied to +`7293c71c`; they are not a new-device pass for this follow-up. + ## Results | Real HTTP mode | Passes × requests | Flips vs reference | Maximum probability difference | Correct | diff --git a/recipe/cua_s1/native.md b/recipe/cua_s1/native.md index 3fae7add..e4421d42 100644 --- a/recipe/cua_s1/native.md +++ b/recipe/cua_s1/native.md @@ -1,12 +1,12 @@ # Cua-S1 4B 0.2 native text worker The native worker ([`src/models/cua_s1/native/`](../../src/models/cua_s1/native/)) serves the `text` adapter like the reference worker in [`text.md`](text.md), with the Qwen3.5-4B forward pass on the CUDA kernels in [`src/backends/cuda/qwen3_5/`](../../src/backends/cuda/qwen3_5/) and no Python or PyTorch. It needs an NVIDIA GPU with compute capability 8.0 or newer. -The unmerged precision profile has historical A800 80 GB (sm_80), CUDA 13.0 -evidence for a cumulative ABI-8 build and the independent ABI-6 source -`27f77e49868d84b091aa804f02ccb4d3474a9a00`, linked in +The unmerged precision profile has source-pinned A800 80 GB (sm_80), CUDA 13.0 +evidence at ABI-8 source `7293c71cfb738504a30ff033ae3567f3368333c1`, plus earlier +cumulative ABI-8 and independent ABI-6 campaigns linked in [PR #143](https://github.com/ThinkFlowLab/system1-omni/pull/143). -The current branch uses ABI 8 after synchronizing with its updated parent; -neither historical campaign qualifies this integrated source. +The current branch uses ABI 8. The later legacy-readout compatibility fix has +CPU regression coverage; fresh GPU validation of that follow-up is pending. The earlier merged path was checked on an RTX 6000 Ada (sm_89) with CUDA 13.2. Run the commands from the repository root. Pass your GPU's compute capability to `build.sh` (89 for Ada, 80 for A100, 90 for H100); the worker finds `libqwen3_5_cuda.so` next to its executable, or at `CUA_S1_CUDA_LIB`: @@ -48,10 +48,13 @@ markers or altered adapter bytes fail before CUDA loading. Prefix continuation is unsupported in this precision mode. Existing `cua-s1-text-merged/1` exports remain loadable with the earlier merged -backbone execution path. PR #143 changes their letter-logit readout too: it now -rounds to BF16 instead of retaining FP32, so close or tied choices can change -after upgrading. No merged-path corpus A/B run qualifies that behavior change; -the A800 parity results cover the unmerged precision profile only. +backbone execution path and retain FP32 letter logits after FP64 accumulation. +The BF16 readout boundary is restricted to the validated unmerged export profile, +not a request flag or process-global setting. An earlier #143 build applied it to +merged exports too, creating four decision changes and reducing frozen-corpus +correctness from 97/139 to 96/139. The follow-up restores legacy readout policy; +that restoration still needs fresh device validation. The A800 unmerged parity +results are scoped to their recorded source and artifact. Merging the adapter, TF32/BF16 GDN intermediates, and different GEMM reduction policies can change decisions on close or tied options. Re-export with the command above to use the precision profile. Rebuild worker and library @@ -74,14 +77,36 @@ CUA_S1_CUDA_LIB=$PWD/target/release/libqwen3_5_cuda.so \ cargo test --release -p omni-qwen3-5-native --test kernels -- --ignored ``` +The maintained legacy-upgrade regression uses the four frozen requests and +parent-baseline responses in `tests/cua_s1/fixtures/legacy_merged_readout.json`. +Those responses are compatibility oracles, not task ground truth. With the same +pinned A800 merged artifact, run each Graph setting in a separate process: + +```sh +for graph in 0 1; do + CUA_S1_GRAPH=$graph NVIDIA_TF32_OVERRIDE=0 \ + CUA_S1_MERGED_TEST_MODEL=$PWD/weights/cua-s1-4b-0.2-text-merged \ + CUA_S1_CUDA_LIB=$PWD/target/release/libqwen3_5_cuda.so \ + cargo test --release --locked -p omni-cua-s1-native --lib \ + engine::legacy_merged_tests::legacy_merged_upgrade_preserves_four_decisions \ + -- --ignored --exact --test-threads=1 --nocapture +done +``` + +This selects one device test per process, running all four requests twice through +prepare/execute/finish. It requires unchanged baseline decisions, probability +error at most 0.02, identity/usage/option order and exact repeat bytes. It does +not substitute for the full 139-request legacy upgrade and unmerged-reference +comparisons. See the [four-case diagnosis and retest scope](../../docs/benchmarks/cua-s1-text-parity-20261013/README.md#legacy-merged-upgrade-regression). + For the screenshot worker, [the vision Graph recipe](native_vision_graph.md) documents `CUA_S1_VISION_GRAPH`, retained four-grid FIFO scratch, memory/lifetime limits and the real CUDA capture-failure wrapper build and test commands. The ABI-8 parent synchronization preserves the unmerged precision profile. -The historical independent-head GPU campaign is linked in +The source-pinned GPU campaigns are linked in [PR #143](https://github.com/ThinkFlowLab/system1-omni/pull/143); the [numerical investigation](../../docs/benchmarks/cua-s1-text-parity-20261013/README.md) -records the separate cumulative campaign. Rebuild consumers together and rerun +records their scopes and the legacy-readout follow-up. Rebuild consumers together and rerun the pinned device checks before extending qualification to this build or another device. diff --git a/src/models/cua_s1/README.md b/src/models/cua_s1/README.md index bb41afdf..08e835bd 100644 --- a/src/models/cua_s1/README.md +++ b/src/models/cua_s1/README.md @@ -10,8 +10,10 @@ The native worker follows the [architecture contracts](../../../docs/architectur with independent [processing](native/src/processing.rs) and [executor](native/src/executor.rs) modules. Preparation returns token IDs and an option count per question, plus a response context for identity/order and usage. -The executor projects the letter rows, rounds to the BF16 `lm_head` output dtype, -then returns FP32 letter logits; response finishing owns per-question +The executor projects the letter rows with FP64 accumulation and returns FP32 +letter logits. Only the unmerged text profile first rounds to the BF16 `lm_head` +output dtype; legacy merged text and screenshot readouts retain FP32. The text +export policy is validated once and owned by each executor. Response finishing owns per-question softmax and the choice/confidence interpretation. The engine owns a [shared serial scheduler](../../runtime/README.md) that admits each question's forward before blocking dispatch. Execution remains one prompt per question @@ -133,12 +135,16 @@ This profile targets the pinned no-FLA reference described above. Installing FLA changes Transformers' GDN implementation; comparisons must record which implementation actually ran. A different Torch version or GPU may choose other reduction orders, so qualification remains tied to the recorded environment. -The old merged export remains an approximate execution mode. PR #143 also -changes that mode's letter-logit readout from FP32 to BF16 rounding, which can -change close or tied choices after upgrading. The A800 unmerged-profile parity -results do not qualify this merged-path behavior change; no merged-path corpus -A/B run was performed. The precision -profile does not change the screenshot adapter's execution mode. +The old merged export remains an approximate execution mode. Its legacy +FP64-accumulated, FP32 letter-logit readout is preserved; BF16 rounding is selected +only by `cua-s1-text-unmerged/1`. Applying that rounding to merged exports at +`7293c71c` created four ordered-tie decision changes in a same-artifact A800 +upgrade comparison. The compatibility fix is covered by CPU red/green projection +and response tests; a frozen four-request GPU regression is also provided. +See the [numerical report](../../../docs/benchmarks/cua-s1-text-parity-20261013/README.md#legacy-merged-upgrade-regression) +for the four cases, source pins and device-validation limits. Restoring merged +compatibility does not make merged weights equivalent to the unmerged reference. +The precision profile does not change the screenshot adapter's execution mode. The historical cumulative A800 investigation and frozen-corpus results are recorded in [the numerical report](../../../docs/benchmarks/cua-s1-text-parity-20261013/README.md). diff --git a/src/models/cua_s1/native/src/engine.rs b/src/models/cua_s1/native/src/engine.rs index 5c0b434e..86dcbcec 100644 --- a/src/models/cua_s1/native/src/engine.rs +++ b/src/models/cua_s1/native/src/engine.rs @@ -40,3 +40,7 @@ impl Engine { #[cfg(test)] #[path = "../../../../../tests/cua_s1/text_export.rs"] mod tests; + +#[cfg(test)] +#[path = "../../../../../tests/cua_s1/legacy_merged_gpu.rs"] +mod legacy_merged_tests; diff --git a/src/models/cua_s1/native/src/executor.rs b/src/models/cua_s1/native/src/executor.rs index 6bcbdba2..2acba636 100644 --- a/src/models/cua_s1/native/src/executor.rs +++ b/src/models/cua_s1/native/src/executor.rs @@ -22,6 +22,7 @@ pub struct Input { pub struct Executor { model: Arc>, letters: Vec, + export: TextExport, } impl Executor { @@ -46,6 +47,7 @@ impl Executor { Ok(Self { model: Arc::new(Mutex::new(model)), letters, + export, }) } @@ -67,13 +69,23 @@ impl Executor { .forward(&input.ids) }) .await?; - rows.push(letter_logits(&self.letters, &last, input.n_options)?); + rows.push(letter_logits( + &self.letters, + &last, + input.n_options, + self.export, + )?); } Ok(rows) } } -fn letter_logits(letters: &[f32], last: &[f32], n_options: usize) -> Result> { +fn letter_logits( + letters: &[f32], + last: &[f32], + n_options: usize, + export: TextExport, +) -> Result> { let logits: Vec = letters .chunks_exact(last.len()) .take(n_options) @@ -83,8 +95,12 @@ fn letter_logits(letters: &[f32], last: &[f32], n_options: usize) -> Result() as f32; - // The tied lm_head emits BF16 before reference option selection/softmax. - half::bf16::from_f32(dot).to_f32() + match export { + // Preserve the original merged worker's FP32 readout on upgrade. + TextExport::Merged => dot, + // The unmerged reference profile's tied lm_head emits BF16. + TextExport::Unmerged => half::bf16::from_f32(dot).to_f32(), + } }) .collect(); // Abort before submitting a later prompt, as the former per-question softmax did. diff --git a/tests/cua_s1/fixtures/LICENSE.jevbench b/tests/cua_s1/fixtures/LICENSE.jevbench new file mode 100644 index 00000000..ebdd5738 --- /dev/null +++ b/tests/cua_s1/fixtures/LICENSE.jevbench @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2026 Florian Standhartinger and contributors + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/tests/cua_s1/fixtures/README.md b/tests/cua_s1/fixtures/README.md new file mode 100644 index 00000000..a22a9f1f --- /dev/null +++ b/tests/cua_s1/fixtures/README.md @@ -0,0 +1,31 @@ +# Legacy merged readout regression oracle + +`legacy_merged_readout.json` keeps four requests from +[`benchmarks/e2e/choice.jsonl` at `86926064`](https://github.com/ThinkFlowLab/system1-omni/blob/869260649f7dd5b56cac4d91eaae9087f2e985ee/benchmarks/e2e/choice.jsonl). +The original corpus SHA-256 is recorded in the fixture. Its JevBench-derived +request material is MIT licensed; the original copyright and full terms are +preserved in [`LICENSE.jevbench`](LICENSE.jevbench). + +The baseline responses come from A800/ABI-8 parent `5bdd625b`, using the same +legacy merged artifact as candidate `7293c71c`. The exact source, model, artifact +and environment hashes are in the +[public raw-evidence archive](https://github.com/Levius-Fubuki/system1-omni/releases/tag/cua-a800-abi8-20261015): +`evidence/pr143/merged-before/pass0.jsonl` supplies each `expected_response`, +`evidence/pr143/merged-after/pass0.jsonl` supplies `rounded_choice`, and +`evidence/pr143/native-eager/manifest.jsonl` supplies the requests. Both measured +passes produced identical response bytes. Only the four cases whose decisions +changed are retained here as a maintained compatibility oracle. Expected +responses preserve the parent behavior; they are **not task ground truth**. + +Consumers: + +- `tests/cua_s1/head.rs` uses the actual option order with explicitly synthetic + BF16 projection vectors to test the CPU readout/response boundary in both modes. +- `tests/cua_s1/legacy_merged_gpu.rs` runs the actual frozen requests twice through + the legacy model after verifying every recorded artifact file's size and + SHA-256, requiring baseline decisions, probability error at + most 0.02, identity/usage/order and exact repeat bytes. Run Graph 0/1 in separate + processes as documented in `recipe/cua_s1/native.md`. + +The GPU oracle is specific to the recorded A800 artifact/environment. Compilation +or a passing synthetic CPU test does not establish a new real-model GPU pass. diff --git a/tests/cua_s1/fixtures/legacy_merged_readout.json b/tests/cua_s1/fixtures/legacy_merged_readout.json new file mode 100644 index 00000000..3e00e482 --- /dev/null +++ b/tests/cua_s1/fixtures/legacy_merged_readout.json @@ -0,0 +1,228 @@ +{ + "schema": "cua-s1-legacy-merged-readout-regression/1", + "purpose": "Compatibility oracle for the four merged-export upgrade changes; expected responses preserve the legacy baseline, not task ground truth.", + "baseline_source": "5bdd625b4c27efb02f43a1f71150e1a7d7c0f071", + "rejected_source": "7293c71cfb738504a30ff033ae3567f3368333c1", + "evidence": "https://github.com/Levius-Fubuki/system1-omni/releases/tag/cua-a800-abi8-20261015", + "corpus_source": "869260649f7dd5b56cac4d91eaae9087f2e985ee", + "corpus_sha256": "8f4b912e5dfd8706992e90f1b87bb07a78a8339e5a9fad3ef64b3b04771f9daf", + "environment": { + "gpu": "A800 sm80", + "abi": 8, + "base_revision": "851bf6e806efd8d0a36b00ddf55e13ccb7b8cd0a", + "adapter_revision": "16818868b0cc7813808aae4e87b417657046ab79", + "torch": "2.14.0+cu130", + "transformers": "5.17.0", + "tf32_override": "0", + "export_format": "cua-s1-text-merged/1" + }, + "cases": [ + { + "id": "original-routing-06-1", + "request": { + "state": "Give me three creative pet-dragon names.", + "questions": { + "q": { + "criteria": { + "coding": "Self-contained code writing or explanation without repository operations", + "coding_agent": "Inspect/edit repository files or run tests", + "document": "Answer from a supplied document", + "general": "None of the specialist categories", + "math": "Self-contained calculation or proof", + "tools": "Carry out an external service action" + }, + "instructions": "Choose the specialist needed for the request. File edits with test execution use coding_agent, even if code-related.", + "type": "choice" + } + }, + "model": "cua-s1-4b-0.2" + }, + "expected_response": { + "model": "cua-ai/cua-s1-4b-0.2@16818868b0cc7813808aae4e87b417657046ab79:text", + "answers": { + "q": { + "type": "choice", + "choice": "tools", + "probabilities": { + "coding": 0.12521012127399445, + "coding_agent": 0.2372714877128601, + "document": 0.0032456598710268736, + "general": 0.3135629892349243, + "math": 0.0015466281911358237, + "tools": 0.31916311383247375 + }, + "confidence": 0.24194648916375006 + } + }, + "usage": { + "input_tokens": 233, + "output_tokens": 0 + } + }, + "rounded_choice": "general" + }, + { + "id": "hard-opus-a-temporal_numeric-07", + "request": { + "state": "NIMBUSGRID CLOUD — BUDGET ALERT CONFIGURATION AND DAILY COST EXPORT (ACCOUNT ANALYTICS-PROD, SEPTEMBER 2026, USD)\n\nBudget BGT-ANALYTICS-09: monthly budget USD 12,000.00.\nAlert rule A1: \"Fires once, at the end of the first day on which the month-to-date net cost is at or above 80% of the monthly budget.\"\nDefinitions (NimbusGrid billing guide): \"Net cost\" = usage charges minus credits applied, excluding taxes. Taxes are shown for information and are never counted toward budgets. Credits are counted on the day they are applied.\nNote: a promotional credit of USD 600.00 was applied on 18 September (migration incentive).\n\nDaily export:\nSep 01 | usage 402.10 | credits 0.00 | tax (8%) 32.17\nSep 02 | usage 389.55 | credits 0.00 | tax (8%) 31.16\nSep 03 | usage 415.00 | credits 0.00 | tax (8%) 33.20\nSep 04 | usage 398.40 | credits 0.00 | tax (8%) 31.87\nSep 05 | usage 377.25 | credits 0.00 | tax (8%) 30.18\nSep 06 | usage 361.80 | credits 0.00 | tax (8%) 28.94\nSep 07 | usage 420.65 | credits 0.00 | tax (8%) 33.65\nSep 08 | usage 433.10 | credits 0.00 | tax (8%) 34.65\nSep 09 | usage 409.95 | credits 0.00 | tax (8%) 32.80\nSep 10 | usage 395.00 | credits 0.00 | tax (8%) 31.60\nSep 11 | usage 388.70 | credits 0.00 | tax (8%) 31.10\nSep 12 | usage 402.35 | credits 0.00 | tax (8%) 32.19\nSep 13 | usage 371.20 | credits 0.00 | tax (8%) 29.70\nSep 14 | usage 366.45 | credits 0.00 | tax (8%) 29.32\nSep 15 | usage 418.90 | credits 0.00 | tax (8%) 33.51\nSep 16 | usage 427.30 | credits 0.00 | tax (8%) 34.18\nSep 17 | usage 411.05 | credits 0.00 | tax (8%) 32.88\nSep 18 | usage 399.60 | credits 600.00 | tax (8%) 31.97\nSep 19 | usage 404.25 | credits 0.00 | tax (8%) 32.34\nSep 20 | usage 392.80 | credits 0.00 | tax (8%) 31.42\nSep 21 | usage 386.15 | credits 0.00 | tax (8%) 30.89\nSep 22 | usage 375.40 | credits 0.00 | tax (8%) 30.03\nSep 23 | usage 410.00 | credits 0.00 | tax (8%) 32.80\nSep 24 | usage 421.75 | credits 0.00 | tax (8%) 33.74\nSep 25 | usage 398.00 | credits 0.00 | tax (8%) 31.84\nSep 26 | usage 223.30 | credits 0.00 | tax (8%) 17.86\nSep 27 | usage 380.00 | credits 0.00 | tax (8%) 30.40\nSep 28 | usage 412.60 | credits 0.00 | tax (8%) 33.01\nSep 29 | usage 405.10 | credits 0.00 | tax (8%) 32.41\nSep 30 | usage 399.85 | credits 0.00 | tax (8%) 31.99\n\nFinOps channel, 24 September: \"With tax we're already over 9,600 — the alert should have fired today. Something's broken?\"\n", + "questions": { + "q": { + "criteria": { + "sep_24": "At the end of 24 September.", + "sep_25": "At the end of 25 September.", + "sep_26": "At the end of 26 September.", + "sep_27": "At the end of 27 September.", + "sep_28": "At the end of 28 September." + }, + "instructions": "On which day does alert rule A1 fire?", + "type": "choice" + } + }, + "model": "cua-s1-4b-0.2" + }, + "expected_response": { + "model": "cua-ai/cua-s1-4b-0.2@16818868b0cc7813808aae4e87b417657046ab79:text", + "answers": { + "q": { + "type": "choice", + "choice": "sep_28", + "probabilities": { + "sep_24": 0.3386138677597046, + "sep_25": 0.07059328258037567, + "sep_26": 0.07448268681764603, + "sep_27": 0.16069446504116058, + "sep_28": 0.3556157052516937 + }, + "confidence": 0.12471221294469215 + } + }, + "usage": { + "input_tokens": 1381, + "output_tokens": 0 + } + }, + "rounded_choice": "sep_24" + }, + { + "id": "hard-sol-b-long_policy-05", + "request": { + "state": "RECORDS DELETION AND LEGAL HOLD HANDBOOK\n1. Rule hierarchy. Legal holds override ordinary deletion schedules. A later written release controls the named scope only. The privacy deletion process does not authorize destruction barred by a hold. The clarification memorandum at the end is binding.\n\n2. Definitions. \"Responsive record\" means a record within the custodians, systems, subjects, and time range stated by a hold. Exact duplicates are each responsive when stored in separate systems unless the hold notice permits deduplication. \"Derived analytics\" are records produced from source data; they are not source records. \"Verified deletion\" means deletion from active systems and expiration from ordinary backups under the stated schedule, with a logged confirmation. Cryptographic erasure counts only if the relevant keys exclusively protect the named records.\n\n3. Customer requests. A verified identity request is due within 30 calendar days. Data needed for contract performance may be retained while necessary. Data under legal hold is retained even after the ordinary purpose ends, and the customer receives the standard limitation notice. Non-held data in the same account remains subject to deletion.\n\n4. Backup process. Active copies are deleted by the due date. Immutable backups expire on a 35-day rotation; restoration procedures must reapply deletion markers. The completion record may therefore distinguish active deletion from final backup expiry.\n\n5. Holds. Hold H-17 covers support messages by custodians June and Omar concerning Project Cedar from 1 January through 31 March 2026, including attachments and exports. Hold H-22 covers all billing records for account 448 from 1 February onward. Neither permits deduplication. A legal email may release a hold only if it identifies the hold number and scope.\n\n6. Case file. Account 448 submitted a verified deletion request on 6 April. Contract service ended 5 April. The account contains profile data, invoices dated February and March, support messages with June about Project Cedar dated 20 March, unrelated support messages with Omar dated 2 April, and a dashboard table derived by counting Cedar messages by week. The table contains only week and count. On 10 April Legal wrote: \"Release H-17 for Omar's records only; June remains held. H-22 remains fully active.\" Privacy proposes deleting the profile, every support message, the derived table, and all invoices from active storage on 20 April, leaving deletion markers for backups.\n\n7. Scope application. A release for one custodian does not release another. Records outside a hold's date range are not held merely because a named custodian created them. Derived analytics are covered only if expressly included or if they independently meet the hold's stated record description.\n\n8. Decision options. The deletion team must choose the option that can be executed on 20 April without violating a hold. Ordinary backup expiry may follow later and does not prevent timely active-system action.\n\nCLARIFICATION MEMORANDUM, 12 April. H-22's term \"billing records\" includes invoices and payment ledger entries but not profile contact fields used only for login. H-17's term \"support messages\" does not include aggregate derived tables that contain no message text, attachment, customer identifier, or custodian identifier. This memorandum clarifies existing scope and does not release either hold.\n\nADMINISTRATIVE AND REFERENCE MATERIAL (part of the document):\n\nAppendix 1. identity verification. The organization keeps a contemporaneous record for this topic. Routine records identify the owner, the date recorded, the system of record, and any later correction. A draft note is not an approval, silence is not consent, and a copied recipient is not treated as the decision maker unless a controlling section expressly says otherwise. Staff should preserve ordinary supporting material and use the defined business calendar. This appendix supplies administration only: it does not enlarge eligibility, erase a deadline, reverse a precedence rule, or create an exception to a requirement stated in the operative sections. Where examples use names or dates, they illustrate recordkeeping and do not amend the operative case. Historical versions remain searchable for audit purposes, but a superseded version has no prospective effect. Questions are logged with enough context for a reviewer to reproduce the decision without relying on memory. The records team checks completeness monthly and reports aggregate defects; it does not re-decide the underlying case. Reference code REC-01 is an index locator, not a priority ranking or outcome code.\n\nAppendix 2. backup inventories. The organization keeps a contemporaneous record for this topic. Routine records identify the owner, the date recorded, the system of record, and any later correction. A draft note is not an approval, silence is not consent, and a copied recipient is not treated as the decision maker unless a controlling section expressly says otherwise. Staff should preserve ordinary supporting material and use the defined business calendar. This appendix supplies administration only: it does not enlarge eligibility, erase a deadline, reverse a precedence rule, or create an exception to a requirement stated in the operative sections. Where examples use names or dates, they illustrate recordkeeping and do not amend the operative case. Historical versions remain searchable for audit purposes, but a superseded version has no prospective effect. Questions are logged with enough context for a reviewer to reproduce the decision without relying on memory. The records team checks completeness monthly and reports aggregate defects; it does not re-decide the underlying case. Reference code REC-02 is an index locator, not a priority ranking or outcome code.\n\nAppendix 3. custodian interviews. The organization keeps a contemporaneous record for this topic. Routine records identify the owner, the date recorded, the system of record, and any later correction. A draft note is not an approval, silence is not consent, and a copied recipient is not treated as the decision maker unless a controlling section expressly says otherwise. Staff should preserve ordinary supporting material and use the defined business calendar. This appendix supplies administration only: it does not enlarge eligibility, erase a deadline, reverse a precedence rule, or create an exception to a requirement stated in the operative sections. Where examples use names or dates, they illustrate recordkeeping and do not amend the operative case. Historical versions remain searchable for audit purposes, but a superseded version has no prospective effect. Questions are logged with enough context for a reviewer to reproduce the decision without relying on memory. The records team checks completeness monthly and reports aggregate defects; it does not re-decide the underlying case. Reference code REC-03 is an index locator, not a priority ranking or outcome code.\n\nAppendix 4. deletion markers. The organization keeps a contemporaneous record for this topic. Routine records identify the owner, the date recorded, the system of record, and any later correction. A draft note is not an approval, silence is not consent, and a copied recipient is not treated as the decision maker unless a controlling section expressly says otherwise. Staff should preserve ordinary supporting material and use the defined business calendar. This appendix supplies administration only: it does not enlarge eligibility, erase a deadline, reverse a precedence rule, or create an exception to a requirement stated in the operative sections. Where examples use names or dates, they illustrate recordkeeping and do not amend the operative case. Historical versions remain searchable for audit purposes, but a superseded version has no prospective effect. Questions are logged with enough context for a reviewer to reproduce the decision without relying on memory. The records team checks completeness monthly and reports aggregate defects; it does not re-decide the underlying case. Reference code REC-04 is an index locator, not a priority ranking or outcome code.\n\nAppendix 5. request correspondence. The organization keeps a contemporaneous record for this topic. Routine records identify the owner, the date recorded, the system of record, and any later correction. A draft note is not an approval, silence is not consent, and a copied recipient is not treated as the decision maker unless a controlling section expressly says otherwise. Staff should preserve ordinary supporting material and use the defined business calendar. This appendix supplies administration only: it does not enlarge eligibility, erase a deadline, reverse a precedence rule, or create an exception to a requirement stated in the operative sections. Where examples use names or dates, they illustrate recordkeeping and do not amend the operative case. Historical versions remain searchable for audit purposes, but a superseded version has no prospective effect. Questions are logged with enough context for a reviewer to reproduce the decision without relying on memory. The records team checks completeness monthly and reports aggregate defects; it does not re-decide the underlying case. Reference code REC-05 is an index locator, not a priority ranking or outcome code.\n\nAppendix 6. analytics catalog. The organization keeps a contemporaneous record for this topic. Routine records identify the owner, the date recorded, the system of record, and any later correction. A draft note is not an approval, silence is not consent, and a copied recipient is not treated as the decision maker unless a controlling section expressly says otherwise. Staff should preserve ordinary supporting material and use the defined business calendar. This appendix supplies administration only: it does not enlarge eligibility, erase a deadline, reverse a precedence rule, or create an exception to a requirement stated in the operative sections. Where examples use names or dates, they illustrate recordkeeping and do not amend the operative case. Historical versions remain searchable for audit purposes, but a superseded version has no prospective effect. Questions are logged with enough context for a reviewer to reproduce the decision without relying on memory. The records team checks completeness monthly and reports aggregate defects; it does not re-decide the underlying case. Reference code REC-06 is an index locator, not a priority ranking or outcome code.\n\nAppendix 7. restoration drills. The organization keeps a contemporaneous record for this topic. Routine records identify the owner, the date recorded, the system of record, and any later correction. A draft note is not an approval, silence is not consent, and a copied recipient is not treated as the decision maker unless a controlling section expressly says otherwise. Staff should preserve ordinary supporting material and use the defined business calendar. This appendix supplies administration only: it does not enlarge eligibility, erase a deadline, reverse a precedence rule, or create an exception to a requirement stated in the operative sections. Where examples use names or dates, they illustrate recordkeeping and do not amend the operative case. Historical versions remain searchable for audit purposes, but a superseded version has no prospective effect. Questions are logged with enough context for a reviewer to reproduce the decision without relying on memory. The records team checks completeness monthly and reports aggregate defects; it does not re-decide the underlying case. Reference code REC-07 is an index locator, not a priority ranking or outcome code.\n\nAppendix 8. hold training. The organization keeps a contemporaneous record for this topic. Routine records identify the owner, the date recorded, the system of record, and any later correction. A draft note is not an approval, silence is not consent, and a copied recipient is not treated as the decision maker unless a controlling section expressly says otherwise. Staff should preserve ordinary supporting material and use the defined business calendar. This appendix supplies administration only: it does not enlarge eligibility, erase a deadline, reverse a precedence rule, or create an exception to a requirement stated in the operative sections. Where examples use names or dates, they illustrate recordkeeping and do not amend the operative case. Historical versions remain searchable for audit purposes, but a superseded version has no prospective effect. Questions are logged with enough context for a reviewer to reproduce the decision without relying on memory. The records team checks completeness monthly and reports aggregate defects; it does not re-decide the underlying case. Reference code REC-08 is an index locator, not a priority ranking or outcome code.", + "questions": { + "q": { + "criteria": { + "delete_everything": "The case results in delete everything under all controlling provisions.", + "delete_nothing": "The case results in delete nothing under all controlling provisions.", + "delete_only_profile": "The case results in delete only profile under all controlling provisions.", + "delete_profile_omar_april_and_table": "The case results in delete profile omar april and table under all controlling provisions." + }, + "instructions": "Apply the controlling document and select the single authorized outcome for the case file.", + "type": "choice" + } + }, + "model": "cua-s1-4b-0.2" + }, + "expected_response": { + "model": "cua-ai/cua-s1-4b-0.2@16818868b0cc7813808aae4e87b417657046ab79:text", + "answers": { + "q": { + "type": "choice", + "choice": "delete_profile_omar_april_and_table", + "probabilities": { + "delete_everything": 0.3959157466888428, + "delete_nothing": 0.1504596620798111, + "delete_only_profile": 0.014968502335250378, + "delete_profile_omar_april_and_table": 0.4386560916900635 + }, + "confidence": 0.22369902796665087 + } + }, + "usage": { + "input_tokens": 2630, + "output_tokens": 0 + } + }, + "rounded_choice": "delete_everything" + }, + { + "id": "hard-sol-b-temporal_numeric-03", + "request": { + "state": { + "event": "Classify the earned service credit.", + "rules": "An annual service credit is prorated by active calendar days in a leap year, including both activation and cancellation dates. The annual amount is 1,098 credits for 2028, which has 366 days. The service activated February 28 and cancelled March 2. A suspension covering February 29 removes that day from active days, but March 1 remains active. Compute exactly before classifying; no rounding is needed because the daily amount is integral." + }, + "questions": { + "q": { + "criteria": { + "12_credits": "The correctly calculated outcome is 12 credits.", + "15_credits": "The correctly calculated outcome is 15 credits.", + "6_credits": "The correctly calculated outcome is 6 credits.", + "9_credits": "The correctly calculated outcome is 9 credits." + }, + "instructions": "Apply all stated timing, unit, inclusion, correction, and threshold rules; select the exact band.", + "type": "choice" + } + }, + "model": "cua-s1-4b-0.2" + }, + "expected_response": { + "model": "cua-ai/cua-s1-4b-0.2@16818868b0cc7813808aae4e87b417657046ab79:text", + "answers": { + "q": { + "type": "choice", + "choice": "9_credits", + "probabilities": { + "12_credits": 0.3566144108772278, + "15_credits": 0.15715421736240387, + "6_credits": 0.1183929517865181, + "9_credits": 0.36783841252326965 + }, + "confidence": 0.07737968315894639 + } + }, + "usage": { + "input_tokens": 318, + "output_tokens": 0 + } + }, + "rounded_choice": "12_credits" + } + ], + "artifact_files": { + "chat_template.jinja": { + "size": 7756, + "sha256": "a4aee8afcf2e0711942cf848899be66016f8d14a889ff9ede07bca099c28f715" + }, + "config.json": { + "size": 1979, + "sha256": "63f47812d0f11118e4d252d2b3ad488707eb9287a11589f4fd382a1d31182724" + }, + "cua_s1_export.json": { + "size": 34, + "sha256": "472b609a176f714e2dc06db60a88659c30d87c6af487ae4986ba6cbf70deb101" + }, + "generation_config.json": { + "size": 116, + "sha256": "62153eb6c69f2e1f426beaa8002b7186437e949c7588167085df14e10e9c0a73" + }, + "model-00001-of-00002.safetensors": { + "size": 4972947968, + "sha256": "d2544c2400e723881244e260ce905eec23b3dca12e949cdcdc491a1457f34004" + }, + "model-00002-of-00002.safetensors": { + "size": 3438610328, + "sha256": "31bf465a185fd605099a11d155582432499c13e2257e4c4fa13d1024630b4349" + }, + "model.safetensors.index.json": { + "size": 41993, + "sha256": "83620b40d724f8b286580b5c7c504222b0bdbdd7dc9f068e565132409c7eec9d" + }, + "tokenizer.json": { + "size": 19989325, + "sha256": "06b9509352d2af50381ab2247e083b80d32d5c0aba91c272ca9ff729b6a0e523" + }, + "tokenizer_config.json": { + "size": 1124, + "sha256": "66e427c470fe580fe8c7b5725d857af23d8417e37fae62667ec698306a19987b" + } + } +} diff --git a/tests/cua_s1/head.rs b/tests/cua_s1/head.rs index 37e03680..bcb34cc5 100644 --- a/tests/cua_s1/head.rs +++ b/tests/cua_s1/head.rs @@ -1,15 +1,38 @@ use super::*; +#[test] +fn legacy_merged_projection_preserves_fp32_close_scores() { + // Real BF16-representable weights and hidden values, with two distinct + // FP32 dot products in the same BF16 bin. Legacy merged exports must not + // acquire the new reference profile's ordered tie when the worker upgrades. + assert_eq!( + letter_logits( + &[20.0, 0.03125, 20.0, 0.046875], + &[1.0, 1.0], + 2, + TextExport::Merged + ) + .unwrap(), + [20.03125, 20.046875] + ); +} + #[test] fn letter_projection_matches_the_reference_bf16_output_boundary() { assert_eq!( - letter_logits(&[1.0078125], &[1.0078125], 1).unwrap(), + letter_logits(&[1.0078125], &[1.0078125], 1, TextExport::Unmerged).unwrap(), [1.015625] ); // Both exact dot products belong to the same BF16 bin. Retaining FP32 // invents a winning second option where the reference has an ordered tie. assert_eq!( - letter_logits(&[20.0, 0.03125, 20.0, 0.046875], &[1.0, 1.0], 2).unwrap(), + letter_logits( + &[20.0, 0.03125, 20.0, 0.046875], + &[1.0, 1.0], + 2, + TextExport::Unmerged + ) + .unwrap(), [20.0, 20.0] ); } @@ -30,14 +53,100 @@ fn letter_projection_preserves_fp64_accumulation_and_bf16_readout() { // The first dot product would be zero with FP32 accumulation. The second // must round before postprocessing; unused letter rows must not be read. assert_eq!( - letter_logits(&letters, &[1.0, 1.0, 1.0], 2).unwrap(), + letter_logits(&letters, &[1.0, 1.0, 1.0], 2, TextExport::Unmerged).unwrap(), + [1.0, 16_777_216.0] + ); + assert_eq!( + letter_logits(&letters, &[1.0, 1.0, 1.0], 1, TextExport::Unmerged).unwrap(), + [1.0] + ); + assert_eq!( + letter_logits(&letters, &[1.0, 1.0, 1.0], 2, TextExport::Merged).unwrap(), [1.0, 16_777_216.0] ); - assert_eq!(letter_logits(&letters, &[1.0, 1.0, 1.0], 1).unwrap(), [1.0]); +} + +#[test] +fn export_profiles_keep_the_four_observed_near_tie_orders_separate() { + use serde_json::{Value, json}; + use tokenizers::{Tokenizer, models::wordlevel::WordLevel}; + + let fixture: Value = + serde_json::from_str(include_str!("fixtures/legacy_merged_readout.json")).unwrap(); + // Tokenization is incidental here: run the real response finishing on the + // frozen request's option order, but use synthetic BF16 projection inputs. + // These vectors reproduce the readout boundary, not the model's hidden state. + let tokenizer = Tokenizer::new( + WordLevel::builder() + .vocab([("[UNK]".into(), 0)].into_iter().collect()) + .unk_token("[UNK]".into()) + .build() + .unwrap(), + ); + let processor = crate::processing::Processor { tokenizer }; + let cases = fixture["cases"].as_array().unwrap(); + assert_eq!(cases.len(), 4); + for case in cases { + let keys: Vec<_> = case["request"]["questions"]["q"]["criteria"] + .as_object() + .unwrap() + .keys() + .collect(); + let legacy = case["expected_response"]["answers"]["q"]["choice"] + .as_str() + .unwrap(); + let rounded = case["rounded_choice"].as_str().unwrap(); + let legacy_index = keys.iter().position(|key| key.as_str() == legacy).unwrap(); + let rounded_index = keys.iter().position(|key| key.as_str() == rounded).unwrap(); + assert!(rounded_index < legacy_index); + let mut letters = [10.0, 0.0].repeat(keys.len()); + letters[rounded_index * 2..rounded_index * 2 + 2].copy_from_slice(&[20.0, 0.03125]); + letters[legacy_index * 2..legacy_index * 2 + 2].copy_from_slice(&[20.0, 0.046875]); + // Alternating profiles must not leak a readout policy between exports. + for export in [TextExport::Merged, TextExport::Unmerged, TextExport::Merged] { + let prepared = processor + .prepare(&serde_json::to_vec(&case["request"]).unwrap()) + .ok() + .unwrap(); + let logits = letter_logits(&letters, &[1.0, 1.0], keys.len(), export).unwrap(); + let response = prepared.context.finish(vec![logits]).unwrap(); + let choice = match export { + TextExport::Merged => legacy, + TextExport::Unmerged => rounded, + }; + assert_eq!( + response["answers"]["q"]["choice"], + json!(choice), + "{}", + case["id"] + ); + let probabilities = response["answers"]["q"]["probabilities"] + .as_object() + .unwrap(); + assert_eq!(probabilities.keys().collect::>(), keys); + match export { + TextExport::Merged => { + assert!(probabilities[legacy].as_f64() > probabilities[rounded].as_f64()) + } + TextExport::Unmerged => assert_eq!(probabilities[legacy], probabilities[rounded]), + } + } + } +} + +#[test] +fn finite_fp32_legacy_scores_are_not_overflowed_by_bf16_conversion() { + assert_eq!( + letter_logits(&[f32::MAX], &[1.0], 1, TextExport::Merged).unwrap(), + [f32::MAX] + ); + assert!(letter_logits(&[f32::MAX], &[1.0], 1, TextExport::Unmerged).is_err()); } #[test] fn letter_projection_rejects_nonfinite_scores_before_later_prompts() { - assert!(letter_logits(&[f32::MAX], &[2.0], 1).is_err()); - assert!(letter_logits(&[f32::NAN], &[1.0], 1).is_err()); + for export in [TextExport::Merged, TextExport::Unmerged] { + assert!(letter_logits(&[f32::MAX], &[2.0], 1, export).is_err()); + assert!(letter_logits(&[f32::NAN], &[1.0], 1, export).is_err()); + } } diff --git a/tests/cua_s1/legacy_merged_gpu.rs b/tests/cua_s1/legacy_merged_gpu.rs new file mode 100644 index 00000000..833170c1 --- /dev/null +++ b/tests/cua_s1/legacy_merged_gpu.rs @@ -0,0 +1,110 @@ +//! Frozen A800 compatibility oracle, separate from unmerged/reference parity. +use super::*; +use serde_json::Value; +use sha2::{Digest, Sha256}; +use std::io::Read; + +#[tokio::test] +#[ignore = "requires CUDA and the frozen A800 legacy merged artifact; run serially at Graph 0/1"] +async fn legacy_merged_upgrade_preserves_four_decisions() { + let fixture: Value = + serde_json::from_str(include_str!("fixtures/legacy_merged_readout.json")).unwrap(); + let model = std::path::PathBuf::from( + std::env::var_os("CUA_S1_MERGED_TEST_MODEL").expect("CUA_S1_MERGED_TEST_MODEL"), + ); + let library = + std::path::PathBuf::from(std::env::var_os("CUA_S1_CUDA_LIB").expect("CUA_S1_CUDA_LIB")); + assert!(matches!( + crate::provenance::text_export(&model).unwrap(), + crate::provenance::TextExport::Merged + )); + // Freeze weights/tokenizer as well as requests; another merged checkpoint + // must not accidentally qualify against this A800 compatibility oracle. + for (name, expected) in fixture["artifact_files"].as_object().unwrap() { + let path = model.join(name); + assert_eq!( + std::fs::metadata(&path).unwrap().len(), + expected["size"].as_u64().unwrap(), + "{name}" + ); + let mut file = std::fs::File::open(&path).unwrap(); + let mut hash = Sha256::new(); + let mut buffer = vec![0; 64 * 1024]; + loop { + let n = file.read(&mut buffer).unwrap(); + if n == 0 { + break; + } + hash.update(&buffer[..n]); + } + assert_eq!( + format!("{:x}", hash.finalize()), + expected["sha256"].as_str().unwrap(), + "{name}" + ); + } + let engine = Engine::load(&model, &library).await.unwrap(); + let cases = fixture["cases"].as_array().unwrap(); + assert_eq!(cases.len(), 4); + let mut first_pass = Vec::new(); + for pass in 0..2 { + for (index, case) in cases.iter().enumerate() { + let prepared = engine + .processor + .prepare(&serde_json::to_vec(&case["request"]).unwrap()) + .ok() + .unwrap(); + let rows = engine + .executor + .execute(&engine.scheduler, prepared.inputs) + .await + .unwrap(); + let response = prepared.context.finish(rows).unwrap(); + let expected = &case["expected_response"]; + assert_eq!(response["model"], expected["model"]); + assert_eq!(response["usage"], expected["usage"]); + let actual = &response["answers"]["q"]; + let baseline = &expected["answers"]["q"]; + assert_eq!(actual["type"], baseline["type"]); + assert_eq!( + actual["choice"], baseline["choice"], + "{} pass {pass}", + case["id"] + ); + let actual_probs = actual["probabilities"].as_object().unwrap(); + let baseline_probs = baseline["probabilities"].as_object().unwrap(); + assert_eq!( + actual_probs.keys().collect::>(), + baseline_probs.keys().collect::>() + ); + let max_error = baseline_probs + .iter() + .map(|(key, value)| { + let p = actual_probs[key].as_f64().unwrap(); + assert!(p.is_finite() && (0.0..=1.0).contains(&p)); + (p - value.as_f64().unwrap()).abs() + }) + .fold(0.0_f64, f64::max); + assert!( + max_error <= 0.02, + "{} pass {pass}: drift {max_error}", + case["id"] + ); + assert!( + (actual["confidence"].as_f64().unwrap() - baseline["confidence"].as_f64().unwrap()) + .abs() + <= 0.02 + ); + let bytes = serde_json::to_vec(&response).unwrap(); + if pass == 0 { + first_pass.push(bytes); + } else { + assert_eq!(bytes, first_pass[index], "{} repeat drift", case["id"]); + } + eprintln!( + "legacy merged pass={pass} id={} choice={} max_probability_error={max_error}", + case["id"], actual["choice"] + ); + } + } +} From 16bf31b802257cf17f6921210b6094f9b0e43cf3 Mon Sep 17 00:00:00 2001 From: levius <2114377220@qq.com> Date: Thu, 15 Oct 2026 11:10:27 +0800 Subject: [PATCH 19/21] docs(cua-s1): record merged readout repair qualification --- .../cua-s1-text-parity-20261013/README.md | 50 ++++++++++++++----- recipe/cua_s1/native.md | 12 +++-- src/models/cua_s1/README.md | 6 ++- 3 files changed, 48 insertions(+), 20 deletions(-) diff --git a/docs/benchmarks/cua-s1-text-parity-20261013/README.md b/docs/benchmarks/cua-s1-text-parity-20261013/README.md index 21a8b3e6..8a639512 100644 --- a/docs/benchmarks/cua-s1-text-parity-20261013/README.md +++ b/docs/benchmarks/cua-s1-text-parity-20261013/README.md @@ -17,12 +17,12 @@ The 2026-10-15 parent synchronization used #118 at `3be46fa270cb74c33d4db4ffd761405d0522018a`, which inherits [#117](https://github.com/ThinkFlowLab/system1-omni/pull/117) and its merged resident-feature increment #135. Rebuild the library and all consumers together -for ABI 8. This parent synchronization has CPU/CI coverage; current integrated -GPU qualification remains unverified. Decider modes and the separate #127 cache +for ABI 8. The later ABI-8 source `7293c71c` and legacy-readout repair `f212c56f` +have separate source-pinned A800 campaigns described below. Decider modes and the separate #127 cache increment are not included. The frozen corpus comes from #141; merging #141 is not a runtime dependency. -The GPU results below were measured on cumulative candidate +The historical cumulative results in the Results section were measured on candidate `76af07996f60905082b8d48199d4a7003f33f4f6`, whose parent is `18e1a5a6f68b9d11d3e11c7ea7b939af87272f53`. That integrated build used ABI 8. The separate independent qualification at `27f77e49` used ABI 6 with additive @@ -118,16 +118,40 @@ vectors are not captured checkpoint activations. The maintained four-request device test uses the actual frozen inputs and parent responses; run it as in the [native recipe](../../../recipe/cua_s1/native.md). -Fresh GPU execution of the compatibility fix is **pending**: the earlier server -was shut down after preserving the published evidence. In particular, restored -97/139 correctness and zero upgrade flips have not been measured for this new -source. Requalify the full corpus on pinned A800: two passes each for legacy -parent and repaired merged eager/Graph, requiring zero parent/candidate decision -changes and probability error at most 0.02; retain exact-byte differences too. -Repeat unmerged eager/Graph/reference checks at the original 0.02/zero-flip gates. -The previous candidate's unmerged results were 97/139 in all three modes, maximum -probability difference 5.960464477539063e-8 and zero flips. They remain tied to -`7293c71c`; they are not a new-device pass for this follow-up. +The [repair GPU campaign](https://github.com/Levius-Fubuki/system1-omni/releases/tag/cua143-merged-readout-f212c56f-20261015) +tested source `f212c56f23ac750ebfa38f4c72e9fcdc914bf1cd` on pinned A800/ABI 8. +First, two four-request passes on rejected source `7293c71c` reproduced all four +changes. The repaired four-request Engine regression then passed twice per +Graph setting, with zero probability error. Graph 1 logged 6 captures, 2 replays +and no fallback; the test verified the nine artifact files' SHA-256 before loading. + +| Fresh full-corpus mode | Passes × requests | Comparator | Maximum probability difference | Decision changes | Correct | +| --- | --- | --- | --- | --- | --- | +| Legacy parent | 2 × 139 | Parent | 0 | 0 | 97/139 | +| Repaired merged eager | 2 × 139 | Parent | 0 | 0 | 97/139 | +| Repaired merged Graph | 2 × 139 | Parent | 0 | 0 | 97/139 | +| Repaired unmerged eager | 2 × 139 | Python reference | 5.960464477539063e-8 | 0 | 97/139 | +| Repaired unmerged Graph | 2 × 139 | Python reference | 5.960464477539063e-8 | 0 | 97/139 | +| Python reference | 2 × 139 | Reference | 0 | 0 | 97/139 | + +Every repaired merged response body and content type matched the parent byte +for byte. All modes repeated identical response bytes; unmerged eager/Graph +bodies also matched each other. Each full-corpus Graph worker logged 207 +captures and 73 replays, with no fallback. The original 0.02/zero-flip gates +were retained and legacy compatibility additionally required byte identity. +Counts are 1,668 full-corpus HTTP measurements, 8 rejected-source reproductions, +7 excluded feasibility requests and 16 separate direct Engine predictions. +Initial bootstrap rejected a packaging-version mismatch before any GPU request: +Torch metadata is `2.14.0` while runtime is `2.14.0+cu130`; both fields and CUDA +runtime `13.0` were then checked explicitly. The failure is preserved. No source +or tolerance changed during the campaign. Raw evidence was verified locally +before the server was shut down. Later documentation-only updates preserve all +non-Markdown source and fixture bytes from the tested repair commit. + +Legacy compatibility remains distinct from reference parity: restoring the +original merged behavior does not remove its known approximation relative to +unmerged FP32 LoRA. These results do not establish full 27B, L20X/Hopper, +FLA-enabled, performance or general GUI task-quality qualification. ## Results diff --git a/recipe/cua_s1/native.md b/recipe/cua_s1/native.md index e4421d42..75e714f4 100644 --- a/recipe/cua_s1/native.md +++ b/recipe/cua_s1/native.md @@ -5,8 +5,9 @@ The unmerged precision profile has source-pinned A800 80 GB (sm_80), CUDA 13.0 evidence at ABI-8 source `7293c71cfb738504a30ff033ae3567f3368333c1`, plus earlier cumulative ABI-8 and independent ABI-6 campaigns linked in [PR #143](https://github.com/ThinkFlowLab/system1-omni/pull/143). -The current branch uses ABI 8. The later legacy-readout compatibility fix has -CPU regression coverage; fresh GPU validation of that follow-up is pending. +The current branch uses ABI 8. The legacy-readout compatibility fix was separately +GPU-tested at `f212c56f23ac750ebfa38f4c72e9fcdc914bf1cd` on the same pinned A800. +See the [source-pinned repair evidence](https://github.com/Levius-Fubuki/system1-omni/releases/tag/cua143-merged-readout-f212c56f-20261015). The earlier merged path was checked on an RTX 6000 Ada (sm_89) with CUDA 13.2. Run the commands from the repository root. Pass your GPU's compute capability to `build.sh` (89 for Ada, 80 for A100, 90 for H100); the worker finds `libqwen3_5_cuda.so` next to its executable, or at `CUA_S1_CUDA_LIB`: @@ -52,9 +53,10 @@ backbone execution path and retain FP32 letter logits after FP64 accumulation. The BF16 readout boundary is restricted to the validated unmerged export profile, not a request flag or process-global setting. An earlier #143 build applied it to merged exports too, creating four decision changes and reducing frozen-corpus -correctness from 97/139 to 96/139. The follow-up restores legacy readout policy; -that restoration still needs fresh device validation. The A800 unmerged parity -results are scoped to their recorded source and artifact. +correctness from 97/139 to 96/139. At repaired source `f212c56f`, merged eager and +Graph each matched every parent response byte in two 139-request passes, with +zero upgrade flips and 97/139 correct. Unmerged eager/Graph retained zero flips +against the pinned reference. Results remain scoped to that source and artifact. Merging the adapter, TF32/BF16 GDN intermediates, and different GEMM reduction policies can change decisions on close or tied options. Re-export with the command above to use the precision profile. Rebuild worker and library diff --git a/src/models/cua_s1/README.md b/src/models/cua_s1/README.md index 08e835bd..0f7904d3 100644 --- a/src/models/cua_s1/README.md +++ b/src/models/cua_s1/README.md @@ -139,8 +139,10 @@ The old merged export remains an approximate execution mode. Its legacy FP64-accumulated, FP32 letter-logit readout is preserved; BF16 rounding is selected only by `cua-s1-text-unmerged/1`. Applying that rounding to merged exports at `7293c71c` created four ordered-tie decision changes in a same-artifact A800 -upgrade comparison. The compatibility fix is covered by CPU red/green projection -and response tests; a frozen four-request GPU regression is also provided. +upgrade comparison. The compatibility fix has CPU red/green projection and +response coverage. At source `f212c56f`, the four-request device regression passed +in eager/Graph, and both modes matched every legacy parent response byte in two +139-request passes. Unmerged/reference parity remained within its original gates. See the [numerical report](../../../docs/benchmarks/cua-s1-text-parity-20261013/README.md#legacy-merged-upgrade-regression) for the four cases, source pins and device-validation limits. Restoring merged compatibility does not make merged weights equivalent to the unmerged reference. From 3d150b0117ff06e28eafde463cbe87754b4e52c6 Mon Sep 17 00:00:00 2001 From: levius <2114377220@qq.com> Date: Thu, 15 Oct 2026 11:17:18 +0800 Subject: [PATCH 20/21] docs(cua-s1): clarify native softmax accumulation precision --- docs/architecture.md | 3 ++- recipe/cua_s1/native.md | 4 +++- src/models/cua_s1/README.md | 5 +++-- 3 files changed, 8 insertions(+), 4 deletions(-) diff --git a/docs/architecture.md b/docs/architecture.md index 5677b13a..b354aebf 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -58,7 +58,8 @@ and reports unavailable health thereafter. Qwen executors own loaded models and CPU head weights with FP64 accumulation. Cua-S1's unmerged text profile rounds answer-letter logits to the tied `lm_head` -BF16 output dtype before FP32 softmax. Legacy merged text and screenshot +BF16 output dtype before response finishing. Cua-S1 finishing normalizes in +FP64 and returns FP32 probabilities. Legacy merged text and screenshot readouts retain FP32 logits; learned scalar heads retain their FP32 rounding and bias order. Finishing checks output cardinality before reconstruction. HTTP validation, error status/body conventions, and real diff --git a/recipe/cua_s1/native.md b/recipe/cua_s1/native.md index 75e714f4..0737bca7 100644 --- a/recipe/cua_s1/native.md +++ b/recipe/cua_s1/native.md @@ -43,7 +43,9 @@ Each question runs one forward pass over its prompt. The `cua-s1-text-unmerged/1 marker selects framework-compatible BF16 normalization and cuBLASLt GEMM, FP32 LoRA branches, and FP32 chunked Gated DeltaNet. The GDN cumulative decay uses the pinned Torch scan order; state is reset for every prompt. The letter -projection rounds its result to BF16 before FP32 softmax, matching `lm_head`. +projection rounds its result to the BF16 `lm_head` output dtype before the existing +per-question softmax. Native finishing normalizes in FP64 and returns FP32 +probabilities. The marker records scale 2 and the SHA-256 of the unchanged adapter; incompatible markers or altered adapter bytes fail before CUDA loading. Prefix continuation is unsupported in this precision mode. diff --git a/src/models/cua_s1/README.md b/src/models/cua_s1/README.md index 0f7904d3..72945c23 100644 --- a/src/models/cua_s1/README.md +++ b/src/models/cua_s1/README.md @@ -13,8 +13,9 @@ option count per question, plus a response context for identity/order and usage. The executor projects the letter rows with FP64 accumulation and returns FP32 letter logits. Only the unmerged text profile first rounds to the BF16 `lm_head` output dtype; legacy merged text and screenshot readouts retain FP32. The text -export policy is validated once and owned by each executor. Response finishing owns per-question -softmax and the choice/confidence interpretation. The engine owns a +export policy is validated once and owned by each executor. Response finishing +owns per-question softmax (FP64 normalization returning FP32 probabilities) and +the choice/confidence interpretation. The engine owns a [shared serial scheduler](../../runtime/README.md) that admits each question's forward before blocking dispatch. Execution remains one prompt per question with a model-state mutex and CPU answer-letter projection after CUDA prefill, From 388570adb992e8872d93146cdf44dbe0250a5e77 Mon Sep 17 00:00:00 2001 From: levius <2114377220@qq.com> Date: Wed, 21 Oct 2026 22:45:02 +0800 Subject: [PATCH 21/21] docs(jemm): date ABI8 qualification and integration boundaries --- docs/architecture.md | 10 ++++- .../jemm-reference-20261011/README.md | 5 +++ recipe/jemm/README.md | 8 ++-- src/models/jemm/README.md | 38 +++++++++++++++---- 4 files changed, 49 insertions(+), 12 deletions(-) diff --git a/docs/architecture.md b/docs/architecture.md index b354aebf..2846528b 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -21,8 +21,13 @@ coordinate independent processors and executors through per loaded executor. Shared processing orchestration, batch budgets, compatibility grouping and dynamic batching are planned. -Qwen workers compute their decision heads on the CPU after downloading the final -hidden state. Laya computes its scorer and action head on CUDA. Its fixed-shape +Cua-S1 and Open-Jev compute their decision heads on the CPU after downloading +the final hidden state. JEMM uses the shared Qwen executor with separate unmerged +FP32 adapters and framework-reference arithmetic. It computes selected BF16 +label-head logits on CUDA, then calibrates typed answers on the CPU. JEMM admits +one complete text or 1–4-image request and synchronizes both model streams before +releasing its resources. See its [model contract](../src/models/jemm/README.md). +Laya computes its scorer and action head on CUDA. Its fixed-shape Graph captures Encoder/Decision; gather, scorer/action head and synchronized readback remain outside capture. Native Metal remains planned; Python workers retain their documented reference/serving roles. @@ -40,6 +45,7 @@ and candidate identity, usage, and response metadata outside the executor. | Cua-S1 | One unpadded token-ID vector and option count per question, in request order. | One FP32 answer-letter logit vector per question. | Per-question softmax, choice/confidence, ordered answers, and token usage. | | Open-Jev | Token-ID vectors grouped by question, then independent candidate, in request order. | One FP32 learned scalar per candidate in the same grouping. | Add the `noul` false logit of zero, calibrate across each complete question, and restore typed answers, usage, and metadata. | | Laya | One padded request: token IDs, true lengths, question types and ordered option markers; at most 16 questions, 512 tokens per row and 2048 markers. | Per-question FP32 option logits and two action logits copied back after GPU heads. | Calibrate and decode ordered `choice`, `score` and `noul` answers, usage and metadata. | +| JEMM | Ordered token IDs, three-axis positions, candidate counts and optional shared image patches for one complete request. | Per-question selected BF16 label logits copied back as FP32. | Apply checksum-covered text/image calibration and reconstruct ordered `choice`, `score` and `noul` answers and usage. | Cua-S1 and Open-Jev-9B input collections are serial work. For Open-Jev-27B-v1.1, Open-Jev's model-specific batch adapter packs up to 16 independent candidates and diff --git a/docs/benchmarks/jemm-reference-20261011/README.md b/docs/benchmarks/jemm-reference-20261011/README.md index 4a16bb59..31b763a2 100644 --- a/docs/benchmarks/jemm-reference-20261011/README.md +++ b/docs/benchmarks/jemm-reference-20261011/README.md @@ -1,5 +1,10 @@ # JEMM 1–4-image numerical regression +This page records the **2026-10-11 ABI 6 campaign** and its original corpus. +The later [2026-10-21 ABI 8 qualification](../../../src/models/jemm/README.md#source-pinned-abi-8-qualification-2026-10-21) +has a separate source pin, extended corpus and combined multimodal-language +Graph evidence. Its results do not change the historical records below. + The optional reference-numerics path passes the frozen 16-request / 25-question corpus on one NVIDIA A800 80 GB. Maximum probability drift is **0.0166172279642417**. The unchanged gates are probability/confidence/noul 0.02, diff --git a/recipe/jemm/README.md b/recipe/jemm/README.md index 10fa1f22..306801e8 100644 --- a/recipe/jemm/README.md +++ b/recipe/jemm/README.md @@ -71,9 +71,11 @@ curl --fail http://127.0.0.1:8080/v1/systemone \ ``` Optional `images` contains up to four base64 PNG/JPEG/WebP strings; images precede -text and are shared across questions. The frozen 16-request / 25-question corpus -passes the original numerical gates, including 2/3/4-image cases. See the -[validation scope and evidence](../../docs/benchmarks/jemm-reference-20261011/README.md). +text and are shared across questions. The source-pinned 2026-10-21 ABI 8 campaign +covered two passes of 28 requests / 49 questions in reference, eager and combined +Graph modes, including the original 16/25 corpus. See the +[dated validation scope and evidence](../../src/models/jemm/README.md#validation) +and the [historical corpus](../../docs/benchmarks/jemm-reference-20261011/README.md). Native CPU/Metal execution is unsupported. ## Checks and reference diff --git a/src/models/jemm/README.md b/src/models/jemm/README.md index 5970d60f..b18a4e84 100644 --- a/src/models/jemm/README.md +++ b/src/models/jemm/README.md @@ -100,15 +100,37 @@ slots, not weights or GEMM plan memory. Startup's image warmup can capture its o shape when enabled; it does not prewarm every user grid. Graph replay does not establish lower latency on every workload. -The archived v2 HTTP campaign below used text-language and vision graphs with -multimodal language eager. The combined v2 reference path and multimodal language -replay has not yet been rerun on a GPU; earlier multimodal replay evidence used -the premerged v1 path. CPU checks compile the graph regressions but cannot prove -device replay equality for this integration. +The 2026-10-11 v2 HTTP campaign used text-language and vision graphs with +multimodal language eager. The separate 2026-10-21 ABI 8 campaign below exercised +combined text, multimodal-language and vision replay at its exact source pin. +CPU checks compile the graph regressions but do not execute device replay. ## Validation -The unmerged FP32-adapter v2 path passes the frozen **16 requests / 25 questions** +### Source-pinned ABI 8 qualification — 2026-10-21 + +The [public report and raw evidence](https://github.com/Levius-Fubuki/system1-omni/releases/tag/jemm-abi8-5bdd625b-20261021) +qualify source `5bdd625b4c27efb02f43a1f71150e1a7d7c0f071` on one A800 80 GB +PCIe (`sm_80`), using a cuDNN-enabled ABI 8 library and a fresh pinned FLA +reference. Reference, native eager and combined Graph each completed two passes +of **28 requests / 49 questions**, including the original 16/25 subset. +Maximum probability drift was **0.0166172279642417**, below 0.02; maximum Score +drift was **7.638334409421077e-14**, below 0.1, with no decision changes. +Eager/Graph semantic responses matched after excluding only `usage.latency_ms`. + +The campaign covered text and 1–4 images, changed pixels/geometry/order, +1240-token multimodal and 2282-token text scratch growth, and invalid-input +recovery. All three Graph paths captured and replayed without fallback; +11 reference operators and one selected-head device test passed. The release +includes raw responses, logs, failed bootstrap attempts, source hashes and an +offline audit. It does not qualify resident-device image embeddings, the +separate cache or Cua-S1 precision increments, broader task accuracy, other +hardware, latency, throughput or peak memory. Later integration checks must +retain this source boundary rather than claim a fresh full-model GPU run. + +### Historical v2 qualification — 2026-10-11 + +The unmerged FP32-adapter v2 path passed the frozen **16 requests / 25 questions** covering text and 1–4 images on A800. Maximum probability drift is **0.0166172279642417**, under the unchanged 0.02 gate. Score tolerance remains 0.1 and winner agreement is required at reference margins of at least 0.05; @@ -134,4 +156,6 @@ resource boundary, response math, image decoding/positions, tokenizer and token budgets, export integrity, BF16 head padding, error-envelope bounds, warmup image decoding and synchronized retirement. The real CUDA head test is ignored in CPU CI. CPU tests do not prove full-model parity or execute the startup image forward. -No fresh GPU campaign or performance claim accompanies the review fixes. +The calibration-loading review fixes are covered by host/export tests; the GPU +campaigns above retain their own source pins and do not constitute a fresh +device run of those fixes. No performance claim accompanies them.