diff --git a/README.md b/README.md index 07815a5c..b154cdcf 100644 --- a/README.md +++ b/README.md @@ -172,7 +172,7 @@ pinned runtime for text state: | Cua-S1 4B 0.2 (`text` adapter) | [Python worker](recipe/cua_s1/text.md); [native worker](recipe/cua_s1/native.md), CUDA; unmerged profile validated on the cumulative A800 sm_80 build, earlier merged path on sm_89 | | Cua-S1 4B 0.2 (`multimodal` adapter) | [Python CUDA reference](src/frontend/cua_s1.py) and [native Rust/CUDA screenshot worker](recipe/cua_s1/native_screenshot.md); one PNG/JPEG image, `choice`; scoped RTX 4090 validation | | JEMM (Qwen3.8-27B) | [Native Rust/CUDA worker](recipe/jemm/README.md); [text and 1–4-image frozen-corpus parity on A800](src/models/jemm/README.md#validation) | -| Valen-Preview-0923 | [Python reference worker](recipe/valen/reference.md); text state or one inline PNG/JPEG image, `choice`; author-run on RTX 5060 Laptop GPU (sm_120, WSL2), owner: community | +| Valen-Preview-0923 | [Python reference worker](recipe/valen/reference.md); text state or one inline PNG/JPEG image, `choice`/`noul`/`score`; author-run on RTX 5060 Laptop GPU (sm_120, WSL2), owner: community | | Open-Jev-27B-v1.1 | [Native Rust/CUDA worker](recipe/open_jev/native.md); eager independent text candidates; [H200 validation](recipe/open_jev/validation.md) | | Open-Jev-9B | The same [native Rust/CUDA worker](recipe/open_jev/native.md); eager independent text candidates; [reference comparison on sm_89](recipe/open_jev/validation-9b.md) | | autotrust/JEV-27B-VL | [Experimental Rust/CUDA worker](recipe/jev_vl/README.md); single-question text and offline-preencoded image inputs; [bounded H800 validation and limits](recipe/jev_vl/validation.md) | diff --git a/docs/supported-models.md b/docs/supported-models.md index aecd6670..9eaebb8b 100644 --- a/docs/supported-models.md +++ b/docs/supported-models.md @@ -16,7 +16,7 @@ Models that are being added are also tracked in issues labeled [new model](https | Cua-S1 4B 0.2, `text` adapter | [Native Rust worker](../recipe/cua_s1/native.md) on the [Qwen3.5 CUDA kernels](../src/backends/cuda/qwen3_5/README.md) | Not supported | Validated on compute capability 8.9 ([#19](https://github.com/ThinkFlowLab/system1-omni/pull/19), [#52](https://github.com/ThinkFlowLab/system1-omni/pull/52)) | Not supported | Compute capability 8.0 or newer, the CUDA toolkit to build, weights merged with `export_text_merged.py` | | Cua-S1 4B 0.2, `multimodal` adapter | Reference worker on Transformers and PEFT, [`src/frontend/cua_s1.py`](../src/frontend/cua_s1.py); no recipe yet | Not supported | Validated ([#17](https://github.com/ThinkFlowLab/system1-omni/pull/17), [#18](https://github.com/ThinkFlowLab/system1-omni/pull/18)) | Not supported | The state is one PNG or JPEG image; upstream's `weights.lock.json` next to the base weights | | Cua-S1 4B 0.2, `multimodal` adapter | [Native Rust/CUDA screenshot worker](../recipe/cua_s1/native_screenshot.md) | Preparation checks only; no CPU inference | RTX 4090 / sm_89, native vision and language Graph regressions; [scoped evidence and controls](../recipe/cua_s1/native_vision_graph.md) | Not supported | Verified BF16 language export, original vision checkpoint and FP32 vision adapter, rebuilt ABI 9 CUDA library; one PNG/JPEG and `choice` | -| Valen-Preview-0923 | [Python reference worker](../recipe/valen/reference.md); text state or one inline PNG/JPEG image; `choice` questions | Not supported | Author-run on RTX 5060 Laptop (sm_120, WSL2): one text and one 2×2 PNG request; input and readout parity ≤1e-6 against reference (model outputs not compared). Owner: community. | Not supported | Python 3.12, pinned Valen source/checkpoint, Qwen3.5-2B base, BF16 CUDA | +| Valen-Preview-0923 | [Python reference worker](../recipe/valen/reference.md); text state or one inline PNG/JPEG image; `choice`, `noul`, and `score` questions | Not supported | Author-run on RTX 5060 Laptop (sm_120, WSL2): one text and one 2×2 PNG request; input and readout parity ≤1e-6 against reference (model outputs not compared); `noul`/`score` parity ≤1e-6 with exact legend and token agreement on the recipe's text fixtures. Owner: community. | Not supported | Python 3.12, pinned Valen source/checkpoint, Qwen3.5-2B base, BF16 CUDA | | Open-Jev-27B-v1.1 | [Native Rust/CUDA worker](../recipe/open_jev/native.md) on the shared Qwen3.5/3.8 executor | Not supported | Validated on H200 (sm_90) for the [74 single-candidate workload](../recipe/open_jev/validation.md) | Not supported | Compute capability 8.0 or newer, CUDA toolkit to build, exported merged weights and trained head | | Open-Jev-9B | The same [native Rust/CUDA worker](../recipe/open_jev/native.md) | Not supported | Validated on compute capability 8.9 against the reference for [253 requests](../recipe/open_jev/validation-9b.md) | Not supported | Compute capability 8.0 or newer, CUDA toolkit to build, exported merged weights and trained head | | JEMM / Qwen3.8-27B | [Native Rust/CUDA worker](../recipe/jemm/README.md) | Host checks only; no CPU inference | A800 (`sm_80`) [text and 1–4-image frozen corpus](../src/models/jemm/README.md#validation) | Not supported | Pinned `jemm-native/2` export with BF16 base and unmerged FP32 LoRA, selected untied LM head, cuDNN 9 and additive reference CUDA symbols | diff --git a/mkdocs.yml b/mkdocs.yml index 70302d5c..9d99186f 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -67,6 +67,7 @@ nav: - Cua-S1 native text worker: recipe/cua_s1/native.md - Cua-S1 text numerical parity: docs/benchmarks/cua-s1-text-parity-20261013/README.md - Valen reference worker: recipe/valen/reference.md + - Valen checkpoint export: recipe/valen/export.md - Cua-S1 native screenshots: recipe/cua_s1/native_screenshot.md - Cua-S1 vision CUDA Graph: recipe/cua_s1/native_vision_graph.md - Open-Jev native text worker: recipe/open_jev/native.md diff --git a/recipe/README.md b/recipe/README.md index a1a77b0f..b3404282 100644 --- a/recipe/README.md +++ b/recipe/README.md @@ -12,7 +12,11 @@ For a first real decision, follow the [complete CPU walkthrough](../docs/getting - [Cua-S1 4B 0.2 text worker](cua_s1/text.md): download the pinned weights, start the worker and connect the Rust frontend. - [Valen-Preview-0923 reference worker](valen/reference.md): prepare the pinned - Preview checkpoint and Qwen base, start the text-or-image `choice` worker and verify the frontend. + Preview checkpoint and Qwen base, start the text-or-image `choice`/`noul`/`score` + worker and verify the frontend. +- [Valen-Preview-0923 checkpoint export](valen/export.md): export the pinned + checkpoint and base to a manifest-described safetensors artifact set and run + the export regression. - [Cua-S1 4B 0.2 native text worker](cua_s1/native.md): build the CUDA library and the Rust worker, export BF16 weights with the original FP32 adapter and start the worker. diff --git a/recipe/valen/compare_reference.py b/recipe/valen/compare_reference.py index 3e355924..d4cee559 100644 --- a/recipe/valen/compare_reference.py +++ b/recipe/valen/compare_reference.py @@ -4,17 +4,22 @@ Runs each request through two paths with the same pinned artifacts: - reference: a Valen-native record (a plain-string text state, or a local - image with a SHA-256 asset) compiled by the pinned ``Compiler`` and decoded - with a plain softmax plus Valen's confidence formula; + image with a SHA-256 asset) compiled by the pinned ``Compiler``; ``choice`` + cases are decoded with a plain softmax plus Valen's confidence formula, + while ``noul`` and ``score`` cases are decoded by the pinned reference + ``valen.evaluation.inference.answer`` module itself; - worker: the production ``frontend.valen.decide`` pipeline (protocol, preprocessing, executor, postprocessing). Both paths share one loaded model instance, so the comparison isolates input construction, candidate ordering, decision-head readout and probability normalization instead of checkpoint loading. The declared criteria are exact -decision agreement, equal token accounting and a probability tolerance of -1e-6 (``--tolerance``). Requires the pinned Valen source, the Preview -checkpoint, the Qwen3.5-2B base and a working torch device; see +discrete agreement (candidate order, the requested answer type, the chosen +key, the score legend), equal token accounting and a tolerance of 1e-6 +(``--tolerance``) on probabilities, confidences and the scalar answer fields +(``noul`` P(true), ``score`` expected level), which the two softmax +implementations never match bit-exactly. Requires the pinned Valen source, +the Preview checkpoint, the Qwen3.5-2B base and a working torch device; see recipe/valen/reference.md. """ @@ -24,13 +29,27 @@ import base64 import hashlib import json +import math +import subprocess import sys +from importlib.machinery import BYTECODE_SUFFIXES, EXTENSION_SUFFIXES, SOURCE_SUFFIXES from pathlib import Path from tempfile import TemporaryDirectory HERE = Path(__file__).resolve().parent PROBABILITY_TOLERANCE = 1e-6 +# A sourceless bytecode or extension module imports exactly like its .py +# source would, so every importable suffix is swept, not just source files. +IMPORTABLE_SUFFIXES = tuple( + suffix.lower() + for suffix in sorted( + set(SOURCE_SUFFIXES) | set(BYTECODE_SUFFIXES) | set(EXTENSION_SUFFIXES), + key=len, + reverse=True, + ) +) + DEFAULT_TEXT_BODY = { "model": "valen-preview-0923", "state": "The card was charged twice for one order.", @@ -51,7 +70,11 @@ def parse_args(argv=None): parser.add_argument("--base", required=True, help="Qwen3.5-2B base directory") parser.add_argument("--device", default="cuda") parser.add_argument("--dtype", choices=("bf16", "fp32"), default="bf16") - parser.add_argument("--case", choices=("image", "text", "both"), default="both") + parser.add_argument( + "--case", + choices=("image", "text", "noul", "score", "both", "all"), + default="both", + ) parser.add_argument( "--fixture", default=str(HERE / "example-request.json"), @@ -62,6 +85,16 @@ def parse_args(argv=None): default=None, help="text-case request body JSON file (default: the recipe text example)", ) + parser.add_argument( + "--noul-body", + default=str(HERE / "example-noul-request.json"), + help="noul-case request body JSON file (default: the recipe noul example)", + ) + parser.add_argument( + "--score-body", + default=str(HERE / "example-score-request.json"), + help="score-case request body JSON file (default: the recipe score example)", + ) parser.add_argument("--tolerance", type=float, default=PROBABILITY_TOLERANCE) return parser.parse_args(argv) @@ -78,20 +111,40 @@ def decode_image_data_url(url): return base64.b64decode(encoded, validate=True), expected[prefix] -def reference_questions(body): - """Rebuild the Valen question mapping straight from the fixture body.""" +def reference_questions(body, special_tokens=None): + """Rebuild the Valen question mapping straight from the fixture body. - return { - name: { - "type": "choice", - "instructions": question.get("instructions") or "", - "criteria": { - key: (label if label is not None else key) - for key, label in question["criteria"].items() - }, - } - for name, question in body["questions"].items() - } + The questions go through the worker's own ``parse_request`` + + ``question_record`` path, so the reference record cannot drift from the + one the worker compiles: custom ``noul`` wording becomes the same + ``true``/``false`` choice record, and unspecified sides get the same + fixed reference wording. + """ + + from models.valen.preprocess import question_record + from models.valen.protocol import DEFAULT_SPECIAL_TOKENS, parse_request + + request = parse_request( + body, special_tokens=special_tokens or DEFAULT_SPECIAL_TOKENS + ) + return {question.name: question_record(question) for question in request.questions} + + +def expected_answer_types(body, special_tokens=None): + """The requested public answer type per question, from the wire body. + + The worker's public ``type`` field mirrors the request's question type, + so parity dispatches on the requested type: an answer that drifts to + another shape (for example a ``noul`` answered as a plain choice) fails + the comparison instead of silently skipping the noul/score checks. + """ + + from models.valen.protocol import DEFAULT_SPECIAL_TOKENS, parse_request + + request = parse_request( + body, special_tokens=special_tokens or DEFAULT_SPECIAL_TOKENS + ) + return {question.name: question.kind for question in request.questions} def reference_image_record(image_bytes, image_format, questions, media_dir): @@ -137,33 +190,156 @@ def reference_confidence(probabilities): return max(0.0, (max(probabilities) - 1 / count) / (1 - 1 / count)) -def compare_case(name, worker_response, reference, tolerance=PROBABILITY_TOLERANCE): +def reference_score_confidence(probabilities): + """Mirror the worker's score concentration formula (see postprocess.py).""" + + mode = max(range(len(probabilities)), key=probabilities.__getitem__) + count = len(probabilities) + distance = sum(p * abs(i - mode) for i, p in enumerate(probabilities)) + uniform_mad = sum(abs(i - (count - 1) / 2) for i in range(count)) / count + return max(0.0, 1.0 - distance / uniform_mad) + + +def scalar_answer_fields(answer, expected_type, ref_answer): + """Every scalar the per-type branches fold into a running maximum. + + max() keeps its first argument against a NaN difference, so each field + here is swept for finiteness before those branches run. Reference values + derived from the probabilities (the choice-shaped noul decode, the + fallback score) are already covered by the probability gate. + """ + + fields = [answer["confidence"]] + if expected_type == "noul": + fields.append(answer["noul"]) + elif expected_type == "score": + fields.append(answer["score"]) + if ref_answer is not None: + if expected_type == "noul" and "noul" in ref_answer: + fields.append(ref_answer["noul"]) + elif expected_type == "score" and "score" in ref_answer: + fields.append(ref_answer["score"]) + if "confidence" in ref_answer: + fields.append(ref_answer["confidence"]) + return fields + + +def compare_case( + name, worker_response, reference, expected_types, tolerance=PROBABILITY_TOLERANCE +): """Compare one worker response against one reference result. - Returns (report lines, passed). + ``expected_types`` maps each question id to its requested public answer + type; every answer must keep that type before any per-type comparison + runs. ``choice`` answers must agree on the chosen key; ``noul`` answers + on P(true); ``score`` answers on the expected level and the level legend. + All scalar comparisons — probabilities, confidences and the scalar answer + fields — use the declared tolerance because the worker and the pinned + reference normalize probabilities through different softmax + implementations. Returns (report lines, passed). """ lines = [f"case: {name}", "reference model: Valen", f"worker model: {worker_response['model']}"] problems = [] + legend_problems = [] decision_exact = True + legend_exact = True max_prob_delta = 0.0 max_conf_delta = 0.0 + max_answer_delta = 0.0 + scalar_answers = False for qid, ref_keys in reference["keys"].items(): answer = worker_response["answers"][qid] + expected_type = expected_types[qid] + if answer["type"] != expected_type: + problems.append( + f"{qid}: answer type {answer['type']!r} != requested {expected_type!r}" + ) + decision_exact = False + continue worker_keys = list(answer["probabilities"]) if worker_keys != list(ref_keys): problems.append(f"{qid}: candidate order {worker_keys} != {list(ref_keys)}") decision_exact = False continue ref_probs = reference["probabilities"][qid] + # A NaN difference keeps the previous finite maximum and a NaN-free + # argmax list silently keeps its first candidate, so a non-finite + # probability on either side fails the case before any comparison. + if any( + not math.isfinite(value) + for value in list(answer["probabilities"].values()) + list(ref_probs) + ): + problems.append( + f"{qid}: non-finite probability (NaN or infinity) on the " + "worker or reference side" + ) + decision_exact = False + continue for key, worker_p, ref_p in zip(ref_keys, answer["probabilities"].values(), ref_probs): max_prob_delta = max(max_prob_delta, abs(worker_p - ref_p)) - decision = ref_keys[max(range(len(ref_probs)), key=ref_probs.__getitem__)] - if answer["choice"] != decision: - problems.append(f"{qid}: worker chose {answer['choice']!r}, reference {decision!r}") + # The pinned reference decode when the harness ran with the pinned + # source; synthetic references may omit it and fall back to the + # shared probabilities and formulas. + ref_answer = reference.get("answers", {}).get(qid) + # The scalar deltas below fold into running maxima, and max() keeps + # its first argument against a NaN difference, so a non-finite + # scalar field on either side fails the case before any tolerance + # verdict. + if any( + not math.isfinite(value) + for value in scalar_answer_fields(answer, expected_type, ref_answer) + ): + problems.append( + f"{qid}: non-finite scalar answer or confidence (NaN or " + "infinity) on the worker or reference side" + ) decision_exact = False - ref_confidence = reference_confidence(ref_probs) - max_conf_delta = max(max_conf_delta, abs(answer["confidence"] - ref_confidence)) + continue + if expected_type == "noul": + scalar_answers = True + if ref_answer is not None and "noul" in ref_answer: + ref_p_true = ref_answer["noul"] + elif ref_answer is not None: + # Custom-wording noul compiles as a choice record, so the + # pinned decoder answers it in choice shape. + ref_p_true = ref_answer["probabilities"]["true"] + else: + ref_p_true = ref_probs[ref_keys.index("true")] + max_answer_delta = max(max_answer_delta, abs(answer["noul"] - ref_p_true)) + if ref_answer is not None and "confidence" in ref_answer: + ref_confidence = ref_answer["confidence"] + else: + # The reference noul answer carries no confidence; the worker + # exposes Valen's choice formula evaluated on the pair. + ref_confidence = reference_confidence(ref_probs) + max_conf_delta = max(max_conf_delta, abs(answer["confidence"] - ref_confidence)) + elif expected_type == "score": + scalar_answers = True + ref_score = ( + ref_answer["score"] + if ref_answer is not None + else sum(index * p for index, p in enumerate(ref_probs)) + ) + max_answer_delta = max(max_answer_delta, abs(answer["score"] - ref_score)) + if ref_answer is not None and "legend" in ref_answer: + if answer["legend"] != ref_answer["legend"]: + legend_problems.append(f"{qid}: score legend mismatch") + legend_exact = False + if ref_answer is not None and "confidence" in ref_answer: + ref_confidence = ref_answer["confidence"] + else: + # The reference score answer carries no confidence; the worker + # exposes Valen's score concentration formula. + ref_confidence = reference_score_confidence(ref_probs) + max_conf_delta = max(max_conf_delta, abs(answer["confidence"] - ref_confidence)) + else: + decision = ref_keys[max(range(len(ref_probs)), key=ref_probs.__getitem__)] + if answer["choice"] != decision: + problems.append(f"{qid}: worker chose {answer['choice']!r}, reference {decision!r}") + decision_exact = False + ref_confidence = reference_confidence(ref_probs) + max_conf_delta = max(max_conf_delta, abs(answer["confidence"] - ref_confidence)) usage = worker_response["usage"] compute = worker_response["internal_usage"]["compute_tokens"] @@ -184,25 +360,48 @@ def compare_case(name, worker_response, reference, tolerance=PROBABILITY_TOLERAN f" {reference['compute_tokens']}" ) lines.append(f"max probability delta: {max_prob_delta:.10e}") + if scalar_answers: + lines.append(f"max answer delta: {max_answer_delta:.10e}") lines.append(f"confidence delta: {max_conf_delta:.10e}") lines.append( "decision agreement: " + ("PASS" if decision_exact else "FAIL: " + "; ".join(problems)) ) prob_ok = max_prob_delta <= tolerance + confidence_ok = max_conf_delta <= tolerance lines.append( f"probability tolerance <= {tolerance:g}: " + ("PASS" if prob_ok else "FAIL") ) + lines.append( + f"confidence tolerance <= {tolerance:g}: " + + ("PASS" if confidence_ok else "FAIL") + ) + if scalar_answers: + answer_ok = max_answer_delta <= tolerance + lines.append( + f"answer tolerance <= {tolerance:g}: " + ("PASS" if answer_ok else "FAIL") + ) + if any("legend" in answer for answer in worker_response["answers"].values()): + lines.append( + "legend agreement: " + + ("PASS" if legend_exact else "FAIL: " + "; ".join(legend_problems)) + ) lines.append("token accounting: " + ("PASS" if tokens_exact else "FAIL")) - return lines, decision_exact and prob_ok and tokens_exact + passed = ( + decision_exact and legend_exact and prob_ok and confidence_ok and tokens_exact + ) + if scalar_answers: + passed = passed and max_answer_delta <= tolerance + return lines, passed def run_reference(model, base_processor, record, media_root, max_length, media_kwargs): - """Compile a Valen-native record and decode it with a plain softmax.""" + """Compile a Valen-native record and decode it with the pinned modules.""" import torch from valen.data.compiler import Compiler + from valen.evaluation.inference import answer as reference_answer compiler = Compiler( base_processor, @@ -213,19 +412,81 @@ def run_reference(model, base_processor, record, media_root, max_length, media_k state = compiler.compile(record) keys = {} probabilities = {} + answers = {} with torch.no_grad(): for question in state.questions: logits = model(question).float().cpu() probabilities[question.qid] = torch.softmax(logits, dim=-1).tolist() keys[question.qid] = list(question.keys) + # The pinned decoder is the answer-field authority, so every + # question is decoded by it as well as by the plain softmax. + answers[question.qid] = reference_answer(question, logits) return { "keys": keys, "probabilities": probabilities, + "answers": answers, "logical_tokens": state.logical_tokens, "compute_tokens": state.compute_tokens, } +def _importable_stray(entry): + """Whether an untracked or ignored path could be imported by name. + + Only a module sitting where its .py would be can shadow a dependency: + a sourceless `peft.pyc` or an extension `peft.so` imports just like + `peft.py`. Bytecode under `__pycache__` is the cache of a real import + and is not importable on its own, so re-runs after a first import stay + clean. + """ + + if "__pycache__" in Path(entry).parts: + return False + return entry.lower().endswith(IMPORTABLE_SUFFIXES) + + +def _verify_clean_checkout(source): + """Reject a dirty pinned checkout before it can produce evidence. + + Every evidence-producing harness in this recipe imports the pinned + valen package from this tree, so a local modification would make the + records certify agreement with code nobody pinned; a clean tracked + tree with no importable strays is part of what a record claims. + """ + + status = subprocess.run( + [ + "git", + "-C", + str(source), + "status", + "--porcelain", + "--untracked-files=all", + "--ignored=matching", + ], + check=True, + capture_output=True, + text=True, + ) + problems = [] + for line in status.stdout.splitlines(): + entry = line[3:] + if line.startswith(("??", "!!")): + # Untracked or ignored non-code cannot change what gets + # imported; a stray importable module on the package path can, + # and an ignored one hides from plain `status`. + if _importable_stray(entry): + state = "untracked" if line.startswith("??") else "ignored" + problems.append(f"{state} importable file {entry}") + else: + problems.append(f"locally changed {entry}") + if problems: + raise SystemExit( + "--valen-source must be a clean checkout of the pinned revision " + "(commit or restore):\n " + "\n ".join(problems) + ) + + def main(argv=None): args = parse_args(argv) sys.path.insert(0, str(HERE.parents[1] / "src")) @@ -235,6 +496,7 @@ def main(argv=None): source = Path(args.valen_source).resolve() _verify_source_revision(source) + _verify_clean_checkout(source) sys.path.insert(0, str(source)) checkpoint = Path(args.checkpoint).resolve() @@ -245,19 +507,24 @@ def main(argv=None): executor = ValenExecutor(checkpoint, Path(args.base), args.device, args.dtype) cases = [] - if args.case in ("image", "both"): + if args.case in ("image", "both", "all"): cases.append(("image", json.loads(Path(args.fixture).read_text(encoding="utf-8")))) - if args.case in ("text", "both"): + if args.case in ("text", "both", "all"): if args.text_body: cases.append(("text", json.loads(Path(args.text_body).read_text(encoding="utf-8")))) else: cases.append(("text", DEFAULT_TEXT_BODY)) + if args.case in ("noul", "all"): + cases.append(("noul", json.loads(Path(args.noul_body).read_text(encoding="utf-8")))) + if args.case in ("score", "all"): + cases.append(("score", json.loads(Path(args.score_body).read_text(encoding="utf-8")))) failed = 0 try: for name, body in cases: worker_response = decide(json.dumps(body).encode("utf-8"), processor, executor) - questions = reference_questions(body) + questions = reference_questions(body, processor.special_tokens) + expected = expected_answer_types(body, processor.special_tokens) with TemporaryDirectory(prefix="valen-parity-") as directory: media_root = Path(directory) if name == "image": @@ -268,7 +535,9 @@ def main(argv=None): reference = run_reference( executor.model, processor.processor, record, media_root, max_length, media_kwargs ) - lines, ok = compare_case(name, worker_response, reference, args.tolerance) + lines, ok = compare_case( + name, worker_response, reference, expected, args.tolerance + ) print("\n".join(lines)) print() if not ok: diff --git a/recipe/valen/example-noul-request.json b/recipe/valen/example-noul-request.json new file mode 100644 index 00000000..48d57fdb --- /dev/null +++ b/recipe/valen/example-noul-request.json @@ -0,0 +1,18 @@ +{ + "model": "valen-preview-0923", + "state": "The card was charged twice for one order.", + "questions": { + "duplicate": { + "type": "noul", + "instructions": "Was the card charged twice for one order?" + }, + "authorized": { + "type": "noul", + "instructions": "Did the customer authorize both charges?", + "criteria": { + "true": "Both charges were authorized by the customer.", + "false": "At least one charge was unauthorized." + } + } + } +} diff --git a/recipe/valen/example-score-request.json b/recipe/valen/example-score-request.json new file mode 100644 index 00000000..e077afab --- /dev/null +++ b/recipe/valen/example-score-request.json @@ -0,0 +1,15 @@ +{ + "model": "valen-preview-0923", + "state": "The card was charged twice for one order.", + "questions": { + "severity": { + "type": "score", + "instructions": "How severe is the billing error?", + "criteria": [ + "No billing error: every charge is correct.", + "Minor error: a single duplicated charge.", + "Major error: repeated duplicated charges." + ] + } + } +} diff --git a/recipe/valen/export.md b/recipe/valen/export.md new file mode 100644 index 00000000..0aa4da25 --- /dev/null +++ b/recipe/valen/export.md @@ -0,0 +1,218 @@ +# Valen-Preview-0923 checkpoint export + +This recipe turns the pinned Preview checkpoint plus its pinned Qwen3.5-2B +base into a safetensors artifact set that any consumer — native or +otherwise — can load without the training-framework pickle or the reference +Python code. One deterministic command performs every transformation and +records each one in a JSON manifest, so the artifact set can be audited on +its own. Building, validating and parity-checking the artifact set is this +recipe's scope; serving from it is the consumer's side. + +The export complements the +[reference worker recipe](reference.md): the worker serves through the +pinned Python reference stack, while the export is the path for everything +that consumes weights directly. + +## Usage + +Requirements: the pinned artifacts prepared exactly as in the reference +recipe's [install section](reference.md#install-and-prepare-the-artifacts), +and the repository venv. The export runs CPU-only and never needs a GPU. + +~~~sh +PYTHONPATH=src .venv/bin/python recipe/valen/export_checkpoint.py \ + --checkpoint "$VALEN_CKPT" \ + --base "$VALEN_BASE" \ + --out weights/valen-export +~~~ + +`--out` names the output directory; it must not exist (or be empty), and +every path is overridable this way. The export streams tensor-by-tensor +(mmap reads, an incremental safetensors writer), so host memory stays well +below the ~4.5 GiB artifact size. Expected console output: + +~~~text +exported weights/valen-export + backbone: 578 tensors + decision-head: 2 tensors + lora: 372 tensors + merger: 6 tensors + vision-top: 48 tensors + downcasts: 36 fp32 -> bf16 + lora: 372 unmerged fp32 keys over 186 targets + manifest: weights/valen-export/valen_export.json +~~~ + +Before writing anything, the export enforces the artifact pins — the +checkpoint SHA-256 against `training_metadata.json`, the base weights +digest against both `valen_manifest.json` and the checkpoint's own +`base_manifest`, and the revision agreement between them — and rejects a +checkpoint whose `stage` is not `vision_top`. A missing pin or a mismatch +fails loudly instead of exporting the wrong weights. + +## The artifact set + +Five files, nothing else: + +| File | Contents | +| --- | --- | +| `model.safetensors` | The backbone in the base's own key layout (`model.language_model.*` / `model.visual.*`), BF16 only: the base's 632 tensors with the trained vision-top blocks 20–23 and the merger substituted as plain replacements (48 + 6), the remaining 578 passing through verbatim (36 of them downcast from FP32). The 186 language LoRA targets are base tensors here, untouched. | +| `lora.safetensors` | The language LoRA, served unmerged: 372 FP32 tensors under their PEFT key names, verbatim from the checkpoint. | +| `head.safetensors` | The decision head: two FP32 matrices `head.decision.weight` and `head.candidate.weight`, no bias. | +| `config.json` | The base config copied verbatim, so the reference builder loads the export unchanged. | +| `valen_export.json` | The manifest (below). | + +Inventory, totaling 1006 manifest tensor records: 578 `backbone`, 372 +`lora`, 48 `vision-top`, 6 `merger`, 2 `decision-head`. The vision-top +blocks and merger land in the base's own `model.visual.*` layout — same +key tails and shapes as the base snapshot — so they are drop-in +replacements. + +The set deliberately carries no tokenizer or processor files: nothing +trained depends on them, and reconstructing the full serving stack still +reads those from the pinned base snapshot alongside the export. + +## The manifest + +`valen_export.json` is format `valen-checkpoint-export/2` and is the +contract for auditing the export without the reference code. Its +top-level structure: + +~~~text +format "valen-checkpoint-export/2" +sources pinned checkpoint and base revisions + SHA-256s, tool versions +lora the adapter file, dtype, rank/alpha/scaling, target and key + counts, and the declared application semantics +downcasts the names of every FP32 -> BF16 downcast tensor +head head file, dtype, projection_dim, hidden_size, formulas +tensors per-tensor name/shape/dtype/component/source/file records +files size + SHA-256 of each artifact file +consumption how to load the artifact set (transcribed below) +parity the parity-gate result; null until the gate fills it +parity_slot_shape the documented shape of the parity record +~~~ + +`export_checkpoint.validate_export` cross-checks a finished export +against its manifest — file digests, tensor inventory against the +safetensors headers, downcast bookkeeping, head config, and the adapter +inventory (every `lora.safetensors` key an FP32 tensor that maps onto a +listed backbone tensor, one A and one B per target, at the declared rank) +— reading headers and digests only, so it also runs cheaply against the +multi-GiB real artifact set. + +## The LoRA adapters + +The export serves the language LoRA separately and unmerged: the 372 FP32 +adapter tensors ride in `lora.safetensors` under their PEFT key names +(`backbone.language_model.base_model.model..lora_A.default.weight` +and the `lora_B` twins), verbatim from the checkpoint, and +`model.safetensors` carries the base weights of the 186 targeted linears +untouched. The pinned config is rank 32, alpha 64, scaling α/r = 64/32 = +2.0. + +The application semantics a consumer must implement, recorded verbatim in +the manifest's `lora` block: per targeted linear, +`y = bf16(fp32(bf16(x @ W.t())) + scaling * ((x.float() @ A.t()) @ B.t()))` +— the base GEMM runs in BF16, the LoRA delta is computed in FP32 on the +activations and added to the upcast base output, and the sum is rounded +to BF16 exactly once. These are the unmerged peft 0.18.1 `Linear.forward` +semantics with the adapters kept FP32 at runtime — the same boundaries +the pinned reference serves with, and what +`export_checkpoint.unmerged_lora_output` computes (tested bit-equal +against the installed peft). Merging the adapters into the base weights +instead — `bf16(fp32(W) + 2·(B@A))` — changes the computed outputs +relative to the trained artifact's unmerged semantics, which is why the +export does not do it. + +## The FP32 downcasts + +The base stores 36 tensors in FP32 — the delta-rule normalizer logs and +norms, `model.language_model.layers..linear_attn.A_log` and +`...linear_attn.norm.weight` — and the export downcasts each to BF16, +listing every one in the manifest's `downcasts` block. The downcast is +lossless against the reference's own BF16 runtime: transformers has no +`keep_in_fp32_modules` entry for this architecture and upcasts these +tensors with `.float()` at use, so the reference already computes on the +BF16-rounded values. Any FP32 base tensor outside the declared families +is an error, not a silent downcast. + +## The decision head + +`head.safetensors` holds the two FP32 projections — +`head.decision.weight` and `head.candidate.weight` — each +256 × 2048 (`projection_dim` × `hidden_size`), no bias, with the +dimensions and formulas recorded in the manifest's head block: +logits are `(W_cand @ h_cand) . (W_dec @ h_dec) / sqrt(projection_dim)` +in FP32. The head is exported in FP32, unchanged from the checkpoint, +which stores it in FP32. + +## Consumption + +The manifest's `consumption` block records how to load the export; in +full: + +~~~text +backbone: Qwen3_5ForConditionalGeneration.from_pretrained(out, dtype=bf16, + attn_implementation='eager', local_files_only=True).model +lora: wrap backbone.language_model in peft get_peft_model(LoraConfig(r=32, + lora_alpha=64, lora_dropout=0, target_modules=, bias='none')), load lora.safetensors' tensors by key and + keep the adapters fp32 at runtime (the application semantics + recorded in the lora block) +head: head.safetensors: DecisionHead(hidden_size=2048, + projection_dim=256), fp32, no bias +~~~ + +The copied `config.json` makes the `from_pretrained` call work against +the export directory directly; the LoRA is re-applied exactly the way the +pinned reference builds it (the same peft wrap at the recorded rank and +alpha, adapters kept FP32), and the head is rebuilt from its two matrices +and recorded dimensions. Nothing else is needed to reconstruct the model. + +## Pins and licenses + +The export consumes the same pinned artifacts as the +[reference worker](reference.md#pinned-artifacts) — the Preview +checkpoint revision `81b9c639...` (SHA-256 +`836622ef...f42f5a012`) and the Qwen3.5-2B base revision +`15852e8c...` — and records both, with per-file digests, in the +manifest's `sources` block on every run. The Valen source, the Preview +checkpoint, and the Qwen base report Apache-2.0 license metadata for +these pins (recorded with the serving slice in +[PR #115](https://github.com/ThinkFlowLab/system1-omni/pull/115)); +re-check the metadata when changing a revision. The exported artifact +set inherits those terms. + +## Parity evidence + +Whether the exported stack reproduces the original checkpoint's answers +is the parity gate's question, not this document's claim: the verdict — +numbers, device, tolerances — lives in the manifest's +`parity` slot (left `null` by the export, shaped by +`parity_slot_shape`) and in the evidence blocks of +[reference.md](reference.md), where the parity run is recorded. Consult +those for the result; do not treat an export as parity-proven until its +manifest's parity slot is filled. + +## Determinism and regression + +Two runs on the same inputs produce byte-identical artifacts and +identical manifests: no timestamps, no absolute paths, tensors and JSON +written in sorted key order, single-threaded CPU math. The regression is +wired into the recipe-test pattern: + +~~~sh +PYTHONPATH=src .venv/bin/python -m pytest tests/valen -q +python recipe/valen/test_export_reference.py +~~~ + +The first is the CPU-only suite (115 tests passed, 2 artifact-gated skips +at the time of writing). The second — implementation in +[`tests/valen/export_reference_checks.py`](../../tests/valen/export_reference_checks.py), +the recipe path is thin entrypoint glue — chains that suite's export +battery plus the env-gated real-artifact determinism gate — set `VALEN_CKPT` +and `VALEN_BASE` to run it; it exports twice and compares every byte — and, +with `VALEN_EXPORT` pointing at an exported directory (defaulting to +`weights/valen-export` when present), validates the finished artifact set +against its manifest. Without the artifacts, both gated phases print a +clear skip line and the script still passes. diff --git a/recipe/valen/export_checkpoint.py b/recipe/valen/export_checkpoint.py new file mode 100644 index 00000000..54e70508 --- /dev/null +++ b/recipe/valen/export_checkpoint.py @@ -0,0 +1,1068 @@ +#!/usr/bin/env python3 +"""Export the pinned Valen checkpoint plus its base to a safetensors set. + +One deterministic command turns the training-framework pickle +(``checkpoint.pt`` + the pinned Qwen3.5-2B base) into the artifact set a +native worker can load, plus a JSON manifest recording every transformation +performed: + +- ``model.safetensors`` — the backbone in the base's own key layout + (``model.language_model.*`` / ``model.visual.*``, BF16 only): the base + weights verbatim except the trained vision-top blocks 20-23 and the + merger substituted as plain BF16 replacements and the base's 36 FP32 + tensors (``linear_attn.A_log`` / ``linear_attn.norm.weight``) downcast + to BF16 — the same cast transformers' own BF16 runtime performs. The + 186 language LoRA targets are base tensors here, untouched. +- ``lora.safetensors`` — the language LoRA served unmerged: the 372 FP32 + adapter tensors under their PEFT key names, verbatim from the + checkpoint. +- ``head.safetensors`` — the decision head's two FP32 projection matrices + (``head.decision.weight``, ``head.candidate.weight``), no bias. +- ``config.json`` — the base config copied verbatim so the reference builder + (``Qwen3_5ForConditionalGeneration.from_pretrained(out, dtype=bf16, + attn_implementation="eager").model``) loads the export unchanged. +- ``valen_export.json`` — the manifest: per-tensor name/shape/dtype and + component attribution (backbone / lora / vision-top / merger / + decision-head), source revisions and SHA-256s, the LoRA configuration + and its declared application semantics, the downcast list, the head + configuration, and a ``parity`` slot filled by + ``verify_export_parity.py``. + +Determinism: no timestamps, no absolute paths, tensors and JSON written in +sorted key order, single-threaded CPU math — two runs on the same inputs +produce byte-identical artifacts and identical manifests. The export runs +CPU-only and streams tensor-by-tensor (mmap reads, an incremental +safetensors writer), so host memory stays well below the artifact sizes. + +Usage: + + PYTHONPATH=src .venv/bin/python recipe/valen/export_checkpoint.py \ + --checkpoint weights/Valen-Preview-0923 --base weights/Qwen3.5-2B \ + --out weights/valen-export +""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import os +import re +import shutil +import struct +import sys +from pathlib import Path +from typing import NamedTuple + +import torch + +FORMAT = "valen-checkpoint-export/2" +BACKBONE_FILE = "model.safetensors" +LORA_FILE = "lora.safetensors" +HEAD_FILE = "head.safetensors" +MANIFEST_FILE = "valen_export.json" + +# Checkpoint key layout: PEFT wraps backbone.language_model, vision-top and +# merger tensors ride under backbone.visual.*, and the decision head is head.*. +PEFT_LANGUAGE_PREFIX = "backbone.language_model.base_model.model." +PEFT_LORA_SUFFIX_A = ".lora_A.default.weight" +PEFT_LORA_SUFFIX_B = ".lora_B.default.weight" +HEAD_PREFIX = "head." + +COMPONENT_BACKBONE = "backbone" +COMPONENT_LORA = "lora" +COMPONENT_VISION_TOP = "vision-top" +COMPONENT_MERGER = "merger" +COMPONENT_HEAD = "decision-head" +COMPONENTS = ( + COMPONENT_BACKBONE, + COMPONENT_LORA, + COMPONENT_VISION_TOP, + COMPONENT_MERGER, + COMPONENT_HEAD, +) + +# The pinned Preview inventory: 428 trainable tensors (372 LoRA keys over +# 186 targeted linears, 48 vision-top, 6 merger, 2 head); the base holds 632 +# tensors, 36 of them FP32. +PINNED_INVENTORY = { + "checkpoint_tensors": 428, + "lora_keys": 372, + "vision_top_keys": 48, + "merger_keys": 6, + "head_keys": 2, + "base_tensors": 632, + "base_fp32_tensors": 36, +} + +# How the exported LoRA is applied at serving time; the manifest records +# this verbatim in its lora block and unmerged_lora_output() is its +# executable form. +LORA_APPLICATION = ( + "per targeted linear, y = bf16(fp32(bf16(x @ W.t())) + scaling * " + "((x.float() @ A.t()) @ B.t())): the base GEMM runs in BF16, the LoRA " + "delta is computed in FP32 on the activations and added to the upcast " + "base output, and the sum is rounded to BF16 exactly once — the unmerged " + "peft 0.18.1 Linear.forward with the adapters kept FP32 at runtime" +) + +# The parity record's shape; the manifest ships it null until measured. +PARITY_SLOT_SHAPE = { + "status": "passed | failed", + "device": "torch device string", + "hardware": "GPU name and memory when the device is cuda", + "tolerance": "the probability-delta gate", + "cases": "text and image choice fixtures compared", + "exact_answer_agreement": True, + "max_probability_delta": "<= tolerance (1e-6)", + "evidence": "pointer to the recorded parity run", +} + + +def lora_scaling(rank, alpha): + """PEFT's scaling constant alpha/r (64/32 = 2.0 for the pinned config).""" + + return alpha / rank + + +def unmerged_lora_output(base_weight, lora_a, lora_b, activations, scaling): + """Apply one LoRA pair the way the manifest declares it is served. + + The base GEMM runs in the base dtype (BF16), the delta + ``scaling * ((x @ A.t()) @ B.t())`` is computed in FP32 on the + activations and added to the FP32-upcast base output, and the sum is + rounded to the base dtype exactly once — bit-equal to peft 0.18.1's + unmerged ``Linear.forward`` with FP32 adapters. A zero-B adapter + reproduces the plain base GEMM bit-exactly. + """ + + y_base = torch.matmul(activations, base_weight.t()) + delta = scaling * ((activations.float() @ lora_a.t()) @ lora_b.t()) + return (y_base.float() + delta).to(base_weight.dtype) + + +def downcast_bf16(tensor): + """Downcast a stored-FP32 base tensor to the BF16 the reference runtime + itself keeps (transformers has no keep_in_fp32_modules for qwen3_5 and + upcasts with ``.float()`` at use).""" + + return tensor.to(torch.bfloat16) + + +class Classification(NamedTuple): + """The checkpoint's trainable tensors decomposed by provenance component. + + ``lora`` maps the base tensor name each adapter pair targets to its + ``(lora_A, lora_B)`` checkpoint keys; ``vision_top``/``merger`` pair each + export key (the base's own ``model.visual.*`` layout) with its checkpoint + source; ``head`` maps the export names identically. Everything is kept in + sorted key order so downstream output is deterministic. + """ + + lora: dict + vision_top: list + merger: list + head: dict + + +_VISION_BLOCK = re.compile(r"^backbone\.visual\.blocks\.(\d+)\.(.+)$") +_HEAD_TENSORS = ("head.decision.weight", "head.candidate.weight") + + +def lora_key_target(key): + """The ``model.safetensors`` tensor a PEFT LoRA key adapts.""" + + for suffix in (PEFT_LORA_SUFFIX_A, PEFT_LORA_SUFFIX_B): + if key.startswith(PEFT_LANGUAGE_PREFIX) and key.endswith(suffix): + return ( + "model.language_model." + + key[len(PEFT_LANGUAGE_PREFIX): -len(suffix)] + + ".weight" + ) + raise ValueError(f"not a language LoRA key: {key!r}") + + +def classify_checkpoint_weights(keys): + """Map the checkpoint's weight keys onto export names by component. + + Rejects anything outside the pinned ``vision_top`` layout: frozen + backbone tensors, non-PEFT language keys, head biases and unpaired or + non-top-4 vision adapters are all errors — the recipe is only valid + for this checkpoint's verified component layout. + """ + + lora_a = {} + lora_b = {} + vision_top = [] + merger = [] + head = {} + for key in sorted(keys): + if key.startswith(PEFT_LANGUAGE_PREFIX) and key.endswith(PEFT_LORA_SUFFIX_A): + base_name = ( + "model.language_model." + + key[len(PEFT_LANGUAGE_PREFIX): -len(PEFT_LORA_SUFFIX_A)] + + ".weight" + ) + lora_a[base_name] = key + elif key.startswith(PEFT_LANGUAGE_PREFIX) and key.endswith(PEFT_LORA_SUFFIX_B): + base_name = ( + "model.language_model." + + key[len(PEFT_LANGUAGE_PREFIX): -len(PEFT_LORA_SUFFIX_B)] + + ".weight" + ) + lora_b[base_name] = key + elif key.startswith("backbone.visual.merger."): + merger.append(("model.visual.merger." + key[len("backbone.visual.merger."):], key)) + elif _VISION_BLOCK.match(key): + block, tail = _VISION_BLOCK.match(key).groups() + if int(block) not in (20, 21, 22, 23): + raise ValueError( + f"unknown checkpoint weight {key!r}: the vision_top stage " + "trains only the last four blocks (20-23)" + ) + vision_top.append((f"model.visual.blocks.{block}.{tail}", key)) + elif key in _HEAD_TENSORS: + head[key] = key + else: + raise ValueError(f"unknown checkpoint weight {key!r}") + if set(lora_a) != set(lora_b): + missing = sorted(set(lora_a) ^ set(lora_b)) + raise ValueError(f"unpaired LoRA adapters for {missing}") + return Classification( + lora={name: (lora_a[name], lora_b[name]) for name in sorted(lora_a)}, + vision_top=sorted(vision_top), + merger=sorted(merger), + head={name: head[name] for name in sorted(head)}, + ) + + +def validate_inventory(classification, expected): + """Check the classified component counts against the expected inventory.""" + + counts = { + "checkpoint_tensors": 2 * len(classification.lora) + + len(classification.vision_top) + + len(classification.merger) + + len(classification.head), + "lora_keys": 2 * len(classification.lora), + "vision_top_keys": len(classification.vision_top), + "merger_keys": len(classification.merger), + "head_keys": len(classification.head), + } + for name, actual in counts.items(): + wanted = expected[name] + if actual != wanted: + raise ValueError( + f"checkpoint inventory mismatch: {name} is {actual}, expected {wanted}" + ) + + +# --- streaming safetensors writer --- +# The pinned base is a single ~4.5 GiB file and host memory is a fraction of +# that, so the export never holds the artifact set in memory: every tensor is +# materialized once, in sorted name order, straight into the output file. +# The byte format is the safetensors v1 spec (8-byte LE header length, JSON +# header padded to an 8-byte boundary, then contiguous tensor data), written +# with fixed ordering so two runs are byte-identical. + +_DTYPE_NAMES = {torch.bfloat16: "BF16", torch.float32: "F32"} +_DTYPE_SIZES = {torch.bfloat16: 2, torch.float32: 4} +_READER_DTYPES = {"BF16": torch.bfloat16, "F32": torch.float32} +_MANIFEST_DTYPES = {"BF16": "bfloat16", "F32": "float32"} +_FP32_TENSOR = re.compile( + r"^model\.language_model\.layers\.\d+\.linear_attn\.(A_log|norm\.weight)$" +) + + +def _tensor_bytes(tensor): + """Raw little-endian bytes of one contiguous tensor (BF16-safe). + + Flattened to 1-D so the byte length is the full payload: ``len()`` of a + multi-dimensional memoryview only reports its first dimension. + """ + + view = tensor.contiguous().view(torch.uint8).reshape(-1) + return memoryview(view.numpy()) + + +def write_safetensors(path, entries): + """Stream ``(name, dtype, shape, materialize)`` entries to ``path``. + + ``materialize`` is called once per entry, in order, so callers can pull + each tensor from an mmap'd source and release it before the next. + Returns ``{"size", "sha256"}`` of the bytes written. + """ + + header = {"__metadata__": {"format": "pt"}} + offset = 0 + for name, dtype, shape, _ in entries: + nbytes = _DTYPE_SIZES[dtype] + for dim in shape: + nbytes *= dim + header[name] = { + "dtype": _DTYPE_NAMES[dtype], + "shape": list(shape), + "data_offsets": [offset, offset + nbytes], + } + offset += nbytes + header_bytes = json.dumps(header, separators=(",", ":")).encode("utf-8") + header_bytes += b" " * (-(8 + len(header_bytes)) % 8) + digest = hashlib.sha256() + with open(path, "wb") as handle: + + def write(chunk): + digest.update(chunk) + handle.write(chunk) + + write(struct.pack(", bias='none')), load " + f"{LORA_FILE}'s tensors by key and keep the adapters fp32 at runtime " + "(the application semantics recorded in the lora block)" + ), + "head": ( + f"{HEAD_FILE}: DecisionHead(hidden_size={head_config['hidden_size']}, " + f"projection_dim={head_config['projection_dim']}), fp32, no bias" + ), + }, + "parity": None, + "parity_slot_shape": PARITY_SLOT_SHAPE, + } + (out_dir / MANIFEST_FILE).write_text( + json.dumps(manifest, indent=2, sort_keys=True) + "\n", encoding="utf-8" + ) + return manifest + + +def validate_export(out_dir): + """Cross-check a finished export against its manifest. + + Reads only headers and digests, not tensor payloads, so it also runs + cheaply against the multi-GiB real artifact set. Returns the manifest + when every recorded claim matches the bytes on disk. + """ + + from safetensors import safe_open + + out_dir = Path(out_dir) + # Any unexpected entry — file, directory or otherwise — fails the audit. + present = sorted(p.name for p in out_dir.iterdir()) + allowed = sorted( + [BACKBONE_FILE, LORA_FILE, HEAD_FILE, MANIFEST_FILE, "config.json"] + ) + if present != allowed: + extra = sorted(set(present) - set(allowed)) + missing = sorted(set(allowed) - set(present)) + raise ValueError( + f"export directory must hold exactly {allowed}: " + f"unexpected entries {extra}, missing {missing}" + ) + manifest = json.loads((out_dir / MANIFEST_FILE).read_text(encoding="utf-8")) + if manifest.get("format") != FORMAT: + raise ValueError(f"manifest format {manifest.get('format')!r} != {FORMAT!r}") + artifact_files = {BACKBONE_FILE, LORA_FILE, HEAD_FILE, "config.json"} + if set(manifest["files"]) != artifact_files: + raise ValueError( + f"manifest files record {sorted(manifest['files'])} != " + f"{sorted(artifact_files)}" + ) + + # Recorded file digests must match the bytes on disk. Digests are + # streamed (sha256_file) and sizes come from stat(), so the multi-GiB + # model.safetensors never enters host memory. + for name, record in sorted(manifest["files"].items()): + path = out_dir / name + size = path.stat().st_size + if record["size"] != size: + raise ValueError(f"{name} is {size} bytes, manifest says {record['size']}") + digest = sha256_file(path) + if record["sha256"] != digest: + raise ValueError(f"{name} sha256 {digest} != manifest {record['sha256']}") + + # Every safetensors tensor is listed with matching metadata, and vice versa. + listed = {} + for entry in manifest["tensors"]: + for field in ("name", "shape", "dtype", "component", "source", "file"): + if field not in entry: + raise ValueError(f"manifest tensor entry missing {field!r}: {entry}") + if entry["component"] not in COMPONENTS: + raise ValueError(f"unknown component {entry['component']!r} for {entry['name']}") + if entry["file"] not in (BACKBONE_FILE, LORA_FILE, HEAD_FILE): + raise ValueError( + f"tensor {entry['name']!r} points at unknown file {entry['file']!r}" + ) + if (entry["file"], entry["name"]) in listed: + raise ValueError(f"manifest lists {entry['name']!r} twice") + listed[(entry["file"], entry["name"])] = entry + on_disk = set() + handles = { + file_name: safe_open(out_dir / file_name, framework="pt") + for file_name in (BACKBONE_FILE, LORA_FILE, HEAD_FILE) + } + try: + for file_name, handle in handles.items(): + on_disk.update((file_name, key) for key in handle.keys()) + finally: + for handle in handles.values(): + handle.__exit__(None, None, None) + orphans = sorted(set(listed) - on_disk) + if orphans: + raise ValueError(f"manifest lists tensors absent from the files: {orphans[:4]}") + unlisted = sorted(on_disk - set(listed)) + if unlisted: + raise ValueError( + f"{unlisted[0][0]} holds unlisted tensor {unlisted[0][1]!r}" + f" ({len(unlisted)} total)" + ) + for file_name in (BACKBONE_FILE, LORA_FILE, HEAD_FILE): + with safe_open(out_dir / file_name, framework="pt") as handle: + for key in sorted(k for f, k in on_disk if f == file_name): + slice_ = handle.get_slice(key) + entry = listed[(file_name, key)] + if entry["shape"] != list(slice_.get_shape()): + raise ValueError( + f"{key!r} shape {entry['shape']} != file {list(slice_.get_shape())}" + ) + if entry["dtype"] != _MANIFEST_DTYPES[slice_.get_dtype()]: + raise ValueError( + f"{key!r} dtype {entry['dtype']} != file " + f"{_MANIFEST_DTYPES[slice_.get_dtype()]}" + ) + + # Downcast bookkeeping: the list and the per-tensor flags must agree. + downcasts = set(manifest["downcasts"]) + flagged = { + entry["name"] + for entry in manifest["tensors"] + if entry.get("downcast_from") is not None + } + if downcasts != flagged: + raise ValueError( + "downcast list and tensor flags disagree: " + f"list-only {sorted(downcasts - flagged)}, flag-only {sorted(flagged - downcasts)}" + ) + for name in sorted(downcasts): + entry = listed.get((BACKBONE_FILE, name)) + if ( + entry is None + or entry.get("downcast_from") != "float32" + or entry["dtype"] != "bfloat16" + ): + raise ValueError(f"downcast entry {name!r} is not a bf16 tensor from fp32") + + # Head config must match the exported head tensors. + head = manifest["head"] + projection = (head["projection_dim"], head["hidden_size"]) + for name in head["tensors"]: + entry = listed.get((HEAD_FILE, name)) + if entry is None: + raise ValueError(f"head config lists unknown tensor {name!r}") + if tuple(entry["shape"]) != projection: + raise ValueError( + f"head tensor {name!r} shape {entry['shape']} contradicts the head config " + f"(projection_dim, hidden_size)={projection}" + ) + if entry["dtype"] != "float32" or entry["component"] != COMPONENT_HEAD: + raise ValueError(f"head tensor {name!r} must be an fp32 decision-head tensor") + + # The LoRA block must match the adapter inventory: every adapter key is + # a listed fp32 tensor in the adapter file that maps onto a listed + # backbone tensor, one A and one B per target, at the declared rank. + lora = manifest["lora"] + if lora["file"] != LORA_FILE: + raise ValueError(f"lora block points at {lora['file']!r}, expected {LORA_FILE!r}") + if lora["scaling"] != lora["alpha"] / lora["rank"]: + raise ValueError("lora scaling is not alpha / rank") + adapters = [e for e in manifest["tensors"] if e["component"] == COMPONENT_LORA] + if lora["keys"] != len(adapters): + raise ValueError( + f"lora block records {lora['keys']} keys, manifest lists {len(adapters)}" + ) + if lora["targets"] * 2 != lora["keys"]: + raise ValueError( + f"lora block records {lora['targets']} targets for {lora['keys']} keys" + ) + by_target = {} + for entry in adapters: + name = entry["name"] + if entry["file"] != LORA_FILE or entry["dtype"] != "float32": + raise ValueError(f"adapter {name!r} must be an fp32 tensor in {LORA_FILE}") + target = lora_key_target(name) + if (BACKBONE_FILE, target) not in listed: + raise ValueError(f"adapter {name!r} targets absent backbone tensor {target!r}") + by_target.setdefault(target, []).append(name) + rank_dim = 0 if name.endswith(PEFT_LORA_SUFFIX_A) else 1 + if entry["shape"][rank_dim] != lora["rank"]: + raise ValueError( + f"adapter {name!r} shape {entry['shape']} contradicts the declared " + f"rank {lora['rank']}" + ) + for target, keys in sorted(by_target.items()): + if len(keys) != 2 or not ( + keys[0].endswith(PEFT_LORA_SUFFIX_A) ^ keys[1].endswith(PEFT_LORA_SUFFIX_A) + ): + raise ValueError(f"target {target!r} does not carry exactly one A and one B") + if len(by_target) != lora["targets"]: + raise ValueError( + f"lora block records {lora['targets']} targets, adapters cover {len(by_target)}" + ) + if manifest["parity"] is not None and not isinstance(manifest["parity"], dict): + raise ValueError("parity slot must be null or the parity-gate record") + return manifest + + +# --- the real-artifact path: pinned checkpoint + pinned base -> export --- + +CHECKPOINT_FILE = "checkpoint.pt" +TRAINING_METADATA_FILE = "training_metadata.json" +BASE_MANIFEST_FILE = "valen_manifest.json" +BASE_INDEX_FILE = "model.safetensors.index.json" +PINNED_STAGE = "vision_top" + + +def sha256_file(path): + """Stream a file through SHA-256 without loading it.""" + + digest = hashlib.sha256() + with open(path, "rb") as handle: + for chunk in iter(lambda: handle.read(1 << 22), b""): + digest.update(chunk) + return digest.hexdigest() + + +def _pinned_checkpoint_sha256(): + """The worker's unconditional checkpoint pin — the load's trust anchor. + + training_metadata.json ships inside the directory it describes, so it + is a consistency record, not a trust anchor; the exporter therefore + enforces the worker's own pinned digest before the pickle is loaded, + exactly as the worker does.""" + + from models.valen.engine import CHECKPOINT_SHA256 + + return CHECKPOINT_SHA256 + + +def checkpoint_revision(checkpoint_dir): + """The HF revision the Preview directory was pinned to, if recorded. + + ``hf download`` leaves a per-revision tree file under .cache; its file + name is the revision. Returns None when the directory carries no pin + (the sha256 remains the authoritative identity either way). + """ + + trees = Path(checkpoint_dir) / ".cache" / "huggingface" / "trees" + if not trees.is_dir(): + return None + revisions = sorted(p.stem for p in trees.glob("*.json")) + if len(revisions) != 1: + return None + return revisions[0] + + +def base_weights_file(base_dir): + """The single safetensors shard the pinned base stores its weights in.""" + + index_path = Path(base_dir) / BASE_INDEX_FILE + index = json.loads(index_path.read_text(encoding="utf-8")) + shards = sorted(set(index["weight_map"].values())) + if len(shards) != 1: + raise ValueError( + f"this recipe exports single-shard bases; {index_path} lists {shards}" + ) + path = Path(base_dir) / shards[0] + if not path.is_file(): + raise ValueError(f"base weights file {path} does not exist") + return path + + +def export(checkpoint_dir, base_dir, out_dir): + """Export the pinned checkpoint + base into ``out_dir`` (must not exist). + + Enforces the pins before writing anything: the checkpoint digest + against the worker's pinned CHECKPOINT_SHA256 (the trust anchor) and + the training pipeline's training_metadata.json record, the base + weights digest against both valen_manifest.json and the checkpoint's + own base_manifest, and the revision agreement between them. Returns + the validated manifest. + """ + + checkpoint_dir = Path(checkpoint_dir) + base_dir = Path(base_dir) + + checkpoint_path = checkpoint_dir / CHECKPOINT_FILE + checkpoint_sha = sha256_file(checkpoint_path) + pinned_sha = _pinned_checkpoint_sha256() + if checkpoint_sha != pinned_sha: + raise ValueError( + f"checkpoint sha256 {checkpoint_sha} != the pinned " + f"Valen-Preview-0923 digest {pinned_sha}" + ) + metadata_path = checkpoint_dir / TRAINING_METADATA_FILE + if not metadata_path.is_file(): + raise ValueError( + f"checkpoint directory {checkpoint_dir} is missing " + f"{TRAINING_METADATA_FILE}: the training pipeline's digest record " + "must also agree before the pickle is loaded" + ) + declared = json.loads(metadata_path.read_text(encoding="utf-8"))["checkpoint_sha256"] + if declared != checkpoint_sha: + raise ValueError( + f"checkpoint sha256 {checkpoint_sha} != training_metadata {declared}" + ) + + payload = torch.load( + checkpoint_path, map_location="cpu", weights_only=False, mmap=True + ) + config = payload["config"] + if config.get("stage") != PINNED_STAGE: + raise ValueError( + f"checkpoint stage {config.get('stage')!r} != {PINNED_STAGE!r}: this " + "recipe only exports the vision_top Preview layout" + ) + + weights_file = base_weights_file(base_dir) + shard_sums = {weights_file.name: sha256_file(weights_file)} + valen_manifest = json.loads( + (base_dir / BASE_MANIFEST_FILE).read_text(encoding="utf-8") + ) + base_manifest = payload.get("base_manifest") or {} + for label, declared in ( + ("valen_manifest.json", valen_manifest.get("weight_sha256")), + ("checkpoint base_manifest", base_manifest.get("weight_sha256")), + ): + # Both pins must declare a per-shard sha mapping covering exactly the + # shards on disk and matching every digest; a missing declaration is + # a pin bypass, not a pass. + if declared is None: + raise ValueError(f"{label} does not declare the base weights sha256") + if dict(declared) != shard_sums: + raise ValueError( + f"base weights sha256 {shard_sums} != {label} {dict(declared)}" + ) + base_revision = valen_manifest.get("revision") + if base_manifest.get("revision") not in (None, base_revision): + raise ValueError( + f"base revision {base_revision!r} != checkpoint base_manifest " + f"{base_manifest['revision']!r}" + ) + + base_config = json.loads((base_dir / "config.json").read_text(encoding="utf-8")) + hidden_size = base_config["text_config"]["hidden_size"] + import safetensors + + environment = { + "torch": torch.__version__, + "safetensors": safetensors.__version__, + } + try: + import peft + + environment["peft"] = peft.__version__ + except ImportError: # pragma: no cover - the pinned venv always has peft + pass + + sources = { + "checkpoint": { + "sha256": checkpoint_sha, + "revision": checkpoint_revision(checkpoint_dir), + "stage": config["stage"], + }, + "base": { + "weights_sha256": shard_sums, + "revision": base_revision, + "repo": valen_manifest.get("repo"), + }, + "environment": environment, + } + manifest = run_export( + payload["weights"], + weights_file, + out_dir, + sources=sources, + lora_config={"rank": config["lora_rank"], "alpha": config["lora_alpha"]}, + head_config={ + "projection_dim": config["projection_dim"], + "hidden_size": hidden_size, + "bias": False, + }, + expected=PINNED_INVENTORY, + base_config_path=base_dir / "config.json", + ) + return validate_export(Path(out_dir)) + + +def parse_args(argv=None): + parser = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + parser.add_argument("--checkpoint", required=True, help="Valen-Preview-0923 directory") + parser.add_argument("--base", required=True, help="Qwen3.5-2B base directory") + parser.add_argument("--out", required=True, type=Path, help="export output directory") + return parser.parse_args(argv) + + +def main(argv=None): + args = parse_args(argv) + manifest = export(args.checkpoint, args.base, args.out) + counts = {} + for entry in manifest["tensors"]: + counts[entry["component"]] = counts.get(entry["component"], 0) + 1 + print(f"exported {args.out}") + for component in sorted(counts): + print(f" {component}: {counts[component]} tensors") + print(f" downcasts: {len(manifest['downcasts'])} fp32 -> bf16") + print( + f" lora: {manifest['lora']['keys']} unmerged fp32 keys over " + f"{manifest['lora']['targets']} targets" + ) + print(f" manifest: {args.out / MANIFEST_FILE}") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/recipe/valen/reference.md b/recipe/valen/reference.md index 863d07a0..3337e5b4 100644 --- a/recipe/valen/reference.md +++ b/recipe/valen/reference.md @@ -17,19 +17,21 @@ The validated slice is: - the Python reference worker behind the Rust frontend; - text state (a non-empty string; objects and arrays are serialized to JSON text) or one inline PNG or JPEG image in `state.image`; -- textual instructions and criteria for `choice` questions; +- textual instructions and criteria for `choice`, `noul`, and `score` + questions, freely mixed in one request against one shared state; - BF16 CUDA inference with the pinned Preview checkpoint. Image and text state are serving-validated on the GPU host, with recorded reference parity - for both. + for all three question types. -The current worker does not implement `noul`, `score`, video, multiple images, -remote image URLs, or a native Rust/CUDA Valen executor. +The current worker does not implement video, multiple images, remote image +URLs, or a native Rust/CUDA Valen executor. The current protocol limits requests to an 8 MiB JSON body, one PNG/JPEG image of at most 4 MiB, 2048 pixels per side, 1,048,576 pixels total, and a 200:1 maximum aspect ratio. Text state is limited to 16,384 characters, the same limit as combined instruction and criterion text. A request may contain at -most 8 questions and 255 candidates per question. +most 8 questions, 255 named candidates per `choice` question, and an ordered +list of 2 to 10 level descriptions per `score` question. The first slice follows [issue #84](https://github.com/ThinkFlowLab/system1-omni/issues/84): establish @@ -99,7 +101,7 @@ Pillow: 12.3.0 av: 16.1.0 GPU capability: sm_120 dtype: bfloat16 -system1-omni: 2fd8ce9 (GPU serving and smoke records); parity blocks re-verified verbatim at c9fb2d2 +system1-omni: 2fd8ce9 (GPU serving and smoke records); parity blocks re-verified verbatim at 507c448 ~~~ FP32 and BF16 small-operator checks passed. Startup memory was approximately @@ -156,6 +158,10 @@ writes `valen_manifest.json`. Verify the final identity before serving: ~~~sh test "$(git -C "$VALEN_SRC" rev-parse HEAD)" = "$VALEN_SOURCE_REV" +test -z "$(git -C "$VALEN_SRC" status --porcelain --untracked-files=no)" +# Untracked *and* ignored modules Python can import by name (source, +# sourceless bytecode, extension) must be absent; __pycache__ caches are fine. +test -z "$(git -C "$VALEN_SRC" status --porcelain --untracked-files=all --ignored=matching | grep -E '^(\?\?|!!).*\.(py|pyc|pyd|so)$' | grep -v __pycache__)" test "$(sha256sum "$VALEN_CKPT/checkpoint.pt" | cut -d' ' -f1)" = "$VALEN_CHECKPOINT_SHA" .venv/bin/python - <<'PY' import json, os @@ -174,6 +180,9 @@ print(manifest["versions"]) PY ~~~ +The pinned checkpoint and base can also be exported to a manifest-described +safetensors artifact set for non-Python consumers: [export.md](export.md). + ## Start the worker The worker loads all model-owned state and performs a real representative @@ -250,6 +259,60 @@ Observed response, same host as the image fixture: } ~~~ +A yes/no judgment uses `noul`, optionally replacing the reference's fixed +true/false wording through `criteria`. The complete runnable request is in +[`example-noul-request.json`](example-noul-request.json); it asks one +fixed-wording and one custom-wording question against one state: + +~~~sh +curl -sS http://127.0.0.1:8000/v1/systemone \ + -H 'Content-Type: application/json' \ + --data-binary @recipe/valen/example-noul-request.json +~~~ + +Expected response shape (the observed response for this fixture is recorded +in the integration pull request): + +~~~json +{ + "model": "valen-preview-0923", + "answers": { + "duplicate": {"type": "noul", "noul": "", "probabilities": {"true": "", "false": ""}, "confidence": ""}, + "authorized": {"type": "noul", "noul": "", "probabilities": {"true": "", "false": ""}, "confidence": ""} + }, + "usage": {"input_tokens": "", "output_tokens": 0}, + "internal_usage": {"compute_tokens": ""} +} +~~~ + +A graded assessment uses `score` with an ordered list of 2 to 10 level +descriptions. The complete runnable request is in +[`example-score-request.json`](example-score-request.json): + +~~~sh +curl -sS http://127.0.0.1:8000/v1/systemone \ + -H 'Content-Type: application/json' \ + --data-binary @recipe/valen/example-score-request.json +~~~ + +Expected response shape (the observed response for this fixture is recorded +in the integration pull request): + +~~~json +{ + "model": "valen-preview-0923", + "answers": {"severity": { + "type": "score", + "probabilities": {"0": "", "1": "", "2": ""}, + "score": "", + "legend": {"0": "", "1": "", "2": ""}, + "confidence": "" + }}, + "usage": {"input_tokens": "", "output_tokens": 0}, + "internal_usage": {"compute_tokens": ""} +} +~~~ + ## Start and verify the Rust frontend ~~~sh @@ -267,8 +330,8 @@ curl -sS http://127.0.0.1:8080/v1/systemone \ --data-binary @recipe/valen/example-request.json ~~~ -The generic `recipe/compare_with_backend.py` also sends `score`/`noul` cases, -which this worker rejects, so compare the Valen fixture directly: +The generic `recipe/compare_with_backend.py` also covers `score`/`noul` +cases; the byte-identical comparison below uses the Valen fixture directly: ~~~sh curl -fsS http://127.0.0.1:8000/v1/systemone \ @@ -282,7 +345,9 @@ cmp /tmp/valen-worker.json /tmp/valen-frontend.json The validated runs returned 200 from both endpoints with matching JSON content types and byte-identical response bodies, for the image fixture above and for -the text-only example. +the text-only example. The noul and score fixtures were re-checked the same +way on the same host: byte-identical through the frontend for all three +fixtures. ## Error behavior @@ -309,7 +374,7 @@ python recipe/valen/test_compare_reference.py Validated results: ~~~text -pytest tests/valen: 29 passed in 0.67s +pytest tests/valen: 47 passed in 1.21s (at 2192c27, before the export battery) compileall: passed cargo fmt --all --check: passed frontend integration: 12 passed; 0 failed @@ -335,6 +400,7 @@ max probability delta: 2.7810645453e-08 confidence delta: 5.5621290906e-08 decision agreement: PASS probability tolerance <= 1e-06: PASS +confidence tolerance <= 1e-06: PASS token accounting: PASS ~~~ @@ -351,18 +417,28 @@ max probability delta: 2.6866340819e-08 confidence delta: 5.3732681637e-08 decision agreement: PASS probability tolerance <= 1e-06: PASS +confidence tolerance <= 1e-06: PASS token accounting: PASS ~~~ The reference model field is `Valen`; the system1-omni worker deliberately uses `valen-preview-0923` as its stable serving identity. -Reproduce with [`compare_reference.py`](compare_reference.py). The script loads -one pinned model instance and drives each request through two paths: a -Valen-native record compiled by the pinned `Compiler` and decoded with a plain -softmax, and the production `frontend.valen.decide` pipeline. The decision -criteria are exact choice agreement, equal token accounting and the declared -probability tolerance (default `1e-6`, `--tolerance` to override): +Reproduce with [`compare_reference.py`](compare_reference.py). Both paths +share one pinned model instance: a Valen-native record compiled by the pinned +`Compiler` from the worker's own record rules, against the production +`frontend.valen.decide` pipeline; `noul` and `score` answers are decoded by +the pinned reference answer module (`valen.evaluation.inference.answer`). The +criteria are exact discrete agreement (candidate order, the requested answer +type, chosen key, score legend), equal token accounting, and the declared +tolerance (default `1e-6`, `--tolerance` to override) on probabilities, +confidences and the scalar answer fields (`noul` P(true), `score` expected +level). A non-finite probability, scalar answer or confidence on either side +fails the run before any tolerance verdict: `max()` keeps its previous finite +maximum against a NaN difference, so a NaN delta would otherwise leave every +tolerance check passing. An answer whose public `type` drifts from the +requested question type fails the run instead of skipping the noul/score +checks: ~~~sh PYTHONPATH=src .venv/bin/python recipe/valen/compare_reference.py \ @@ -372,11 +448,106 @@ PYTHONPATH=src .venv/bin/python recipe/valen/compare_reference.py \ PYTHONPATH=src .venv/bin/python recipe/valen/compare_reference.py \ --valen-source "$VALEN_SRC" --checkpoint "$VALEN_CKPT" \ --base "$VALEN_BASE" --device cuda --dtype bf16 --case text + +PYTHONPATH=src .venv/bin/python recipe/valen/compare_reference.py \ + --valen-source "$VALEN_SRC" --checkpoint "$VALEN_CKPT" \ + --base "$VALEN_BASE" --device cuda --dtype bf16 --case noul + +PYTHONPATH=src .venv/bin/python recipe/valen/compare_reference.py \ + --valen-source "$VALEN_SRC" --checkpoint "$VALEN_CKPT" \ + --base "$VALEN_BASE" --device cuda --dtype bf16 --case score ~~~ The image case uses [`example-request.json`](example-request.json); the text case uses the recipe's text example unless `--text-body` points elsewhere. -Both blocks above are verbatim output of the committed tool. +The `noul` and `score` cases use [`example-noul-request.json`](example-noul-request.json) +and [`example-score-request.json`](example-score-request.json) unless +`--noul-body`/`--score-body` point elsewhere; `--case all` runs all four in +one model load. Both blocks above are verbatim output of the committed tool; +the `noul` and `score` blocks are recorded in the integration pull request. + +## Export parity + +The exported artifact set ([`export.md`](export.md)) was compared against the +original pinned checkpoint stack with +[`verify_export_parity.py`](verify_export_parity.py) on the same host as the +records above. The two stacks are loaded serially, each in its own subprocess +(the ~6.6 GiB BF16 stack does not fit twice in 8 GiB VRAM): + +- original: the worker's own `ValenExecutor` build — pinned base, + PEFT-wrapped language model with the FP32 LoRA applied unmerged, trained + vision-top blocks, merger and head from `checkpoint.pt`; +- exported: the consumption path the export manifest documents — + `Qwen3_5ForConditionalGeneration.from_pretrained(weights/valen-export, + dtype=bf16, eager).model`, the language LoRA re-applied through the same + peft build the pinned reference uses (every language Linear wrapped at the + manifest-declared rank 32 / alpha 64, the 372 FP32 tensors of + `lora.safetensors` loaded by key, adapters kept FP32), plus the decision + head rebuilt from `head.safetensors` (DecisionHead 2048→256, FP32, no + bias), run through the pinned `Valen` class. + +Both stacks answer the text example and the image fixture compiled by the +pinned `Compiler`; the declared gate is exact answer agreement with +probability deltas ≤ 1e-6. The block below is the verbatim tool output +(device `cuda:0`, NVIDIA GeForce RTX 5060 Laptop GPU, sm_120, 7.96 GiB, +torch 2.11.0+cu128; recorded 2026-10-15): + +~~~text +case: text +logical tokens: 48 +compute tokens: 48 +max probability delta: 0.0000000000e+00 +answer agreement: PASS +probability tolerance <= 1e-06: PASS +token accounting: PASS + +case: image +logical tokens: 108 +compute tokens: 108 +max probability delta: 0.0000000000e+00 +answer agreement: PASS +probability tolerance <= 1e-06: PASS +token accounting: PASS + +device: cuda:0 (NVIDIA GeForce RTX 5060 Laptop GPU (sm_120, 7.96 GiB)) +parity slot filled: weights/valen-export/valen_export.json +verify_export_parity: ok +~~~ + +Per-candidate probabilities (original stack → exported stack, printed by the +tool's own stage runner): + +~~~text +text/refund: 0.5159630775 0.4840368927 -> 0.5159630775 0.4840368927 (both choose refund) +image/color: 0.4459854066 0.5540145636 -> 0.4459854066 0.5540145636 (both choose black) +~~~ + +Verdict: **the parity gate passes at bit level**. The unmerged export +answers both fixtures with probabilities identical to the original stack's +(max delta exactly 0.0 on every candidate), the discrete answers and token +accounting agree exactly, and the original side matches the committed +serving records above (0.5159631044… / 0.5540145914…) within the ~3e-8 +worker-vs-reference spread. Both stacks apply the LoRA through +the same peft 0.18.1 unmerged `Linear.forward` with FP32 adapters, and the +exported backbone differs from the reference's `from_pretrained` state only +in tensors that are bit-equal by construction (the trained vision-top and +merger replacements, and the 36 FP32→BF16 downcasts the reference's own +BF16 runtime performs at load), so the two forward graphs are numerically +identical. The verdict — per-case deltas, device, tolerance — is recorded +with `parity.status = "passed"` in +`weights/valen-export/valen_export.json`. + +Reproduce (the export itself per [`export.md`](export.md) first): + +~~~sh +PYTHONPATH=src .venv/bin/python recipe/valen/verify_export_parity.py \ + --valen-source "$VALEN_SRC" --checkpoint "$VALEN_CKPT" \ + --base "$VALEN_BASE" --export weights/valen-export --device cuda +~~~ + +The script validates the artifact set and its recorded source pins before +loading, refuses to overwrite an existing manifest parity record unless +`--force` is given, and exits 0 only when the gate passes. ## Troubleshooting and limits @@ -392,4 +563,4 @@ Both blocks above are verbatim output of the committed tool. - `/health` is exposed only after model loading and real warmup complete. - The memory and smoke-response observations are host-specific; they are not general accuracy or performance claims. -- Video, native execution, `noul`, and `score` remain outside this recipe. +- Video and native execution remain outside this recipe. diff --git a/recipe/valen/test_compare_reference.py b/recipe/valen/test_compare_reference.py index d124f09d..b0d5b752 100644 --- a/recipe/valen/test_compare_reference.py +++ b/recipe/valen/test_compare_reference.py @@ -3,7 +3,8 @@ Checks the pure comparison helpers without the pinned Valen source, model weights or a GPU: the reference confidence formula, data-URL decoding, the -reference record builders, and the PASS/FAIL verdicts of compare_case. +reference record builders (including the noul/score record rules), and the +PASS/FAIL verdicts of compare_case. python recipe/valen/test_compare_reference.py """ @@ -12,32 +13,36 @@ import base64 import hashlib +import json import sys import tempfile from pathlib import Path HERE = Path(__file__).resolve().parent sys.path.insert(0, str(HERE)) +sys.path.insert(0, str(HERE.parents[1] / "src")) import compare_reference as tool -def worker_response(probabilities, choice, confidence, input_tokens=17, compute_tokens=17): +def worker_response(answers, input_tokens=17, compute_tokens=17): return { "model": "valen-preview-0923", - "answers": { - "move": { - "type": "choice", - "choice": choice, - "probabilities": probabilities, - "confidence": confidence, - } - }, + "answers": answers, "usage": {"input_tokens": input_tokens, "output_tokens": 0}, "internal_usage": {"compute_tokens": compute_tokens}, } +def choice_answer(probabilities, choice, confidence): + return { + "type": "choice", + "choice": choice, + "probabilities": probabilities, + "confidence": confidence, + } + + REFERENCE = { "keys": {"move": ["up", "down"]}, "probabilities": {"move": [0.25, 0.75]}, @@ -80,7 +85,6 @@ def check(name: str, condition: bool, detail: str = "") -> None: pass else: check(f"rejects {bad!r}", False) - bad = None check("rejects non-PNG/JPEG data URLs", True) # 3. Reference record builders. @@ -102,19 +106,78 @@ def check(name: str, condition: bool, detail: str = "") -> None: "text record uses the compiler's plain-string state", record["request"]["state"] == "plain context" and record["request"]["questions"] is questions - and "assets" not in record, + and "assets" not in record ) + + # 3b. Reference question records reuse the worker's own record rules. + def request_body(question): + return { + "model": "valen-preview-0923", + "state": "The card was charged twice for one order.", + "questions": {"q": question}, + } + check( "reference questions map null labels to their keys", - tool.reference_questions( - {"questions": {"q": {"instructions": None, - "criteria": {"a": None, "b": "B"}}}} - )["q"]["criteria"] == {"a": "a", "b": "B"}, + tool.reference_questions(request_body( + {"type": "choice", "instructions": "Choose.", + "criteria": {"a": None, "b": "B"}}))["q"]["criteria"] == {"a": "a", "b": "B"}, + ) + check( + "noul without wording stays the reference's native noul record", + tool.reference_questions(request_body( + {"type": "noul", "instructions": "Was it duplicated?"}))["q"] + == {"type": "noul", "instructions": "Was it duplicated?"}, + ) + custom = tool.reference_questions(request_body( + {"type": "noul", "instructions": "Was it duplicated?", + "criteria": {"true": "Duplicated.", "false": None}}))["q"] + check( + "custom noul wording becomes a true/false choice record", + custom["type"] == "choice" + and custom["criteria"]["true"] == "Duplicated." + and custom["criteria"]["false"] == "False / 否：不满足问题中的条件。", + ) + check( + "score levels pass through as an ordered list", + tool.reference_questions(request_body( + {"type": "score", "instructions": "Grade.", + "criteria": ["low", "high"]}))["q"] + == {"type": "score", "instructions": "Grade.", "criteria": ["low", "high"]}, + ) + + # 3c. The committed noul and score fixtures build valid records. + noul_fixture = json.loads( + (HERE / "example-noul-request.json").read_text(encoding="utf-8") + ) + noul_records = tool.reference_questions(noul_fixture) + check( + "noul fixture builds one native and one worded record", + noul_records["duplicate"] == { + "type": "noul", + "instructions": "Was the card charged twice for one order?", + } + and noul_records["authorized"]["type"] == "choice" + and set(noul_records["authorized"]["criteria"]) == {"true", "false"}, + ) + score_fixture = json.loads( + (HERE / "example-score-request.json").read_text(encoding="utf-8") + ) + score_records = tool.reference_questions(score_fixture) + check( + "score fixture builds one three-level score record", + score_records["severity"]["type"] == "score" + and len(score_records["severity"]["criteria"]) == 3, ) # 4. A matching worker response passes every verdict. lines, ok = tool.compare_case( - "text", worker_response({"up": 0.25, "down": 0.75}, "down", 0.5), REFERENCE + "text", + worker_response( + {"move": choice_answer({"up": 0.25, "down": 0.75}, "down", 0.5)} + ), + REFERENCE, + {"move": "choice"}, ) report = "\n".join(lines) check( @@ -122,6 +185,7 @@ def check(name: str, condition: bool, detail: str = "") -> None: ok and "decision agreement: PASS" in report and "probability tolerance <= 1e-06: PASS" in report + and "confidence tolerance <= 1e-06: PASS" in report and "token accounting: PASS" in report and "input_tokens: 17 == 17" in report, report, @@ -130,14 +194,20 @@ def check(name: str, condition: bool, detail: str = "") -> None: # 5. A tiny delta stays inside the declared tolerance. lines, ok = tool.compare_case( "text", - worker_response({"up": 0.25 + 1e-9, "down": 0.75 - 1e-9}, "down", 0.5), + worker_response( + {"move": choice_answer({"up": 0.25 + 1e-9, "down": 0.75 - 1e-9}, "down", 0.5)} + ), REFERENCE, + {"move": "choice"}, ) check("delta within tolerance passes", ok, "\n".join(lines)) # 6. Each failure mode fails on its own. lines, ok = tool.compare_case( - "text", worker_response({"up": 0.6, "down": 0.4}, "up", 0.2), REFERENCE + "text", + worker_response({"move": choice_answer({"up": 0.6, "down": 0.4}, "up", 0.2)}), + REFERENCE, + {"move": "choice"}, ) check( "probability and decision mismatch fails", @@ -148,8 +218,25 @@ def check(name: str, condition: bool, detail: str = "") -> None: ) lines, ok = tool.compare_case( - "text", worker_response({"up": 0.25, "down": 0.75}, "down", 0.5, input_tokens=18), + "text", + worker_response({"move": choice_answer({"up": 0.25, "down": 0.75}, "down", 0.9)}), + REFERENCE, + {"move": "choice"}, + ) + check( + "confidence drift beyond tolerance fails", + not ok and "confidence tolerance <= 1e-06: FAIL" in "\n".join(lines), + "\n".join(lines), + ) + + lines, ok = tool.compare_case( + "text", + worker_response( + {"move": choice_answer({"up": 0.25, "down": 0.75}, "down", 0.5)}, + input_tokens=18, + ), REFERENCE, + {"move": "choice"}, ) check( "token mismatch fails", @@ -159,7 +246,12 @@ def check(name: str, condition: bool, detail: str = "") -> None: ) lines, ok = tool.compare_case( - "text", worker_response({"down": 0.75, "up": 0.25}, "down", 0.5), REFERENCE + "text", + worker_response( + {"move": choice_answer({"down": 0.75, "up": 0.25}, "down", 0.5)} + ), + REFERENCE, + {"move": "choice"}, ) check( "candidate order mismatch fails", @@ -167,6 +259,326 @@ def check(name: str, condition: bool, detail: str = "") -> None: "\n".join(lines), ) + # 6b. Non-finite probabilities fail before any delta or argmax work. + # A NaN difference would keep the previous finite maximum and the argmax + # would silently keep a NaN-free first candidate, so both sides are + # rejected up front instead of compared. + lines, ok = tool.compare_case( + "text", + worker_response( + {"move": choice_answer({"up": float("nan"), "down": 0.4}, "up", 0.5)} + ), + REFERENCE, + {"move": "choice"}, + ) + check( + "worker NaN probability fails", + not ok + and "non-finite probability" in "\n".join(lines) + and "decision agreement: FAIL" in "\n".join(lines), + "\n".join(lines), + ) + + nan_reference = { + "keys": {"move": ["up", "down"]}, + "probabilities": {"move": [float("nan"), 0.75]}, + "logical_tokens": 17, + "compute_tokens": 17, + } + lines, ok = tool.compare_case( + "text", + worker_response({"move": choice_answer({"up": 0.25, "down": 0.75}, "down", 0.5)}), + nan_reference, + {"move": "choice"}, + ) + check( + "reference NaN probability fails", + not ok and "non-finite probability" in "\n".join(lines), + "\n".join(lines), + ) + + inf_reference = { + "keys": {"move": ["up", "down"]}, + "probabilities": {"move": [float("inf"), 0.0]}, + "logical_tokens": 17, + "compute_tokens": 17, + } + lines, ok = tool.compare_case( + "text", + worker_response( + {"move": choice_answer({"up": float("inf"), "down": 0.0}, "up", 0.5)} + ), + inf_reference, + {"move": "choice"}, + ) + check( + "matching infinities on both sides still fail", + not ok and "non-finite probability" in "\n".join(lines), + "\n".join(lines), + ) + + # 6c. Non-finite scalar answers and confidences fail the same way: the + # per-type branches fold them into running maxima, and max() keeps its + # first argument against a NaN difference, so a NaN delta would + # otherwise leave the tolerance verdicts passing. + lines, ok = tool.compare_case( + "text", + worker_response( + {"move": choice_answer({"up": 0.25, "down": 0.75}, "down", float("nan"))} + ), + REFERENCE, + {"move": "choice"}, + ) + check( + "worker NaN confidence fails", + not ok + and "non-finite scalar answer or confidence" in "\n".join(lines) + and "decision agreement: FAIL" in "\n".join(lines), + "\n".join(lines), + ) + + # 7. noul answers compare P(true) with tolerance. + noul_reference = { + "keys": {"ask": ["true", "false"]}, + "probabilities": {"ask": [0.75, 0.25]}, + # The pinned decoder answers native noul with P(true) only; custom + # wording decodes as a choice pair instead. + "answers": {"ask": {"type": "noul", "noul": 0.75}}, + "logical_tokens": 17, + "compute_tokens": 17, + } + + def noul_answer(p_true, confidence): + return { + "type": "noul", + "noul": p_true, + "probabilities": {"true": p_true, "false": 1.0 - p_true}, + "confidence": confidence, + } + + lines, ok = tool.compare_case( + "noul", + worker_response({"ask": noul_answer(0.75, 0.5)}), + noul_reference, + {"ask": "noul"}, + ) + report = "\n".join(lines) + check( + "noul exact match passes with the answer tolerance", + ok + and "answer tolerance <= 1e-06: PASS" in report + and "legend agreement" not in report, + report, + ) + + lines, ok = tool.compare_case( + "noul", + worker_response({"ask": noul_answer(0.75 + 1e-4, 0.5)}), + noul_reference, + {"ask": "noul"}, + ) + check( + "noul answer drift beyond tolerance fails", + not ok and "answer tolerance <= 1e-06: FAIL" in "\n".join(lines), + "\n".join(lines), + ) + + # A NaN P(true) with finite probabilities must fail instead of being + # swallowed by the answer-delta maximum. + nan_noul = { + "type": "noul", + "noul": float("nan"), + "probabilities": {"true": 0.75, "false": 0.25}, + "confidence": 0.5, + } + lines, ok = tool.compare_case( + "noul", worker_response({"ask": nan_noul}), noul_reference, {"ask": "noul"} + ) + check( + "worker NaN noul answer fails", + not ok and "non-finite scalar answer or confidence" in "\n".join(lines), + "\n".join(lines), + ) + + # A noul request answered in choice shape must fail even when the + # candidate probabilities, decision and tokens all match. + drifted = { + "type": "choice", + "choice": "true", + "probabilities": {"true": 0.75, "false": 0.25}, + "confidence": 0.5, + } + lines, ok = tool.compare_case( + "noul", worker_response({"ask": drifted}), noul_reference, {"ask": "noul"} + ) + check( + "noul answered as choice fails", + not ok + and "decision agreement: FAIL" in "\n".join(lines) + and "answer type 'choice' != requested 'noul'" in "\n".join(lines), + "\n".join(lines), + ) + + choice_shaped = dict(noul_reference) + choice_shaped["answers"] = { + "ask": {"type": "choice", "probabilities": {"true": 0.75, "false": 0.25}, + "choice": "true", "confidence": 0.5} + } + lines, ok = tool.compare_case( + "noul", + worker_response({"ask": noul_answer(0.75, 0.5)}), + choice_shaped, + {"ask": "noul"}, + ) + check( + "custom-wording noul compares against the choice-shaped decode", + ok and "confidence delta: 0.0000000000e+00" in "\n".join(lines), + "\n".join(lines), + ) + + # 8. score answers compare the expected level, legend and confidence. + score_reference = { + "keys": {"grade": ["0", "1", "2"]}, + "probabilities": {"grade": [0.7, 0.2, 0.1]}, + "answers": { + "grade": { + "type": "score", + "probabilities": {"0": 0.7, "1": 0.2, "2": 0.1}, + "score": 0.4, + "legend": {"0": "No error", "1": "A minor error", "2": "A major error"}, + "confidence": 0.4, + } + }, + "logical_tokens": 17, + "compute_tokens": 51, + } + + def score_answer(score, legend, confidence): + return { + "type": "score", + "probabilities": {"0": 0.7, "1": 0.2, "2": 0.1}, + "score": score, + "legend": legend, + "confidence": confidence, + } + + legend = {"0": "No error", "1": "A minor error", "2": "A major error"} + lines, ok = tool.compare_case( + "score", + worker_response({"grade": score_answer(0.4, legend, 0.4)}, compute_tokens=51), + score_reference, + {"grade": "score"}, + ) + report = "\n".join(lines) + check( + "score exact match passes with legend agreement", + ok + and "answer tolerance <= 1e-06: PASS" in report + and "legend agreement: PASS" in report, + report, + ) + + wrong_legend = dict(legend, **{"2": "A catastrophic error"}) + lines, ok = tool.compare_case( + "score", + worker_response( + {"grade": score_answer(0.4, wrong_legend, 0.4)}, compute_tokens=51 + ), + score_reference, + {"grade": "score"}, + ) + check( + "score legend mismatch fails", + not ok + and "legend agreement: FAIL" in "\n".join(lines) + and "score legend mismatch" in "\n".join(lines), + "\n".join(lines), + ) + + lines, ok = tool.compare_case( + "score", + worker_response({"grade": score_answer(0.5, legend, 0.4)}, compute_tokens=51), + score_reference, + {"grade": "score"}, + ) + check( + "score expected-level drift fails the answer tolerance", + not ok and "answer tolerance <= 1e-06: FAIL" in "\n".join(lines), + "\n".join(lines), + ) + + # A score request answered in choice shape must fail even when the + # candidate probabilities, decision and tokens all match. + drifted_score = { + "type": "choice", + "choice": "1", + "probabilities": {"0": 0.7, "1": 0.2, "2": 0.1}, + "confidence": 0.4, + } + lines, ok = tool.compare_case( + "score", + worker_response({"grade": drifted_score}, compute_tokens=51), + score_reference, + {"grade": "score"}, + ) + check( + "score answered as choice fails", + not ok + and "decision agreement: FAIL" in "\n".join(lines) + and "answer type 'choice' != requested 'score'" in "\n".join(lines), + "\n".join(lines), + ) + + # A synthetic score reference without a reference confidence falls back + # to the worker's score concentration formula (0.4 here), not the choice + # formula (0.55) that would fail the confidence tolerance. + synthetic_score = { + "keys": {"grade": ["0", "1", "2"]}, + "probabilities": {"grade": [0.7, 0.2, 0.1]}, + "answers": {"grade": {"type": "score", "score": 0.4, "legend": legend}}, + "logical_tokens": 17, + "compute_tokens": 51, + } + lines, ok = tool.compare_case( + "score", + worker_response({"grade": score_answer(0.4, legend, 0.4)}, compute_tokens=51), + synthetic_score, + {"grade": "score"}, + ) + check( + "synthetic score without confidence uses the score formula", + ok and "confidence tolerance <= 1e-06: PASS" in "\n".join(lines), + "\n".join(lines), + ) + + # A NaN confidence carried by the reference decode must fail instead of + # being swallowed by the confidence-delta maximum. + nan_conf_reference = { + "keys": {"grade": ["0", "1", "2"]}, + "probabilities": {"grade": [0.7, 0.2, 0.1]}, + "answers": { + "grade": { + "type": "score", + "score": 0.4, + "legend": legend, + "confidence": float("nan"), + } + }, + "logical_tokens": 17, + "compute_tokens": 51, + } + lines, ok = tool.compare_case( + "score", + worker_response({"grade": score_answer(0.4, legend, 0.4)}, compute_tokens=51), + nan_conf_reference, + {"grade": "score"}, + ) + check( + "reference NaN confidence fails", + not ok and "non-finite scalar answer or confidence" in "\n".join(lines), + "\n".join(lines), + ) + failed = 0 for name, passed, detail in results: print(f" {'ok ' if passed else 'FAIL'} {name}") diff --git a/recipe/valen/test_export_reference.py b/recipe/valen/test_export_reference.py new file mode 100644 index 00000000..ed21626a --- /dev/null +++ b/recipe/valen/test_export_reference.py @@ -0,0 +1,23 @@ +#!/usr/bin/env python3 +"""Recipe entrypoint for the Valen export regression battery. + +Thin glue over the implementation in ``tests/valen/export_reference_checks.py`` +(the contract battery plus the env-gated artifact validation); see that +module for what runs and how to enable the artifact-gated phases. + + python recipe/valen/test_export_reference.py +""" + +from __future__ import annotations + +import sys +from pathlib import Path + +REPO_ROOT = Path(__file__).resolve().parents[2] +sys.path.insert(0, str(REPO_ROOT / "tests" / "valen")) + +from export_reference_checks import main # noqa: E402 + + +if __name__ == "__main__": + main() diff --git a/recipe/valen/verify_export_parity.py b/recipe/valen/verify_export_parity.py new file mode 100644 index 00000000..bab947f0 --- /dev/null +++ b/recipe/valen/verify_export_parity.py @@ -0,0 +1,595 @@ +#!/usr/bin/env python3 +"""Prove the exported Valen artifact set answers like the original checkpoint. + +Two model stacks are loaded serially — only one lives on the GPU at a time, +each in its own subprocess so the ~6.6 GiB of VRAM is fully released between +loads: + +- original: the pinned reference stack exactly as the worker builds it + (``ValenExecutor`` -> ``valen.modeling.model.build_model`` + + ``load_checkpoint``: pinned base, PEFT-wrapped language model with the + FP32 LoRA applied unmerged, trained vision-top blocks, merger and head + from ``checkpoint.pt``); +- exported: the consumption path the export manifest documents — + ``Qwen3_5ForConditionalGeneration.from_pretrained(weights/valen-export, + dtype=bf16, eager).model`` for the backbone, the language LoRA re-applied + through the same peft build the pinned reference uses (every language + Linear wrapped at the manifest-declared rank and alpha, the 372 FP32 + tensors of ``lora.safetensors`` loaded by key, adapters kept FP32), and + the decision head rebuilt from ``head.safetensors`` as the pinned + ``DecisionHead`` (2048 -> 256, FP32, no bias), all through the pinned + ``Valen`` class so the forward path is the reference's, not a second + invention. + +Both stacks answer the recipe's existing text and image ``choice`` fixtures +(``compare_reference.py``'s text example and ``example-request.json``, +compiled by the pinned ``Compiler`` through the worker's own record rules). +The gate requires exact answer agreement and a probability delta <= 1e-6 +(the tolerance the reference serving already declared). + +Finally it fills the manifest's ``parity`` slot in +``weights/valen-export/valen_export.json`` with the verdict (status, device, +cases, agreement, max probability delta, evidence pointer) — a reproducible +command output, never a hand edit — and re-validates the artifact set. + +Usage (from the repository root, on the GPU host): + + .venv/bin/python recipe/valen/verify_export_parity.py \ + --valen-source weights/Valen --checkpoint weights/Valen-Preview-0923 \ + --base weights/Qwen3.5-2B --export weights/valen-export --device cuda + +Exit status 0 means the gate passed; 1 means it failed. +""" + +from __future__ import annotations + +import argparse +import json +import math +import subprocess +import sys +from pathlib import Path +from tempfile import TemporaryDirectory + +HERE = Path(__file__).resolve().parent +sys.path.insert(0, str(HERE)) + +import compare_reference as reference # noqa: E402 (the recipe's fixture helpers) +import export_checkpoint as exporter # noqa: E402 + +PROBABILITY_TOLERANCE = 1e-6 +STAGE_SENTINEL = "VERIFY-PARITY-STAGE-RESULT " +PARITY_CASES = ("text", "image") + + +# --- the gate's verdict: one case answered by both stacks --- + + +def compare_case(case, original, exported, tolerance=PROBABILITY_TOLERANCE): + """Compare one fixture answered by the original and exported stacks. + + Each side carries ``questions`` (per qid: candidate ``keys`` in order and + ``probabilities``), the compiler's ``logical_tokens`` and + ``compute_tokens``. The gate requires the same candidate order, one + probability per candidate on both sides, exact answer (argmax) agreement, + equal token accounting and probability deltas within ``tolerance``. + """ + + problems = [] + exact = True + max_delta = 0.0 + original_questions = original["questions"] + exported_questions = exported["questions"] + for qid, record in original_questions.items(): + if qid not in exported_questions: + problems.append(f"{qid}: missing from the exported stack") + exact = False + continue + other = exported_questions[qid] + keys = list(record["keys"]) + if list(other["keys"]) != keys: + problems.append( + f"{qid}: candidate order {list(other['keys'])} != {list(keys)}" + ) + exact = False + continue + if ( + len(record["probabilities"]) != len(keys) + or len(other["probabilities"]) != len(keys) + ): + problems.append( + f"{qid}: {len(record['probabilities'])} original and " + f"{len(other['probabilities'])} exported probabilities " + f"for {len(keys)} candidates" + ) + exact = False + continue + # A NaN difference keeps the previous finite maximum and a NaN-free + # argmax list silently keeps its first candidate, so a non-finite + # probability on either side fails the case before any comparison. + if any( + not math.isfinite(value) + for values in (record["probabilities"], other["probabilities"]) + for value in values + ): + problems.append( + f"{qid}: non-finite probability (NaN or infinity) on the " + "original or exported side" + ) + exact = False + continue + for key, p_original, p_exported in zip( + keys, record["probabilities"], other["probabilities"] + ): + max_delta = max(max_delta, abs(p_original - p_exported)) + chosen = max(range(len(keys)), key=record["probabilities"].__getitem__) + other_chosen = max(range(len(keys)), key=other["probabilities"].__getitem__) + if chosen != other_chosen: + problems.append( + f"{qid}: original chose {keys[chosen]!r}, exported {keys[other_chosen]!r}" + ) + exact = False + for qid in sorted(set(exported_questions) - set(original_questions)): + problems.append(f"{qid}: not asked of the original stack") + exact = False + tokens_agree = ( + original["logical_tokens"] == exported["logical_tokens"] + and original["compute_tokens"] == exported["compute_tokens"] + ) + if not tokens_agree: + problems.append( + "token accounting: " + f"({original['logical_tokens']}, {original['compute_tokens']}) != " + f"({exported['logical_tokens']}, {exported['compute_tokens']})" + ) + tolerance_pass = max_delta <= tolerance + if not tolerance_pass: + problems.append( + f"probability delta {max_delta:.6e} exceeds tolerance {tolerance:g}" + ) + return { + "case": case, + "exact_answer_agreement": exact, + "max_probability_delta": max_delta, + "probability_tolerance_pass": tolerance_pass, + "tokens_agree": tokens_agree, + "logical_tokens": int(original["logical_tokens"]), + "compute_tokens": int(original["compute_tokens"]), + "passed": exact and tokens_agree and tolerance_pass, + "problems": problems, + } + + +# --- the manifest parity slot --- + + +def build_parity_record( + *, + status, + device, + hardware, + cases, + exact_answer_agreement, + max_probability_delta, + tolerance, + evidence, +): + """Assemble the record written into the manifest's ``parity`` slot.""" + + if status not in ("passed", "failed"): + raise ValueError(f"parity status must be 'passed' or 'failed', got {status!r}") + return { + "status": status, + "device": device, + "hardware": hardware, + "tolerance": tolerance, + "cases": cases, + "exact_answer_agreement": bool(exact_answer_agreement), + "max_probability_delta": max_probability_delta, + "evidence": evidence, + } + + +def fill_parity_slot(manifest_path, record, *, force=False): + """Write ``record`` into the manifest's ``parity`` slot. + + Refuses to silently replace a different existing record unless ``force`` + is set, keeps the manifest's documented formatting (sorted keys, indent + 2, trailing newline) and returns the updated manifest. + """ + + manifest_path = Path(manifest_path) + manifest = json.loads(manifest_path.read_text(encoding="utf-8")) + existing = manifest.get("parity") + if existing is not None and existing != record and not force: + raise ValueError( + "parity slot already holds a different record (pass force=True to replace)" + ) + manifest["parity"] = record + manifest_path.write_text( + json.dumps(manifest, indent=2, sort_keys=True, allow_nan=False) + "\n", + encoding="utf-8", + ) + return manifest + + +# --- the two model stacks --- + + +def load_original_stack(checkpoint, base, device): + """The reference stack exactly as the worker's executor builds it.""" + + sys.path.insert(0, str(HERE.parents[1] / "src")) + from models.valen.engine import ValenExecutor + + executor = ValenExecutor(Path(checkpoint), Path(base), device, "bf16") + return executor + + +def load_exported_stack(export_dir, manifest, device): + """The exported artifact set loaded through its documented consumption + path: the ``from_pretrained`` backbone, the language LoRA re-applied + through the same peft build the pinned reference uses, and the head + rebuilt from ``head.safetensors`` into the pinned ``Valen`` class.""" + + import torch + import torch.nn as nn + from peft import LoraConfig, get_peft_model + from safetensors import safe_open + from transformers import Qwen3_5ForConditionalGeneration + + from valen.modeling.model import Valen + + container, loading_info = Qwen3_5ForConditionalGeneration.from_pretrained( + str(export_dir), + dtype=torch.bfloat16, + attn_implementation="eager", + local_files_only=True, + output_loading_info=True, + ) + if any( + loading_info.get(key) + for key in ("missing_keys", "unexpected_keys", "mismatched_keys", "error_msgs") + ): + raise ValueError(f"Incomplete exported model load: {loading_info}") + backbone = container.model + # Only container.model is served; drop the vocabulary/generation wrapper. + del container + backbone.requires_grad_(False) + + # The pinned build path's own adaptation: every language Linear wrapped + # at the manifest-declared rank and alpha, adapters kept fp32 at runtime. + lora = manifest["lora"] + targets = [ + name + for name, module in backbone.language_model.named_modules() + if isinstance(module, nn.Linear) + ] + if len(targets) != lora["targets"]: + raise ValueError( + f"backbone enumerates {len(targets)} language linears, the manifest " + f"declares {lora['targets']}" + ) + backbone.language_model = get_peft_model( + backbone.language_model, + LoraConfig( + r=lora["rank"], + lora_alpha=lora["alpha"], + lora_dropout=0.0, + target_modules=targets, + bias="none", + ), + ) + model = Valen(backbone, manifest["head"]["projection_dim"]) + with safe_open(Path(export_dir) / lora["file"], framework="pt") as handle: + adapters = {key: handle.get_tensor(key) for key in sorted(handle.keys())} + expected = {name for name, _ in model.named_parameters() if "lora_" in name} + if set(adapters) != expected: + raise ValueError( + "lora.safetensors keys do not match the wrapped model's adapters: " + f"file-only {sorted(set(adapters) - expected)[:2]}, " + f"model-only {sorted(expected - set(adapters))[:2]}" + ) + model.load_state_dict(adapters, strict=False) + with safe_open(Path(export_dir) / exporter.HEAD_FILE, framework="pt") as handle: + head_state = { + key[len(exporter.HEAD_PREFIX):]: handle.get_tensor(key) + for key in sorted(handle.keys()) + } + model.head.load_state_dict(head_state) + model.to(device) + model.eval() + return model + + +# --- answering the fixtures through one loaded stack --- + + +def choice_fixtures(): + """The recipe's existing text and image ``choice`` fixtures.""" + + return ( + ("text", reference.DEFAULT_TEXT_BODY, "compare_reference.py DEFAULT_TEXT_BODY"), + ( + "image", + json.loads((HERE / "example-request.json").read_text(encoding="utf-8")), + "recipe/valen/example-request.json", + ), + ) + + +def run_fixtures(model, processor, max_length, media_kwargs): + """Answer every fixture with one loaded stack; return per-case results.""" + + import torch + + from valen.data.compiler import Compiler + + cases = {} + with torch.no_grad(): + for name, body, fixture in choice_fixtures(): + questions = reference.reference_questions(body, processor.special_tokens) + with TemporaryDirectory(prefix="valen-parity-") as directory: + media_root = Path(directory) + if name == "image": + image_bytes, image_format = reference.decode_image_data_url( + body["state"]["image"] + ) + record = reference.reference_image_record( + image_bytes, image_format, questions, media_root + ) + else: + record = reference.reference_text_record(body["state"], questions) + compiler = Compiler( + processor.processor, + media_root=media_root, + max_length=max_length, + media_kwargs=media_kwargs, + ) + state = compiler.compile(record) + answered = {} + for question in state.questions: + logits = model(question).float().cpu() + answered[question.qid] = { + "keys": list(question.keys), + "probabilities": torch.softmax(logits, dim=-1).tolist(), + } + cases[name] = { + "fixture": fixture, + "questions": answered, + "logical_tokens": int(state.logical_tokens), + "compute_tokens": int(state.compute_tokens), + } + return cases + + +def stage_environment(args): + """Verify the pinned source and build the shared compile-side state.""" + + sys.path.insert(0, str(HERE.parents[1] / "src")) + from frontend.valen import _verify_source_revision + + source = Path(args.valen_source).resolve() + _verify_source_revision(source) + reference._verify_clean_checkout(source) + sys.path.insert(0, str(source)) + + from models.valen.preprocess import ValenProcessor + + config = json.loads( + (Path(args.checkpoint) / "config.json").read_text(encoding="utf-8") + ) + processor = ValenProcessor( + Path(args.base), int(config["max_length"]), config.get("media_kwargs") + ) + return processor, int(config["max_length"]), config.get("media_kwargs") or {} + + +def run_stage(args): + """Load one stack in this (sub)process, answer the fixtures, emit JSON.""" + + import torch + + torch.set_grad_enabled(False) + processor, max_length, media_kwargs = stage_environment(args) + + if args.stage == "original": + executor = load_original_stack(args.checkpoint, args.base, args.device) + model = executor.model + else: + manifest = json.loads( + (Path(args.export) / exporter.MANIFEST_FILE).read_text(encoding="utf-8") + ) + model = load_exported_stack(Path(args.export), manifest, args.device) + + device = str(next(model.parameters()).device) + hardware = None + if device.startswith("cuda"): + index = torch.device(device).index or 0 + capability = torch.cuda.get_device_capability(index) + _, total = torch.cuda.mem_get_info(index) + hardware = ( + f"{torch.cuda.get_device_name(index)} " + f"(sm_{capability[0]}{capability[1]}, {total / (1024 ** 3):.2f} GiB)" + ) + try: + cases = run_fixtures(model, processor, max_length, media_kwargs) + finally: + if args.stage == "original": + executor.close() + del model + if device.startswith("cuda"): + torch.cuda.empty_cache() + print( + STAGE_SENTINEL + + json.dumps( + { + "stage": args.stage, + "device": device, + "hardware": hardware, + "torch": torch.__version__, + "cases": cases, + }, + allow_nan=False, + ) + ) + return 0 + + +def run_stage_subprocess(args, stage): + """Run one stack in its own subprocess so VRAM is fully released.""" + + command = [ + sys.executable, + str(Path(__file__).resolve()), + "--valen-source", args.valen_source, + "--checkpoint", args.checkpoint, + "--base", args.base, + "--export", args.export, + "--device", args.device, + "--stage", stage, + ] + result = subprocess.run(command, capture_output=True, text=True, timeout=3600) + if result.returncode != 0: + raise RuntimeError( + f"{stage} stage failed with exit {result.returncode}:\n" + f"{result.stdout[-2000:]}\n{result.stderr[-4000:]}" + ) + payload = None + for line in result.stdout.splitlines(): + if line.startswith(STAGE_SENTINEL): + payload = json.loads(line[len(STAGE_SENTINEL):]) + if payload is None: + raise RuntimeError(f"{stage} stage produced no result: {result.stdout[-2000:]}") + return payload + + +# --- reporting --- + + +def format_case_report(verdict, tolerance): + lines = [f"case: {verdict['case']}"] + lines.append(f"logical tokens: {verdict['logical_tokens']}") + lines.append(f"compute tokens: {verdict['compute_tokens']}") + lines.append(f"max probability delta: {verdict['max_probability_delta']:.10e}") + lines.append( + "answer agreement: " + ("PASS" if verdict["exact_answer_agreement"] else "FAIL") + ) + lines.append( + f"probability tolerance <= {tolerance:g}: " + + ("PASS" if verdict["probability_tolerance_pass"] else "FAIL") + ) + lines.append("token accounting: " + ("PASS" if verdict["tokens_agree"] else "FAIL")) + for problem in verdict["problems"]: + lines.append(f"problem: {problem}") + return lines + + +def evidence_pointer(): + return ( + "recipe/valen/reference.md (export parity block); reproduce with " + "recipe/valen/verify_export_parity.py --valen-source $VALEN_SRC " + "--checkpoint $VALEN_CKPT --base $VALEN_BASE --export --device cuda" + ) + + +def parse_args(argv=None): + parser = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + parser.add_argument("--valen-source", required=True, help="pinned Valen source checkout") + parser.add_argument("--checkpoint", required=True, help="Valen-Preview-0923 directory") + parser.add_argument("--base", required=True, help="Qwen3.5-2B base directory") + parser.add_argument("--export", required=True, type=Path, help="exported artifact directory") + parser.add_argument("--device", default="cuda") + parser.add_argument("--tolerance", type=float, default=PROBABILITY_TOLERANCE) + parser.add_argument("--stage", choices=("original", "export"), default=None, help=argparse.SUPPRESS) + parser.add_argument("--force", action="store_true", help="replace an existing parity record") + return parser.parse_args(argv) + + +def verify_sources_match_manifest(args, manifest): + """The parity evidence only holds for the artifacts the export recorded. + + Verifies the checkpoint sha, the base revision and — because 578 tensors + pass through the export byte by byte — the base weights shard digests + themselves against the manifest's recorded ``sources.base.weights_sha256``. + """ + + checkpoint_sha = exporter.sha256_file(Path(args.checkpoint) / exporter.CHECKPOINT_FILE) + recorded = manifest["sources"]["checkpoint"]["sha256"] + if recorded != checkpoint_sha: + raise ValueError( + f"the export was built from checkpoint sha256 {recorded}, but " + f"--checkpoint hashes {checkpoint_sha}" + ) + base_manifest = json.loads( + (Path(args.base) / exporter.BASE_MANIFEST_FILE).read_text(encoding="utf-8") + ) + revision = manifest["sources"]["base"]["revision"] + if base_manifest.get("revision") != revision: + raise ValueError( + f"the export was built from base revision {revision}, but --base " + f"pins {base_manifest.get('revision')}" + ) + weights_file = exporter.base_weights_file(Path(args.base)) + shard_sums = {weights_file.name: exporter.sha256_file(weights_file)} + recorded_sums = manifest["sources"]["base"]["weights_sha256"] + if recorded_sums != shard_sums: + raise ValueError( + f"the export was built from base weights sha256 {recorded_sums}, " + f"but --base hashes {shard_sums}" + ) + + +def main(argv=None): + args = parse_args(argv) + if args.stage is not None: + return run_stage(args) + + export_dir = Path(args.export) + manifest = exporter.validate_export(export_dir) + verify_sources_match_manifest(args, manifest) + + original = run_stage_subprocess(args, "original") + exported = run_stage_subprocess(args, "export") + verdicts = [ + compare_case(name, original["cases"][name], exported["cases"][name], args.tolerance) + for name in PARITY_CASES + ] + + status = "passed" if all(verdict["passed"] for verdict in verdicts) else "failed" + record = build_parity_record( + status=status, + device=original["device"], + hardware=original["hardware"], + cases=[ + { + "case": verdict["case"], + "fixture": original["cases"][verdict["case"]]["fixture"], + "exact_answer_agreement": verdict["exact_answer_agreement"], + "max_probability_delta": verdict["max_probability_delta"], + "logical_tokens": verdict["logical_tokens"], + "compute_tokens": verdict["compute_tokens"], + "passed": verdict["passed"], + "problems": verdict["problems"], + } + for verdict in verdicts + ], + exact_answer_agreement=all(v["exact_answer_agreement"] for v in verdicts), + max_probability_delta=max(v["max_probability_delta"] for v in verdicts), + tolerance=args.tolerance, + evidence=evidence_pointer(), + ) + fill_parity_slot(export_dir / exporter.MANIFEST_FILE, record, force=args.force) + exporter.validate_export(export_dir) + + for verdict in verdicts: + print("\n".join(format_case_report(verdict, args.tolerance))) + print() + print(f"device: {record['device']} ({record['hardware']})") + print(f"parity slot filled: {export_dir / exporter.MANIFEST_FILE}") + print("verify_export_parity: ok" if status == "passed" else "verify_export_parity: FAILED") + return 0 if status == "passed" else 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/src/models/valen/README.md b/src/models/valen/README.md index 8d107ce5..6df70240 100644 --- a/src/models/valen/README.md +++ b/src/models/valen/README.md @@ -1,9 +1,10 @@ # Valen-Preview-0923 reference contract -This directory contains the first System1-Omni integration slice for +This directory contains the System1-Omni reference integration for Valen-Preview-0923. It is a reference-serving boundary only: the Python worker, -processor, executor, and postprocessor are implemented for the pinned text and -single-image `choice` slice. Native CUDA execution is not part of this slice. +processor, executor, and postprocessor serve the pinned text and single-image +`choice`, `noul`, and `score` questions. Native CUDA execution is not part of +it. ## Pinned artifacts @@ -29,9 +30,14 @@ does not contain the frozen Qwen base model. The published config uses criteria are limited to 16,384 characters; - `instructions` must be non-empty after trimming; criterion keys and texts must be non-empty after trimming (a null criterion text defaults to its key); -- `choice` questions only, at most 8 per request; -- up to 255 named candidates per question; -- no `noul`, `score`, video, multi-image, or remote URL inputs; +- `choice`, `noul` and `score` questions, at most 8 per request, freely mixed + against one shared state; +- `choice`: up to 255 named candidates per question; +- `noul`: fixed `true`/`false` candidates; `criteria` is optional and may + replace either side's wording (`{"true": ..., "false": ...}`, null keeps + the reference's fixed wording); +- `score`: an ordered list of 2 to 10 level descriptions in `criteria`; +- no video, multi-image, or remote URL inputs; - no token generation: the decision head returns candidate logits directly. The public request keeps the Cua-S1-compatible shape: @@ -69,12 +75,30 @@ A text-only request replaces the image object with plain text: Preprocessing validates and decodes the data URL, writes it to a request-scoped directory, hashes the bytes, and converts it to Valen's local-media `messages` record. Text state skips materialization and compiles a text-only -user message with no assets. The worker owns that directory's lifetime. Postprocessing -uses Valen's original choice confidence formula for this reference slice: +user message with no assets. The worker owns that directory's lifetime. + +The pinned compiler builds every question type from the same record family: + +- `choice` and `noul` compile to one branch holding all candidates; +- `score` compiles one isolated branch per level — each branch sees only its + own level description, never the level ID or the other levels — and the K + per-level decision logits are softmaxed across branches; +- `noul` without custom wording is the reference's native noul record with + the schema's fixed candidate wording; custom wording cannot flow through + it, so it compiles as the equivalent `choice` record keyed `true`/`false`, + with unspecified sides filled from the same fixed wording — `P(true)` still + comes from the two-candidate softmax. + +Postprocessing keeps Valen's original formulas: ~~~text -K = 1: confidence = 1 -K > 1: confidence = max(0, (max(p) - 1/K) / (1 - 1/K)) +choice/noul confidence (n candidates): + n = 1: 1 + n > 1: max(0, (max(p) - 1/n) / (1 - 1/n)) # noul is the n = 2 case +score confidence (K ordered levels, mode = argmax p): + distance = sum_i p_i * |i - mode| + uniform_mad = (sum_i |i - (K-1)/2|) / K + confidence = max(0, 1 - distance / uniform_mad) ~~~ This confidence choice is intentionally temporary. The shared `/v1/systemone` @@ -82,17 +106,29 @@ contract, including confidence semantics, error details, busy behavior, model identity, and usage fields, remains coordinated with [system1-omni#61](https://github.com/ThinkFlowLab/system1-omni/issues/61). -Each answer is `{"type": "choice", "choice": ..., "probabilities": ..., -"confidence": ...}`: `choice` is the candidate with the highest probability, -ties go to the earliest candidate in request order, and `probabilities` maps -every candidate key to its probability in request order. +Each `choice` answer is `{"type": "choice", "choice": ..., +"probabilities": ..., "confidence": ...}`: `choice` is the candidate with the +highest probability, ties go to the earliest candidate in request order, and +`probabilities` maps every candidate key to its probability in request order. + +Each `noul` answer is `{"type": "noul", "noul": ..., +"probabilities": ..., "confidence": ...}`: `noul` is `P(true)`, the reference +answer module's own field; the pair and confidence are this worker's +extension (the reference answer carries only `P(true)`). + +Each `score` answer is `{"type": "score", "probabilities": ..., +"score": ..., "legend": ..., "confidence": ...}`: `probabilities` maps the +level keys `"0"`… to their cross-branch softmax probability, `score` is the +expected level `sum(i * p_i)`, and `legend` maps each level key back to its +request-order description. The response `model` is `valen-preview-0923`, the worker's stable serving identity. `usage.input_tokens` counts the compiled prompt's logical tokens: the shared state tokens plus each question's task suffix, counted once. `usage.output_tokens` is 0; the decision head returns logits without generation. `internal_usage.compute_tokens` counts the full per-question -branch lengths the executor runs, with the shared state repeated per branch. +branch lengths the executor runs, with the shared state repeated per branch — +a K-level `score` question therefore counts K branches. ## Errors @@ -111,14 +147,17 @@ and `503` that another inference is running. The status is `422` when a well-formed request cannot be evaluated: - a `model` other than `valen-preview-0923`; -- a `score` or `noul` question, or unsupported request or question fields; +- a question whose `type` is not `choice`, `noul` or `score`, or unsupported + request or question fields; - text over its limit, or containing a tokenizer control token — the media tokens such as `<|image_pad|>` at minimum, plus the loaded tokenizer's full special-token set (for example `<|im_end|>`) when the worker is serving; - this guard covers text state, instructions, criterion texts and option keys - (candidate names enter the prompt); -- empty or whitespace-only `instructions`, option keys, criterion texts, or - text state; + this guard covers text state, instructions, criterion texts, level + descriptions and option keys (candidate names enter the prompt); +- empty or whitespace-only `instructions`, option keys, criterion texts, + noul descriptions, score level descriptions, or text state; +- `noul` criteria that name anything other than `true`/`false`, or `score` + criteria that are not an ordered list of 2 to 10 level descriptions; - an empty text state, or a state that is neither an image data URL nor text; - an image that is not a single-frame PNG/JPEG matching its MIME type and limits; @@ -130,6 +169,15 @@ The status is `422` when a well-formed request cannot be evaluated: Any other failure is `500` with a generic detail and no request content in the log (the exception type only). +## Checkpoint export + +The pinned checkpoint and base also export to a native-loadable +safetensors artifact set — the language LoRA served separately as unmerged +FP32 adapters, vision-top and merger replaced, FP32 decision head — with +every transformation recorded in a JSON +manifest: see [recipe/valen/export.md](../../../recipe/valen/export.md). +Consuming that export does not change this worker. + ## Ownership boundary - `protocol.py` owns wire validation and safe inline-image limits. diff --git a/src/models/valen/postprocess.py b/src/models/valen/postprocess.py index 08d0c6f7..c739d0d5 100644 --- a/src/models/valen/postprocess.py +++ b/src/models/valen/postprocess.py @@ -23,28 +23,67 @@ def _probabilities(logits: Sequence[float], temperature: float) -> list[float]: return [value / total for value in exponentials] +def _choice_confidence(probabilities: Sequence[float]) -> float: + """Valen's original choice confidence formula (reference answer module).""" + + n = len(probabilities) + if n == 1: + return 1.0 + return max(0.0, (max(probabilities) - 1 / n) / (1 - 1 / n)) + + +def _noul_answer( + question: Question, probabilities: Sequence[float] +) -> dict[str, Any]: + """The reference's noul answer (P(true)) plus the full candidate pair.""" + + pair = dict(zip(question.keys, probabilities)) + return { + "type": "noul", + "noul": pair["true"], + "probabilities": pair, + "confidence": _choice_confidence(probabilities), + } + + +def _score_answer( + question: Question, probabilities: Sequence[float] +) -> dict[str, Any]: + """Valen's pinned score decoding: expected level and concentration.""" + + mode = max(range(len(probabilities)), key=probabilities.__getitem__) + count = len(probabilities) + distance = sum(p * abs(i - mode) for i, p in enumerate(probabilities)) + uniform_mad = sum(abs(i - (count - 1) / 2) for i in range(count)) / count + return { + "type": "score", + "probabilities": dict(zip(question.keys, probabilities)), + "score": sum(i * p for i, p in enumerate(probabilities)), + "legend": dict(zip(question.keys, question.descriptions)), + "confidence": max(0.0, 1.0 - distance / uniform_mad), + } + + def answer( question: Question, logits: Sequence[float], temperature: float = 1.0, ) -> dict[str, Any]: - """Match Valen's pinned choice decoding and confidence formula.""" + """Match Valen's pinned decoding and confidence formulas per type.""" if len(logits) != len(question.keys): - raise ValueError("model returned one logit per candidate") + raise ValueError("model must return one logit per candidate") probabilities = _probabilities(logits, temperature) + if question.kind == "noul": + return _noul_answer(question, probabilities) + if question.kind == "score": + return _score_answer(question, probabilities) mode = max(range(len(probabilities)), key=probabilities.__getitem__) - n = len(probabilities) - confidence = ( - 1.0 - if n == 1 - else max(0.0, (max(probabilities) - 1 / n) / (1 - 1 / n)) - ) return { "type": "choice", "choice": question.keys[mode], "probabilities": dict(zip(question.keys, probabilities)), - "confidence": confidence, + "confidence": _choice_confidence(probabilities), } diff --git a/src/models/valen/preprocess.py b/src/models/valen/preprocess.py index e0fec410..38e60363 100644 --- a/src/models/valen/preprocess.py +++ b/src/models/valen/preprocess.py @@ -13,7 +13,14 @@ from pathlib import Path from typing import Any -from .protocol import DEFAULT_SPECIAL_TOKENS, Question, Request, TextState +from .protocol import ( + DEFAULT_SPECIAL_TOKENS, + NOUL_FALSE_DESCRIPTION, + NOUL_TRUE_DESCRIPTION, + Question, + Request, + TextState, +) @dataclass(frozen=True) @@ -83,6 +90,47 @@ def compile(self, prepared: PreparedInput) -> CompiledInput: ) +def question_record(question: Question) -> dict[str, Any]: + """Build the pinned compiler's question record for one wire question. + + ``noul`` without custom wording compiles as the reference's native noul + record, whose candidate descriptions the pinned schema fixes. Custom + true/false wording cannot flow through a noul record (the pinned + ``candidates`` ignores noul criteria), so it compiles as a choice record + keyed ``true``/``false`` with the unspecified sides filled from the + reference's fixed wording — the same single-branch, two-candidate + structure. ``score`` passes its ordered level descriptions straight + through; the pinned schema assigns the level keys ``"0"``… + """ + + if question.kind == "noul": + descriptions = [ + description if description is not None else fixed + for description, fixed in zip( + question.descriptions, + (NOUL_TRUE_DESCRIPTION, NOUL_FALSE_DESCRIPTION), + ) + ] + if tuple(descriptions) == (NOUL_TRUE_DESCRIPTION, NOUL_FALSE_DESCRIPTION): + return {"type": "noul", "instructions": question.instructions} + return { + "type": "choice", + "instructions": question.instructions, + "criteria": dict(zip(question.keys, descriptions)), + } + if question.kind == "score": + return { + "type": "score", + "instructions": question.instructions, + "criteria": list(question.descriptions), + } + return { + "type": "choice", + "instructions": question.instructions, + "criteria": dict(zip(question.keys, question.descriptions)), + } + + def prepare_request(request: Request, media_root: Path) -> PreparedInput: """Materialize one validated request into the pinned Valen record format.""" @@ -122,12 +170,7 @@ def prepare_request(request: Request, media_root: Path) -> PreparedInput: } assets = [{"path": image_path.name, "sha256": image_sha256}] criteria = { - question.name: { - "type": "choice", - "instructions": question.instructions, - "criteria": dict(zip(question.keys, question.descriptions)), - } - for question in request.questions + question.name: question_record(question) for question in request.questions } record = { "request": {"state": state, "questions": criteria}, diff --git a/src/models/valen/protocol.py b/src/models/valen/protocol.py index 512b6190..7122104b 100644 --- a/src/models/valen/protocol.py +++ b/src/models/valen/protocol.py @@ -25,6 +25,17 @@ MAX_QUESTIONS = 8 MAX_OPTIONS = 255 MAX_TEXT = 16384 +MIN_SCORE_LEVELS = 2 +MAX_SCORE_LEVELS = 10 + +QUESTION_TYPES = ("choice", "noul", "score") + +# Verbatim from the pinned valen.data.schema.candidates noul wording at +# revision 750bfcfbb48a5275534a9c912257ebe83ca57a97; the compiler emits these +# descriptions for every noul question, so the worker reuses them whenever the +# wire leaves a side unspecified. +NOUL_TRUE_DESCRIPTION = "True / 是：满足问题中的条件。" +NOUL_FALSE_DESCRIPTION = "False / 否：不满足问题中的条件。" DEFAULT_SPECIAL_TOKENS = ( "<|image_pad|>", @@ -68,10 +79,19 @@ class TextState: @dataclass(frozen=True) class Question: + """One validated wire question. + + ``keys`` are the candidate names the pinned compiler assigns: wire option + keys for ``choice``, ``true``/``false`` for ``noul``, and the level + indices ``"0"``.. for ``score``. A ``None`` description marks a ``noul`` + side left to the reference's fixed wording. + """ + name: str instructions: str keys: tuple[str, ...] - descriptions: tuple[str, ...] + descriptions: tuple[str | None, ...] + kind: str = "choice" @dataclass(frozen=True) @@ -225,8 +245,11 @@ def parse_request(value: dict[str, Any], special_tokens=DEFAULT_SPECIAL_TOKENS) "criteria", }: raise RequestError(f"question {name!r} contains unsupported fields") - if question.get("type") != "choice": - raise RequestError(f"question {name!r}: only choice is supported") + kind = question.get("type") + if kind not in QUESTION_TYPES: + raise RequestError( + f"question {name!r}: type must be one of {', '.join(QUESTION_TYPES)}" + ) if "instructions" not in question: raise RequestError(f"question {name!r}: instructions is required") instructions = _text( @@ -234,34 +257,83 @@ def parse_request(value: dict[str, Any], special_tokens=DEFAULT_SPECIAL_TOKENS) ) if not instructions.strip(): raise RequestError(f"question {name!r}: instructions must be nonempty") - criteria = question.get("criteria") - if not isinstance(criteria, dict) or not 1 <= len(criteria) <= MAX_OPTIONS: - raise RequestError( - f"question {name!r}: choice requires 1 to {MAX_OPTIONS} options" - ) keys: list[str] = [] - descriptions: list[str] = [] - for key, label in criteria.items(): - if not isinstance(key, str) or not key.strip() or len(key) > 256: - raise RequestError("option keys must be nonempty and at most 256 characters") - if any(token in key for token in special_tokens): - # Candidate names enter the prompt, so keys get the same - # control-token guard as instructions and descriptions. + descriptions: list[str | None] = [] + if kind == "noul": + keys = ["true", "false"] + criteria = question.get("criteria") + if criteria is not None: + if not isinstance(criteria, dict) or not criteria: + raise RequestError( + f"question {name!r}: noul criteria must describe true and/or false" + ) + if set(criteria) - {"true", "false"}: + raise RequestError( + f"question {name!r}: noul criteria must describe true and/or false" + ) + for key in ("true", "false"): + label = (criteria or {}).get(key) + # None keeps the reference's fixed candidate wording; the + # preprocess layer substitutes it when building the record. + description = None + if label is not None: + description = _text( + label, f"question {name!r} criteria {key!r}", special_tokens + ) + if not description.strip(): + raise RequestError( + f"question {name!r}: criterion {key!r} must be nonempty" + ) + descriptions.append(description) + elif kind == "score": + criteria = question.get("criteria") + if ( + not isinstance(criteria, list) + or not MIN_SCORE_LEVELS <= len(criteria) <= MAX_SCORE_LEVELS + ): raise RequestError( - f"question {name!r}: option key {key!r} contains an" - " unsupported tokenizer control token" + f"question {name!r}: score requires an ordered list of" + f" {MIN_SCORE_LEVELS} to {MAX_SCORE_LEVELS} level descriptions" ) - keys.append(key) - description = ( - key - if label is None - else _text(label, f"question {name!r} criteria {key!r}", special_tokens) - ) - if not description.strip(): - raise RequestError(f"question {name!r}: criterion {key!r} must be nonempty") - descriptions.append(description) - if len(instructions) + sum(map(len, descriptions)) > MAX_TEXT: + keys = [str(index) for index in range(len(criteria))] + for index, label in enumerate(criteria): + description = _text( + label, f"question {name!r} level {index}", special_tokens + ) + if not description.strip(): + raise RequestError( + f"question {name!r}: level {index} must be nonempty" + ) + descriptions.append(description) + else: + criteria = question.get("criteria") + if not isinstance(criteria, dict) or not 1 <= len(criteria) <= MAX_OPTIONS: + raise RequestError( + f"question {name!r}: choice requires 1 to {MAX_OPTIONS} options" + ) + for key, label in criteria.items(): + if not isinstance(key, str) or not key.strip() or len(key) > 256: + raise RequestError("option keys must be nonempty and at most 256 characters") + if any(token in key for token in special_tokens): + # Candidate names enter the prompt, so keys get the same + # control-token guard as instructions and descriptions. + raise RequestError( + f"question {name!r}: option key {key!r} contains an" + " unsupported tokenizer control token" + ) + keys.append(key) + description = ( + key + if label is None + else _text(label, f"question {name!r} criteria {key!r}", special_tokens) + ) + if not description.strip(): + raise RequestError(f"question {name!r}: criterion {key!r} must be nonempty") + descriptions.append(description) + if len(instructions) + sum(len(d) for d in descriptions if d is not None) > MAX_TEXT: raise RequestError("combined question text exceeds 16384 characters") - parsed.append(Question(name, instructions, tuple(keys), tuple(descriptions))) + parsed.append( + Question(name, instructions, tuple(keys), tuple(descriptions), kind) + ) return Request(MODEL_NAME, _state(value["state"], special_tokens), tuple(parsed)) diff --git a/tests/valen/export_reference_checks.py b/tests/valen/export_reference_checks.py new file mode 100644 index 00000000..b97de101 --- /dev/null +++ b/tests/valen/export_reference_checks.py @@ -0,0 +1,158 @@ +"""Regression wiring for the Valen checkpoint export. + +Chains the export contract tests in ``tests/valen/test_export.py`` — the +CPU-only battery plus the env-gated real-artifact determinism gate — and +validates an exported artifact set through the export tool's own +``validate_export``, without duplicating a single assertion: every check +lives in the test module or the tool, and this module only orchestrates. +The recipe entrypoint (``recipe/valen/test_export_reference.py``) is thin +glue over ``main``. + +Importing it runs no tests; the recipe script (or an explicit ``main()``) +drives the battery: + + python recipe/valen/test_export_reference.py + +Without the pinned artifacts it still runs the CPU battery and prints a +clear skip line for each artifact-gated phase, exiting 0. With +``VALEN_CKPT`` and ``VALEN_BASE`` set (see ``recipe/valen/reference.md``, +"Pinned artifacts") the determinism gate runs too, exporting the real +checkpoint twice and comparing every byte. With ``VALEN_EXPORT`` set — or +a ``weights/valen-export`` directory next to the repository root — the +finished artifact set is validated against its manifest. See +``recipe/valen/export.md`` for the export itself. +""" + +from __future__ import annotations + +import os +import sys +import tempfile +from pathlib import Path + +HERE = Path(__file__).resolve().parent +REPO_ROOT = HERE.parents[1] +sys.path.insert(0, str(HERE)) +sys.path.insert(0, str(REPO_ROOT / "recipe" / "valen")) +sys.path.insert(0, str(REPO_ROOT / "src")) + +import export_checkpoint as tool # noqa: E402 +import test_export as contract # noqa: E402 + + +FIXTURE_PARAMS = ("tmp_path", "monkeypatch") + + +def contract_tests(): + """The export contract battery from tests/valen/test_export.py, in module + order; each recognized fixture parameter gets a fresh instance per + test (``tmp_path`` a fresh directory, ``monkeypatch`` a pytest + ``MonkeyPatch`` undone right after).""" + + tests = [] + for name in sorted(dir(contract)): + if not name.startswith("test_"): + continue + function = getattr(contract, name) + params = tuple( + param for param in FIXTURE_PARAMS if param in function.__code__.co_varnames + ) + tests.append((name, function, params)) + return tests + + +def run_contract_battery(results): + """Run every test in tests/valen/test_export.py directly. + + A test that raises pytest's ``Skipped`` outcome (the real-artifact gate + without ``VALEN_CKPT``/``VALEN_BASE``) is reported as a skip with the + test's own reason, never as a failure. + """ + + import pytest + + for name, function, params in contract_tests(): + try: + with tempfile.TemporaryDirectory() as tmp: + monkeypatch = pytest.MonkeyPatch() + kwargs = {} + if "tmp_path" in params: + kwargs["tmp_path"] = Path(tmp) + if "monkeypatch" in params: + kwargs["monkeypatch"] = monkeypatch + try: + function(**kwargs) + finally: + monkeypatch.undo() + except BaseException as error: # noqa: B036 - classified below + if type(error).__name__ == "Skipped": + record(results, name, "skip", str(error)) + else: + record(results, name, False, f"{type(error).__name__}: {error}") + else: + record(results, name, True) + + +def validate_artifact_set(results, export_dir): + """Validate an exported directory through the tool's own cross-check.""" + + try: + manifest = tool.validate_export(Path(export_dir)) + except Exception as error: + record(results, f"validate {export_dir}", False, str(error)) + return + counts = {} + for entry in manifest["tensors"]: + counts[entry["component"]] = counts.get(entry["component"], 0) + 1 + parity = manifest["parity"] + lora = manifest["lora"] + summary = ( + f"{manifest['format']}, {len(manifest['tensors'])} tensors " + + ", ".join(f"{counts[c]} {c}" for c in sorted(counts)) + + f", {len(manifest['downcasts'])} fp32->bf16 downcasts" + + f", lora unmerged r={lora['rank']} alpha={lora['alpha']}" + + f" ({lora['keys']} fp32 keys / {lora['targets']} targets)" + + f", parity slot {'filled' if parity is not None else 'not recorded'}" + ) + record(results, f"validate {export_dir}", True, summary) + + +def record(results, name, outcome, detail=""): + results.append((name, outcome, detail)) + + +def main() -> None: + results: list[tuple[str, object, str]] = [] + + run_contract_battery(results) + + export_dir = os.environ.get("VALEN_EXPORT") + if export_dir: + validate_artifact_set(results, export_dir) + elif (REPO_ROOT / "weights" / "valen-export").is_dir(): + validate_artifact_set(results, REPO_ROOT / "weights" / "valen-export") + else: + record( + results, + "validate exported artifact set", + "skip", + "set VALEN_EXPORT to an exported directory (the export's default " + "output is weights/valen-export) to validate it against its manifest", + ) + + failed = skipped = 0 + for name, outcome, detail in results: + if outcome is True: + print(f" ok {name}" + (f" — {detail}" if detail else "")) + elif outcome == "skip": + skipped += 1 + print(f" skip {name} — {detail}") + else: + failed += 1 + print(f" FAIL {name}") + print(" " + detail.strip().replace("\n", "\n ")[:400]) + verdict = "export_reference tests: ok" if not failed else ( + f"export_reference tests: {failed} FAILED" + ) + print(verdict + (f" ({skipped} skipped)" if skipped else "")) + sys.exit(1 if failed else 0) diff --git a/tests/valen/test_export.py b/tests/valen/test_export.py new file mode 100644 index 00000000..48b19847 --- /dev/null +++ b/tests/valen/test_export.py @@ -0,0 +1,1137 @@ +"""Export-tool tests for recipe/valen/export_checkpoint.py. + +CPU-only checks of the export's observable contract: the FP32->BF16 +downcast and the unmerged-LoRA application semantics on synthetic tensors +(against the installed PEFT as the independent reference), checkpoint-key +classification, and the manifest/artifact consistency on a tiny synthetic +fixture. The full determinism run against the real pinned artifacts is +env-gated on VALEN_CKPT / VALEN_BASE. + +PYTHONPATH=src python -m pytest tests/valen -q +""" + +from __future__ import annotations + +import hashlib +import json +import os +import sys +from pathlib import Path + +HERE = Path(__file__).resolve().parent +RECIPE_VALEN = HERE.parents[1] / "recipe" / "valen" +sys.path.insert(0, str(RECIPE_VALEN)) + +import export_checkpoint as tool # noqa: E402 + + +def test_lora_scaling_is_alpha_over_rank(): + assert tool.lora_scaling(rank=32, alpha=64) == 2.0 + assert tool.lora_scaling(rank=4, alpha=8) == 2.0 + assert tool.lora_scaling(rank=8, alpha=4) == 0.5 + + +def test_unmerged_output_matches_peft_layer_forward_bit_exactly(): + """The declared application semantics must reproduce peft 0.18.1's + unmerged Linear.forward on the same tensors: BF16 base GEMM, FP32 LoRA + path, one final rounding.""" + + import torch + import torch.nn as nn + from peft import LoraConfig, get_peft_model + + torch.manual_seed(3) + linear = nn.Linear(8, 6, bias=False).to(torch.bfloat16) + with torch.no_grad(): + linear.weight.copy_(torch.randn(6, 8).to(torch.bfloat16)) + + class Wrapped(nn.Module): + def __init__(self): + super().__init__() + self.lin = linear + + def forward(self, x): + return self.lin(x) + + model = get_peft_model( + Wrapped(), + LoraConfig(r=4, lora_alpha=8, lora_dropout=0.0, target_modules=["lin"], bias="none"), + ) + lora_linear = model.base_model.model.lin + with torch.no_grad(): + lora_linear.lora_A["default"].weight.copy_(torch.randn(4, 8)) + lora_linear.lora_B["default"].weight.copy_(torch.randn(6, 4) * 0.05) + scaling = lora_linear.scaling["default"] + base_weight = linear.weight.detach().clone() + lora_a = lora_linear.lora_A["default"].weight.detach().clone() + lora_b = lora_linear.lora_B["default"].weight.detach().clone() + probe = torch.randn(32, 8).to(torch.bfloat16) + + model.eval() + with torch.no_grad(): + peft_unmerged = model(probe).detach() + + mine = tool.unmerged_lora_output(base_weight, lora_a, lora_b, probe, scaling) + assert mine.dtype == torch.bfloat16 + assert torch.equal(mine, peft_unmerged.to(torch.bfloat16)) + + +def test_unmerged_output_hand_computed_rounding(): + """bf16(fp32(base GEMM) + fp32 delta) with one rounding of the sum, + hand-computed: base W = 1.0, probe x = 3.0, delta 2*(3*0.0019) = 0.0114 + sits above the 3.0078125 midpoint of the [2, 4) bf16 grid, so the + output rounds to 3.015625.""" + + import torch + + base = torch.tensor([[1.0]], dtype=torch.bfloat16) + lora_a = torch.tensor([[0.0019]], dtype=torch.float32) + lora_b = torch.tensor([[1.0]], dtype=torch.float32) + probe = torch.tensor([[3.0]], dtype=torch.bfloat16) + output = tool.unmerged_lora_output(base, lora_a, lora_b, probe, 2.0) + assert output.dtype == torch.bfloat16 + assert output.item() == 3.015625 + + +def test_unmerged_output_with_zero_lora_b_is_base_gemm(): + """An untrained LoRA (zero B, PEFT init) must reduce to the plain base + GEMM bit-exactly.""" + + import torch + + torch.manual_seed(1) + base = torch.randn(6, 8).to(torch.bfloat16) + probe = torch.randn(16, 8).to(torch.bfloat16) + output = tool.unmerged_lora_output( + base, torch.randn(4, 8), torch.zeros(6, 4), probe, 2.0 + ) + assert torch.equal(output, torch.matmul(probe, base.t())) + + +def test_downcast_rounds_to_nearest_bf16(): + """FP32 -> BF16 uses round-to-nearest-even, matching the reference's own + from_pretrained(dtype=bf16) cast; already-BF16 input passes through.""" + import torch + + values = torch.tensor( + [1.0, 1.003, 1.004, 0.5, -2.75, 3.14159265], dtype=torch.float32 + ) + downcast = tool.downcast_bf16(values) + assert downcast.dtype == torch.bfloat16 + assert torch.equal(downcast, values.to(torch.bfloat16)) + # A BF16-representable value survives the downcast bit-exactly. + bf16_source = torch.tensor([1.0, -0.5], dtype=torch.bfloat16) + assert torch.equal(tool.downcast_bf16(bf16_source), bf16_source) + + +# --- checkpoint-key classification (synthetic keys in the pinned layout) --- + +def synthetic_checkpoint_keys(): + """A miniature of the 428-key layout: one LoRA pair per family, the + vision-top blocks 20-23, the merger and the head.""" + + lora_modules = [ + "layers.0.linear_attn.in_proj_qkv", + "layers.0.linear_attn.in_proj_z", + "layers.0.linear_attn.in_proj_b", + "layers.0.linear_attn.in_proj_a", + "layers.0.linear_attn.out_proj", + "layers.3.self_attn.q_proj", + "layers.3.self_attn.k_proj", + "layers.3.self_attn.v_proj", + "layers.3.self_attn.o_proj", + "layers.0.mlp.gate_proj", + "layers.0.mlp.up_proj", + "layers.0.mlp.down_proj", + ] + keys = [] + for module in lora_modules: + for side, suffix in (("A", tool.PEFT_LORA_SUFFIX_A), ("B", tool.PEFT_LORA_SUFFIX_B)): + keys.append(tool.PEFT_LANGUAGE_PREFIX + module + suffix) + vision_blocks = [] + for block in (20, 21, 22, 23): + for tail in ( + "norm1.weight", "norm1.bias", "norm2.weight", "norm2.bias", + "attn.qkv.weight", "attn.qkv.bias", + "attn.proj.weight", "attn.proj.bias", + "mlp.linear_fc1.weight", "mlp.linear_fc1.bias", + "mlp.linear_fc2.weight", "mlp.linear_fc2.bias", + ): + vision_blocks.append(f"backbone.visual.blocks.{block}.{tail}") + keys.extend(vision_blocks) + keys.extend( + [ + "backbone.visual.merger.norm.weight", + "backbone.visual.merger.norm.bias", + "backbone.visual.merger.linear_fc1.weight", + "backbone.visual.merger.linear_fc1.bias", + "backbone.visual.merger.linear_fc2.weight", + "backbone.visual.merger.linear_fc2.bias", + "head.decision.weight", + "head.candidate.weight", + ] + ) + return keys + + +def test_classify_checkpoint_weights_layout(): + classification = tool.classify_checkpoint_weights(synthetic_checkpoint_keys()) + # Every LoRA pair maps to the base's own tensor name. + assert classification.lora["model.language_model.layers.0.mlp.down_proj.weight"] == ( + tool.PEFT_LANGUAGE_PREFIX + "layers.0.mlp.down_proj" + tool.PEFT_LORA_SUFFIX_A, + tool.PEFT_LANGUAGE_PREFIX + "layers.0.mlp.down_proj" + tool.PEFT_LORA_SUFFIX_B, + ) + assert len(classification.lora) == 12 + # Vision-top blocks land in the base's model.visual.* layout. + assert ( + "model.visual.blocks.20.attn.qkv.weight", + "backbone.visual.blocks.20.attn.qkv.weight", + ) in classification.vision_top + assert len(classification.vision_top) == 48 + assert ( + "model.visual.merger.linear_fc2.weight", + "backbone.visual.merger.linear_fc2.weight", + ) in classification.merger + assert len(classification.merger) == 6 + assert classification.head == { + "head.decision.weight": "head.decision.weight", + "head.candidate.weight": "head.candidate.weight", + } + + +def test_lora_key_target_maps_peft_names_onto_base_tensors(): + for key, target in ( + ( + tool.PEFT_LANGUAGE_PREFIX + "layers.0.mlp.down_proj" + tool.PEFT_LORA_SUFFIX_A, + "model.language_model.layers.0.mlp.down_proj.weight", + ), + ( + tool.PEFT_LANGUAGE_PREFIX + "layers.3.self_attn.q_proj" + tool.PEFT_LORA_SUFFIX_B, + "model.language_model.layers.3.self_attn.q_proj.weight", + ), + ): + assert tool.lora_key_target(key) == target + import pytest + + with pytest.raises(ValueError, match="not a language LoRA key"): + tool.lora_key_target("backbone.visual.merger.norm.weight") + + +def test_classify_checkpoint_weights_rejects_unknown_keys(): + import pytest + + for key in ( + "backbone.visual.patch_embed.proj.weight", + "backbone.language_model.layers.0.mlp.down_proj.weight", + "head.decision.bias", + "optimizer.steps", + ): + with pytest.raises(ValueError, match="unknown checkpoint weight"): + tool.classify_checkpoint_weights(synthetic_checkpoint_keys() + [key]) + + +def test_classify_checkpoint_weights_rejects_unpaired_lora(): + import pytest + + keys = synthetic_checkpoint_keys() + keys.remove(tool.PEFT_LANGUAGE_PREFIX + "layers.0.mlp.down_proj" + tool.PEFT_LORA_SUFFIX_B) + with pytest.raises(ValueError, match="unpaired LoRA"): + tool.classify_checkpoint_weights(keys) + + +def test_classify_checkpoint_weights_rejects_non_visual_block_tower(): + import pytest + + # A trainable vision block outside the trained top-4 set contradicts the + # pinned vision_top stage. + with pytest.raises(ValueError, match="vision_top"): + tool.classify_checkpoint_weights( + synthetic_checkpoint_keys() + ["backbone.visual.blocks.19.norm1.weight"] + ) + + +def test_classify_checkpoint_weights_rejects_block_beyond_the_top_four(): + """Blocks at or above 24 are outside the trained 20-23 set and must be + rejected exactly like sub-20 blocks (the pinned inventory counts alone + would not catch a swapped-in block 24 replacing a missing one).""" + + import pytest + + keys = synthetic_checkpoint_keys() + keys.remove("backbone.visual.blocks.20.norm1.weight") + keys.append("backbone.visual.blocks.24.norm1.weight") + with pytest.raises(ValueError, match="vision_top"): + tool.classify_checkpoint_weights(keys) + + +def test_validate_inventory_counts(): + classification = tool.classify_checkpoint_weights(synthetic_checkpoint_keys()) + expected = { + "checkpoint_tensors": 80, + "lora_keys": 24, + "vision_top_keys": 48, + "merger_keys": 6, + "head_keys": 2, + } + tool.validate_inventory(classification, expected) + import pytest + + with pytest.raises(ValueError, match="lora_keys"): + tool.validate_inventory(classification, dict(expected, lora_keys=23)) + + +# --- run_export on a tiny synthetic fixture (no real artifacts) --- + +TINY_INVENTORY = { + "checkpoint_tensors": 11, + "lora_keys": 6, + "vision_top_keys": 2, + "merger_keys": 1, + "head_keys": 2, + "base_tensors": 12, + "base_fp32_tensors": 2, +} +TINY_LORA = {"rank": 4, "alpha": 8} +TINY_HEAD = {"projection_dim": 3, "hidden_size": 6, "bias": False} +TINY_SOURCES = { + "checkpoint": {"revision": "c0ffee", "sha256": "ab" * 32}, + "base": {"revision": "badbee", "sha256": "cd" * 32}, +} + + +def tiny_base_tensors(): + import torch + + return { + "model.language_model.embed_tokens.weight": torch.randn(12, 6).to(torch.bfloat16), + "model.language_model.layers.0.input_layernorm.weight": torch.randn(6).to(torch.bfloat16), + "model.language_model.layers.0.linear_attn.A_log": torch.randn(4, dtype=torch.float32), + "model.language_model.layers.0.linear_attn.norm.weight": torch.randn(4, dtype=torch.float32), + "model.language_model.layers.0.linear_attn.in_proj_qkv.weight": torch.randn(8, 6).to(torch.bfloat16), + "model.language_model.layers.0.linear_attn.in_proj_z.weight": torch.randn(4, 6).to(torch.bfloat16), + "model.language_model.layers.0.mlp.gate_proj.weight": torch.randn(8, 6).to(torch.bfloat16), + "model.visual.blocks.19.norm1.weight": torch.randn(4).to(torch.bfloat16), + "model.visual.blocks.20.norm1.weight": torch.randn(4).to(torch.bfloat16), + "model.visual.blocks.20.attn.qkv.weight": torch.randn(12, 4).to(torch.bfloat16), + "model.visual.merger.linear_fc1.weight": torch.randn(8, 8).to(torch.bfloat16), + "mtp.fc.weight": torch.randn(6, 6).to(torch.bfloat16), + } + + +def tiny_checkpoint_weights(): + import torch + + prefix = tool.PEFT_LANGUAGE_PREFIX + weights = {} + for module, (out, inp) in ( + ("layers.0.linear_attn.in_proj_qkv", (8, 6)), + ("layers.0.linear_attn.in_proj_z", (4, 6)), + ("layers.0.mlp.gate_proj", (8, 6)), + ): + weights[prefix + module + tool.PEFT_LORA_SUFFIX_A] = torch.randn(4, inp) + weights[prefix + module + tool.PEFT_LORA_SUFFIX_B] = torch.randn(out, 4) * 0.1 + weights["backbone.visual.blocks.20.norm1.weight"] = torch.randn(4).to(torch.bfloat16) + weights["backbone.visual.blocks.20.attn.qkv.weight"] = torch.randn(12, 4).to(torch.bfloat16) + weights["backbone.visual.merger.linear_fc1.weight"] = torch.randn(8, 8).to(torch.bfloat16) + weights["head.decision.weight"] = torch.randn(3, 6) + weights["head.candidate.weight"] = torch.randn(3, 6) + return weights + + +def materialize_input_dir(directory, tensors=None, config_text='{"text_config": {"hidden_size": 6}}\n'): + """Write the tiny base safetensors + config into an input directory.""" + from safetensors.torch import save_file + + directory = Path(directory) + directory.mkdir(parents=True) + if tensors is None: + tensors = tiny_base_tensors() + save_file({k: v.contiguous() for k, v in tensors.items()}, directory / "base.safetensors") + (directory / "config.json").write_text(config_text, encoding="utf-8") + return directory / "base.safetensors", directory / "config.json" + + +def seeded_tiny_inputs(): + """The tiny fixture with fixed values (seed 7): the export must be a pure + function of its inputs, so every run sees byte-identical content.""" + + import torch + + torch.manual_seed(7) + return tiny_base_tensors(), tiny_checkpoint_weights() + + +def run_tiny_export(out_dir, input_dir): + base, checkpoint = seeded_tiny_inputs() + base_file, config_file = materialize_input_dir(input_dir, base) + return tool.run_export( + checkpoint, + base_file, + Path(out_dir), + sources=TINY_SOURCES, + lora_config=TINY_LORA, + head_config=TINY_HEAD, + expected=TINY_INVENTORY, + base_config_path=config_file, + ) + + +def read_manifest(out_dir): + return json.loads((Path(out_dir) / tool.MANIFEST_FILE).read_text(encoding="utf-8")) + + +def tensor_map(manifest): + return {(t["file"], t["name"]): t for t in manifest["tensors"]} + + +def test_run_export_tiny_fixture_contract(tmp_path): + import torch + from safetensors import safe_open + + out = tmp_path / "export" + manifest = run_tiny_export(out, tmp_path / "in-a") + assert sorted(p.name for p in out.iterdir()) == [ + "config.json", + "head.safetensors", + "lora.safetensors", + "model.safetensors", + "valen_export.json", + ] + + # Manifest header fields (the auditable contract). + assert manifest["format"] == tool.FORMAT + assert manifest["sources"] == TINY_SOURCES + assert "merge" not in manifest + assert manifest["lora"]["file"] == "lora.safetensors" + assert manifest["lora"]["dtype"] == "float32" + assert manifest["lora"]["rank"] == 4 and manifest["lora"]["alpha"] == 8 + assert manifest["lora"]["scaling"] == 2.0 + assert manifest["lora"]["targets"] == 3 and manifest["lora"]["keys"] == 6 + assert "peft 0.18.1" in manifest["lora"]["application"] + assert manifest["downcasts"] == [ + "model.language_model.layers.0.linear_attn.A_log", + "model.language_model.layers.0.linear_attn.norm.weight", + ] + assert manifest["head"]["projection_dim"] == 3 + assert manifest["head"]["hidden_size"] == 6 + assert manifest["head"]["bias"] is False + assert manifest["head"]["dtype"] == "float32" + assert manifest["head"]["tensors"] == ["head.candidate.weight", "head.decision.weight"] + assert manifest["parity"] is None + assert "parity_slot_shape" in manifest + + # config.json copied verbatim. + assert (out / "config.json").read_text(encoding="utf-8") == '{"text_config": {"hidden_size": 6}}\n' + + # Every tensor on disk appears in the manifest with matching metadata. + tensors = tensor_map(manifest) + for file_name in (tool.BACKBONE_FILE, tool.LORA_FILE, tool.HEAD_FILE): + with safe_open(out / file_name, framework="pt") as f: + keys = sorted(f.keys()) + for key in keys: + slice_ = f.get_slice(key) + entry = tensors[(file_name, key)] + assert entry["shape"] == list(slice_.get_shape()) + assert entry["dtype"] == { + "BF16": "bfloat16", "F32": "float32" + }[slice_.get_dtype()] + assert len(manifest["tensors"]) == 12 + 6 + 2 + assert [(t["file"], t["name"]) for t in manifest["tensors"]] == sorted( + (t["file"], t["name"]) for t in manifest["tensors"] + ) + + # Component attribution covers all five components. + components = {} + for entry in manifest["tensors"]: + components.setdefault(entry["component"], []).append(entry["name"]) + assert set(components) == set(tool.COMPONENTS) + assert len(components["lora"]) == 6 + assert len(components["backbone"]) == 9 + assert components["vision-top"] == [ + "model.visual.blocks.20.attn.qkv.weight", "model.visual.blocks.20.norm1.weight", + ] + assert components["merger"] == ["model.visual.merger.linear_fc1.weight"] + assert components["decision-head"] == ["head.candidate.weight", "head.decision.weight"] + assert "model.visual.blocks.19.norm1.weight" in components["backbone"] + + # File digests recorded match the bytes on disk. + for file_name, record in manifest["files"].items(): + data = (out / file_name).read_bytes() + assert record["size"] == len(data) + assert record["sha256"] == hashlib.sha256(data).hexdigest() + assert set(manifest["files"]) == { + "model.safetensors", "lora.safetensors", "head.safetensors", "config.json" + } + + # The transformations are the declared ones. + base, checkpoint = seeded_tiny_inputs() + with safe_open(out / tool.BACKBONE_FILE, framework="pt") as f, safe_open( + tmp_path / "in-a" / "base.safetensors", framework="pt" + ) as original: + # LoRA targets pass through as base tensors, bit-exact and untouched. + target = "model.language_model.layers.0.linear_attn.in_proj_qkv.weight" + assert torch.equal(f.get_tensor(target), original.get_tensor(target)) + assert ( + tensor_map(manifest)[(tool.BACKBONE_FILE, target)]["component"] == "backbone" + ) + # Untouched backbone tensors pass through bit-exactly. + assert torch.equal( + f.get_tensor("mtp.fc.weight"), original.get_tensor("mtp.fc.weight") + ) + assert torch.equal( + f.get_tensor("model.visual.blocks.19.norm1.weight"), + original.get_tensor("model.visual.blocks.19.norm1.weight"), + ) + # Downcast semantics. + assert torch.equal( + f.get_tensor("model.language_model.layers.0.linear_attn.A_log").float(), + base["model.language_model.layers.0.linear_attn.A_log"].to(torch.bfloat16).float(), + ) + # Vision-top and merger are the checkpoint's trained replacements. + assert torch.equal( + f.get_tensor("model.visual.blocks.20.attn.qkv.weight"), + checkpoint["backbone.visual.blocks.20.attn.qkv.weight"], + ) + assert torch.equal( + f.get_tensor("model.visual.merger.linear_fc1.weight"), + checkpoint["backbone.visual.merger.linear_fc1.weight"], + ) + # The adapters ride in their own file, FP32 under their PEFT key names. + with safe_open(out / tool.LORA_FILE, framework="pt") as f: + keys = sorted(f.keys()) + assert keys == sorted( + key + for pair in tool.classify_checkpoint_weights( + sorted(checkpoint) + ).lora.values() + for key in pair + ) + for key in keys: + tensor = f.get_tensor(key) + assert tensor.dtype == torch.float32 + assert torch.equal(tensor, checkpoint[key]) + with safe_open(out / tool.HEAD_FILE, framework="pt") as f: + assert torch.equal(f.get_tensor("head.decision.weight"), checkpoint["head.decision.weight"]) + assert f.get_tensor("head.decision.weight").dtype == torch.float32 + + # The downcast entries carry their origin dtype. + for name in manifest["downcasts"]: + assert tensors[(tool.BACKBONE_FILE, name)]["downcast_from"] == "float32" + + +def test_run_export_is_deterministic_and_path_independent(tmp_path): + first_out, second_out = tmp_path / "export-1", tmp_path / "export-2" + run_tiny_export(first_out, tmp_path / "in-a") + run_tiny_export(second_out, tmp_path / "in-b-different-name") + for name in ( + "model.safetensors", "lora.safetensors", "head.safetensors", "config.json", + "valen_export.json", + ): + assert (first_out / name).read_bytes() == (second_out / name).read_bytes(), name + # No absolute paths leak into the manifest. + manifest_text = (first_out / tool.MANIFEST_FILE).read_text(encoding="utf-8") + assert str(tmp_path) not in manifest_text + assert '"/' not in manifest_text + + +def test_run_export_rejects_existing_output(tmp_path): + import pytest + + out = tmp_path / "export" + run_tiny_export(out, tmp_path / "in-a") + with pytest.raises(ValueError, match="already exists"): + run_tiny_export(out, tmp_path / "in-b") + + +def test_run_export_failure_leaves_no_partial_output(tmp_path, monkeypatch): + """A failed export must not leave a half-written output directory: the + not-empty guard would then block every retry with the same arguments.""" + + import pytest + + real_write = tool.write_safetensors + + def exploding_write(path, entries): + real_write(path, entries) + raise ValueError("boom after writing") + + monkeypatch.setattr(tool, "write_safetensors", exploding_write) + base_file, config_file = materialize_input_dir(tmp_path / "in") + out = tmp_path / "export" + with pytest.raises(ValueError, match="boom"): + tool.run_export( + tiny_checkpoint_weights(), + base_file, + out, + sources=TINY_SOURCES, + lora_config=TINY_LORA, + head_config=TINY_HEAD, + expected=TINY_INVENTORY, + base_config_path=config_file, + ) + assert not out.exists() or not any(out.iterdir()) + assert not (tmp_path / "export.export-partial").exists() + + # The same arguments now succeed into the same directory. + monkeypatch.setattr(tool, "write_safetensors", real_write) + run_tiny_export(out, tmp_path / "in-a") + + +def test_run_export_failure_cleans_preexisting_empty_output(tmp_path, monkeypatch): + """Same contract when --out pointed at a pre-existing empty directory: + the retry must find it empty again (the directory itself may survive).""" + + import pytest + + real_write = tool.write_safetensors + + def exploding_write(path, entries): + real_write(path, entries) + raise ValueError("boom after writing") + + monkeypatch.setattr(tool, "write_safetensors", exploding_write) + base_file, config_file = materialize_input_dir(tmp_path / "in") + out = tmp_path / "export" + out.mkdir() + with pytest.raises(ValueError, match="boom"): + tool.run_export( + tiny_checkpoint_weights(), + base_file, + out, + sources=TINY_SOURCES, + lora_config=TINY_LORA, + head_config=TINY_HEAD, + expected=TINY_INVENTORY, + base_config_path=config_file, + ) + assert not any(out.iterdir()) + assert not (tmp_path / "export.export-partial").exists() + + +def test_run_export_requires_base_config_path(tmp_path): + """run_export without the base config would write a set its own + validate_export rejects (the allowed set includes config.json).""" + + import pytest + + base_file, _ = materialize_input_dir(tmp_path / "in") + out = tmp_path / "export" + with pytest.raises(ValueError, match="base_config_path"): + tool.run_export( + tiny_checkpoint_weights(), + base_file, + out, + sources=TINY_SOURCES, + lora_config=TINY_LORA, + head_config=TINY_HEAD, + expected=TINY_INVENTORY, + base_config_path=None, + ) + assert not out.exists() + + +def test_run_export_clears_stale_staging_directory(tmp_path): + """A run killed mid-write (SIGKILL/OOM) can leave the staging directory + behind; out_dir itself is never partial, and the next run with the same + target clears the staging remains instead of failing.""" + + out = tmp_path / "export" + stale = tmp_path / "export.export-partial" + stale.mkdir() + (stale / "model.safetensors").write_bytes(b"partial junk") + run_tiny_export(out, tmp_path / "in-a") + assert not stale.exists() + assert (out / tool.MANIFEST_FILE).is_file() + + +def test_run_export_rejects_unexpected_fp32_base_tensor(tmp_path): + import pytest + import torch + from safetensors.torch import save_file + + base = tiny_base_tensors() + # An FP32 tensor outside the A_log / linear_attn.norm.weight families is + # a silent dtype surprise, not a declared downcast. + base["model.language_model.layers.0.mlp.gate_proj.weight"] = torch.randn(8, 6) + directory = tmp_path / "in" + directory.mkdir() + save_file({k: v.contiguous() for k, v in base.items()}, directory / "base.safetensors") + (directory / "config.json").write_text("{}", encoding="utf-8") + with pytest.raises(ValueError, match="fp32"): + tool.run_export( + tiny_checkpoint_weights(), + directory / "base.safetensors", + tmp_path / "out", + sources=TINY_SOURCES, + lora_config=TINY_LORA, + head_config=TINY_HEAD, + expected=dict(TINY_INVENTORY, base_fp32_tensors=3), + base_config_path=directory / "config.json", + ) + + +def test_run_export_rejects_lora_shape_mismatch(tmp_path): + import pytest + import torch + + weights = tiny_checkpoint_weights() + weights[tool.PEFT_LANGUAGE_PREFIX + "layers.0.mlp.gate_proj" + tool.PEFT_LORA_SUFFIX_B] = ( + torch.randn(7, 4) + ) + base_file, config_file = materialize_input_dir(tmp_path / "in") + with pytest.raises(ValueError, match="shape"): + tool.run_export( + weights, + base_file, + tmp_path / "out", + sources=TINY_SOURCES, + lora_config=TINY_LORA, + head_config=TINY_HEAD, + expected=TINY_INVENTORY, + base_config_path=config_file, + ) + + +def test_run_export_rejects_head_config_mismatch(tmp_path): + import pytest + + base_file, config_file = materialize_input_dir(tmp_path / "in") + with pytest.raises(ValueError, match="head"): + tool.run_export( + tiny_checkpoint_weights(), + base_file, + tmp_path / "out", + sources=TINY_SOURCES, + lora_config=TINY_LORA, + head_config=dict(TINY_HEAD, projection_dim=5), + expected=TINY_INVENTORY, + base_config_path=config_file, + ) + + +# --- validate_export: manifest/artifact consistency and tamper detection --- + +def rewrite_manifest(out_dir, mutate): + manifest = read_manifest(out_dir) + mutate(manifest) + (Path(out_dir) / tool.MANIFEST_FILE).write_text( + json.dumps(manifest, indent=2, sort_keys=True) + "\n", encoding="utf-8" + ) + + +def test_validate_export_accepts_a_fresh_export(tmp_path): + out = tmp_path / "export" + run_tiny_export(out, tmp_path / "in-a") + manifest = tool.validate_export(out) + assert manifest["format"] == tool.FORMAT + + +def test_validate_export_streams_file_digests(tmp_path, monkeypatch): + """File digests must be computed streaming, never via a whole-file read: + the real model.safetensors is ~4.5 GiB and the contract is "headers and + digests only" (validate_export docstring, export.md).""" + + out = tmp_path / "export" + run_tiny_export(out, tmp_path / "in-a") + + def refuse(self): + raise AssertionError(f"validate_export read {self} whole into memory") + + monkeypatch.setattr(Path, "read_bytes", refuse) + manifest = tool.validate_export(out) + assert manifest["format"] == tool.FORMAT + + +def test_validate_export_detects_tampered_bytes(tmp_path): + import pytest + + out = tmp_path / "export" + run_tiny_export(out, tmp_path / "in-a") + for target_name in (tool.BACKBONE_FILE, tool.LORA_FILE): + target = out / target_name + data = bytearray(target.read_bytes()) + data[-1] ^= 0xFF + target.write_bytes(bytes(data)) + with pytest.raises(ValueError, match="sha256"): + tool.validate_export(out) + data[-1] ^= 0xFF + target.write_bytes(bytes(data)) + + +def test_validate_export_detects_manifest_tensor_removal(tmp_path): + import pytest + + out = tmp_path / "export" + run_tiny_export(out, tmp_path / "in-a") + + # Removing a lora.safetensors entry leaves an unlisted tensor on disk. + def drop_one(manifest): + manifest["tensors"] = [ + entry + for entry in manifest["tensors"] + if entry["name"] + != tool.PEFT_LANGUAGE_PREFIX + "layers.0.mlp.gate_proj" + tool.PEFT_LORA_SUFFIX_A + ] + + rewrite_manifest(out, drop_one) + with pytest.raises(ValueError, match="unlisted tensor"): + tool.validate_export(out) + + +def test_validate_export_detects_downcast_list_drift(tmp_path): + import pytest + + out = tmp_path / "export" + run_tiny_export(out, tmp_path / "in-a") + + def drop_downcast(manifest): + manifest["downcasts"] = manifest["downcasts"][:1] + + rewrite_manifest(out, drop_downcast) + with pytest.raises(ValueError, match="downcast"): + tool.validate_export(out) + + +def test_validate_export_detects_head_config_drift(tmp_path): + import pytest + + out = tmp_path / "export" + run_tiny_export(out, tmp_path / "in-a") + + def wrong_dim(manifest): + manifest["head"]["projection_dim"] = 5 + + rewrite_manifest(out, wrong_dim) + with pytest.raises(ValueError, match="head"): + tool.validate_export(out) + + +def test_validate_export_detects_lora_block_drift(tmp_path): + import pytest + + rounds = iter((1, 2, 3)) + + def check(mutation, pattern): + out = tmp_path / f"export-{next(rounds)}" + run_tiny_export(out, tmp_path / f"in-{out.name}") + rewrite_manifest(out, mutation) + with pytest.raises(ValueError, match=pattern): + tool.validate_export(out) + + def wrong_keys(manifest): + manifest["lora"]["keys"] = 5 + + check(wrong_keys, "lora block records") + + def wrong_scaling(manifest): + manifest["lora"]["scaling"] = 3.0 + + check(wrong_scaling, "lora scaling is not alpha / rank") + + def wrong_rank(manifest): + manifest["lora"]["rank"] = 2 + manifest["lora"]["alpha"] = 4 + + check(wrong_rank, "contradicts the declared rank") + + +def test_validate_export_detects_extra_files_and_forgotten_entries(tmp_path): + import pytest + + out = tmp_path / "export" + run_tiny_export(out, tmp_path / "in-a") + (out / "surprise.bin").write_bytes(b"x") + with pytest.raises(ValueError, match="unexpected"): + tool.validate_export(out) + (out / "surprise.bin").unlink() + + def drop_file_record(manifest): + del manifest["files"][tool.LORA_FILE] + + rewrite_manifest(out, drop_file_record) + with pytest.raises(ValueError, match="lora.safetensors"): + tool.validate_export(out) + + +def test_validate_export_rejects_extra_directories(tmp_path): + """The "exactly five files" audit must reject any unexpected entry, not + only unexpected regular files — otherwise a stray subdirectory passes.""" + + import pytest + + out = tmp_path / "export" + run_tiny_export(out, tmp_path / "in-a") + (out / "scratch").mkdir() + with pytest.raises(ValueError, match="unexpected"): + tool.validate_export(out) + + +# --- real-artifact pin helpers (CPU-only, no multi-GB reads) --- + +def test_base_weights_file_requires_single_shard(tmp_path): + import pytest + + base = tmp_path / "base" + base.mkdir() + (base / "model.safetensors.index.json").write_text( + json.dumps({"weight_map": {"a": "model-00001.safetensors"}}), encoding="utf-8" + ) + (base / "model-00001.safetensors").write_bytes(b"") + assert tool.base_weights_file(base) == base / "model-00001.safetensors" + + (base / "model.safetensors.index.json").write_text( + json.dumps( + {"weight_map": {"a": "model-00001.safetensors", "b": "model-00002.safetensors"}} + ), + encoding="utf-8", + ) + with pytest.raises(ValueError, match="single-shard"): + tool.base_weights_file(base) + + +def test_checkpoint_revision_from_tree_pin(tmp_path): + checkpoint = tmp_path / "ckpt" + trees = checkpoint / ".cache" / "huggingface" / "trees" + trees.mkdir(parents=True) + assert tool.checkpoint_revision(checkpoint) is None + (trees / "81b9c6396ded3a45a92ae8079af85f988aa40501.json").write_text("{}", encoding="utf-8") + assert tool.checkpoint_revision(checkpoint) == "81b9c6396ded3a45a92ae8079af85f988aa40501" + (trees / "second.json").write_text("{}", encoding="utf-8") + assert tool.checkpoint_revision(checkpoint) is None + + +# --- export()'s artifact pins (tiny synthetic checkpoint + base) --- + +def tiny_pin_base_dir(directory): + """A miniature pinned base: one shard, index, config, valen_manifest. + + The manifest declares the shard's real digest, so the pin stage passes + unless a test tampers with it.""" + + import hashlib + + directory = Path(directory) + directory.mkdir(parents=True) + shard = directory / "model.safetensors-00001-of-00001.safetensors" + shard.write_bytes(b"shard-bytes") + (directory / tool.BASE_INDEX_FILE).write_text( + json.dumps({"weight_map": {"a": shard.name}}), encoding="utf-8" + ) + (directory / "config.json").write_text( + '{"text_config": {"hidden_size": 6}}', encoding="utf-8" + ) + (directory / tool.BASE_MANIFEST_FILE).write_text( + json.dumps( + { + "repo": "Qwen/Qwen3.5-2B", + "revision": "badbee", + "weight_sha256": { + shard.name: hashlib.sha256(b"shard-bytes").hexdigest() + }, + } + ), + encoding="utf-8", + ) + return directory + + +def tiny_pin_checkpoint( + directory, *, base_manifest=None, checkpoint_sha=None, monkeypatch=None +): + """Save a miniature loadable checkpoint.pt plus its training_metadata. + + With a monkeypatch handle the fixture also points the exporter's + worker-pin seam at the tiny file's real digest, so the synthetic bytes + pass the pinned-digest gate and the remaining pin checks stay testable.""" + + directory = Path(directory) + directory.mkdir(parents=True, exist_ok=True) + import torch + + payload = { + "config": { + "stage": tool.PINNED_STAGE, + "lora_rank": 4, + "lora_alpha": 8, + "projection_dim": 3, + }, + "weights": {}, + "base_manifest": base_manifest + if base_manifest is not None + else { + "repo": "Qwen/Qwen3.5-2B", + "revision": "badbee", + "weight_sha256": {}, + }, + } + checkpoint_path = directory / tool.CHECKPOINT_FILE + torch.save(payload, checkpoint_path) + real_sha = tool.sha256_file(checkpoint_path) + (directory / tool.TRAINING_METADATA_FILE).write_text( + json.dumps({"checkpoint_sha256": checkpoint_sha or real_sha}), encoding="utf-8" + ) + if monkeypatch is not None: + monkeypatch.setattr(tool, "_pinned_checkpoint_sha256", lambda: real_sha) + return directory, real_sha + + +def test_export_rejects_checkpoint_below_the_worker_pin(tmp_path): + """The exporter must enforce the worker's pinned digest before the + pickle is ever loaded: training_metadata.json lives beside the file it + describes, so a directory that controls one controls both — only the + hard-coded worker pin is a trust anchor (the P1 the review bot raised).""" + + import pytest + + checkpoint = tmp_path / "ckpt" + checkpoint.mkdir() + (checkpoint / tool.CHECKPOINT_FILE).write_bytes(b"not the pinned checkpoint") + with pytest.raises(ValueError, match="pinned Valen-Preview-0923 digest"): + tool.export(checkpoint, tiny_pin_base_dir(tmp_path / "base"), tmp_path / "out") + + +def test_export_requires_training_metadata_pin(tmp_path, monkeypatch): + """A checkpoint directory without training_metadata.json must be rejected: + the digest pin has to be verified before the pickle is ever unpickled + (the worker precedent: src/models/valen/engine.py pins unconditionally).""" + + import pytest + + checkpoint, _ = tiny_pin_checkpoint( + tmp_path / "ckpt", monkeypatch=monkeypatch + ) + (checkpoint / tool.TRAINING_METADATA_FILE).unlink() + with pytest.raises(ValueError, match="training_metadata.json"): + tool.export(checkpoint, tiny_pin_base_dir(tmp_path / "base"), tmp_path / "out") + + +def test_export_rejects_checkpoint_sha_mismatch(tmp_path, monkeypatch): + import pytest + + checkpoint, _ = tiny_pin_checkpoint( + tmp_path / "ckpt", checkpoint_sha="00" * 32, monkeypatch=monkeypatch + ) + with pytest.raises(ValueError, match="training_metadata"): + tool.export(checkpoint, tiny_pin_base_dir(tmp_path / "base"), tmp_path / "out") + + +def test_export_requires_valen_manifest_base_digest(tmp_path, monkeypatch): + """valen_manifest.json must declare the base weights digest; a missing + declaration is a pin bypass, not a pass.""" + + import pytest + + checkpoint, _ = tiny_pin_checkpoint( + tmp_path / "ckpt", monkeypatch=monkeypatch + ) + base = tiny_pin_base_dir(tmp_path / "base") + manifest = json.loads((base / tool.BASE_MANIFEST_FILE).read_text(encoding="utf-8")) + del manifest["weight_sha256"] + (base / tool.BASE_MANIFEST_FILE).write_text(json.dumps(manifest), encoding="utf-8") + with pytest.raises(ValueError, match="valen_manifest.json"): + tool.export(checkpoint, base, tmp_path / "out") + + +def test_export_requires_checkpoint_base_manifest_digest(tmp_path, monkeypatch): + """The checkpoint's own base_manifest must declare the base weights digest + too; absence must fail the pin instead of skipping half of it.""" + + import pytest + + checkpoint, _ = tiny_pin_checkpoint( + tmp_path / "ckpt", + base_manifest={"revision": "badbee"}, + monkeypatch=monkeypatch, + ) + base = tiny_pin_base_dir(tmp_path / "base") + with pytest.raises(ValueError, match="checkpoint base_manifest"): + tool.export(checkpoint, base, tmp_path / "out") + + +def test_export_rejects_declared_base_digest_mismatch(tmp_path, monkeypatch): + import pytest + + checkpoint, _ = tiny_pin_checkpoint( + tmp_path / "ckpt", + base_manifest={"revision": "badbee", "weight_sha256": {"wrong.safetensors": "ff" * 32}}, + monkeypatch=monkeypatch, + ) + with pytest.raises(ValueError, match="checkpoint base_manifest"): + tool.export(checkpoint, tiny_pin_base_dir(tmp_path / "base"), tmp_path / "out") + + +def test_export_pin_stage_passes_on_consistent_pins(tmp_path, monkeypatch): + """Sanity: with every pin present and consistent, the pin stage completes + and the run proceeds into the inventory check (the tiny payload cannot + satisfy the pinned 428-tensor inventory).""" + + import pytest + + base = tiny_pin_base_dir(tmp_path / "base") + declared = json.loads( + (base / tool.BASE_MANIFEST_FILE).read_text(encoding="utf-8") + )["weight_sha256"] + checkpoint, _ = tiny_pin_checkpoint( + tmp_path / "ckpt", + base_manifest={"revision": "badbee", "weight_sha256": declared}, + monkeypatch=monkeypatch, + ) + with pytest.raises(ValueError, match="inventory"): + tool.export(checkpoint, base, tmp_path / "out") + + +# --- the full determinism check against the real pinned artifacts --- +# Set VALEN_CKPT and VALEN_BASE (the recipe reference.md artifact exports) to +# run; it exports twice and compares every byte, mirroring the CUA_S1_BASE +# artifact-gating precedent. + +def test_real_artifact_export_is_deterministic(tmp_path): + import pytest + + if not (os.environ.get("VALEN_CKPT") and os.environ.get("VALEN_BASE")): + pytest.skip("set VALEN_CKPT and VALEN_BASE to run") + + runs = [] + for index in (1, 2): + out = tmp_path / f"run-{index}" + manifest = tool.export( + Path(os.environ["VALEN_CKPT"]), Path(os.environ["VALEN_BASE"]), out + ) + runs.append((out, manifest)) + + (first, manifest_one), (second, manifest_two) = runs + for name in ( + tool.BACKBONE_FILE, tool.LORA_FILE, tool.HEAD_FILE, "config.json", + tool.MANIFEST_FILE, + ): + assert (first / name).read_bytes() == (second / name).read_bytes(), name + + # The pinned inventory, end to end. + counts = {} + for entry in manifest_one["tensors"]: + counts[entry["component"]] = counts.get(entry["component"], 0) + 1 + # 632 base tensors: 48 vision-top and 6 merger replaced, the remaining + # 578 passing through verbatim (36 of them downcast from fp32) — the + # 186 LoRA targets among them, untouched. + assert counts == { + "backbone": 578, + "lora": 372, + "vision-top": 48, + "merger": 6, + "decision-head": 2, + } + assert len(manifest_one["downcasts"]) == 36 + lora = manifest_one["lora"] + assert lora["rank"] == 32 and lora["alpha"] == 64 + assert lora["scaling"] == 2.0 + assert lora["targets"] == 186 and lora["keys"] == 372 + assert lora["dtype"] == "float32" + assert manifest_one["head"]["projection_dim"] == 256 + assert manifest_one["head"]["hidden_size"] == 2048 + assert manifest_one["sources"]["checkpoint"]["sha256"] == ( + "836622efe78fe757e2627aa6050c223d461c424ea430a1c110c15e6f42f5a012" + ) + assert manifest_one["sources"]["base"]["revision"] == ( + "15852e8c16360a2fea060d615a32b45270f8a8fc" + ) + assert manifest_one["parity"] is None diff --git a/tests/valen/test_export_parity.py b/tests/valen/test_export_parity.py new file mode 100644 index 00000000..bc85c07b --- /dev/null +++ b/tests/valen/test_export_parity.py @@ -0,0 +1,524 @@ +"""Parity-gate tests for recipe/valen/verify_export_parity.py. + +CPU-only checks of the export-parity tool's observable contract: the +original-vs-exported case comparison verdicts and the manifest parity-slot +filling. The full parity gate against the real pinned artifacts is +env-gated on VALEN_CKPT / VALEN_BASE / VALEN_SRC plus CUDA. + +PYTHONPATH=src python -m pytest tests/valen -q +""" + +from __future__ import annotations + +import json +import os +import sys +from pathlib import Path + +import pytest + +HERE = Path(__file__).resolve().parent +RECIPE_VALEN = HERE.parents[1] / "recipe" / "valen" +sys.path.insert(0, str(RECIPE_VALEN)) + +import compare_reference as reference # noqa: E402 +import verify_export_parity as tool # noqa: E402 + + +# --- the original-vs-exported case comparison (the gate's verdict) --- + +def case_side(probs, keys=("refund", "wait"), logical=48, compute=48): + return { + "questions": {"refund": {"keys": list(keys), "probabilities": probs}}, + "logical_tokens": logical, + "compute_tokens": compute, + } + + +def test_compare_case_identical_stacks_pass(): + original = case_side([0.5159631, 0.4840369]) + verdict = tool.compare_case("text", original, case_side([0.5159631, 0.4840369])) + assert verdict["exact_answer_agreement"] is True + assert verdict["max_probability_delta"] == 0.0 + assert verdict["probability_tolerance_pass"] is True + assert verdict["tokens_agree"] is True + assert verdict["passed"] is True + assert verdict["problems"] == [] + + +def test_compare_case_delta_within_tolerance_passes(): + original = case_side([0.5159631, 0.4840369]) + drifted = case_side([0.5159631 + 5e-7, 0.4840369 - 5e-7]) + verdict = tool.compare_case("text", original, drifted) + assert verdict["passed"] is True + assert verdict["max_probability_delta"] == pytest.approx(5e-7) + assert verdict["exact_answer_agreement"] is True + + +def test_compare_case_delta_beyond_tolerance_fails(): + original = case_side([0.5159631, 0.4840369]) + drifted = case_side([0.5159631 + 2e-6, 0.4840369 - 2e-6]) + verdict = tool.compare_case("text", original, drifted) + assert verdict["exact_answer_agreement"] is True + assert verdict["probability_tolerance_pass"] is False + assert verdict["passed"] is False + assert any("tolerance" in problem for problem in verdict["problems"]) + + +def test_compare_case_flipped_answer_fails(): + original = case_side([0.5159631, 0.4840369]) + flipped = case_side([0.4840369, 0.5159631]) + verdict = tool.compare_case("text", original, flipped) + assert verdict["exact_answer_agreement"] is False + assert verdict["passed"] is False + + +def test_compare_case_candidate_order_mismatch_fails(): + original = case_side([0.5159631, 0.4840369]) + reordered = case_side([0.4840369, 0.5159631], keys=("wait", "refund")) + verdict = tool.compare_case("text", original, reordered) + assert verdict["passed"] is False + assert any("candidate order" in problem for problem in verdict["problems"]) + + +def test_compare_case_probability_length_mismatch_fails(): + """A malformed stage payload with fewer probabilities than candidates + must fail the case, not silently truncate the delta scan.""" + + original = case_side([0.5159631, 0.4840369]) + truncated = case_side([0.5159631, 0.4840369]) + truncated["questions"]["refund"]["probabilities"] = [0.5159631] + verdict = tool.compare_case("text", original, truncated) + assert verdict["passed"] is False + assert verdict["exact_answer_agreement"] is False + assert any("probabilities" in problem for problem in verdict["problems"]) + + +def test_compare_case_exported_nan_probability_fails(): + """A NaN difference keeps the previous finite maximum and the argmax + keeps a NaN-free first candidate, so a non-finite probability on either + side must fail the case before any delta or argmax work.""" + + original = case_side([0.6, 0.4]) + poisoned = case_side([float("nan"), 0.4]) + verdict = tool.compare_case("text", original, poisoned) + assert verdict["passed"] is False + assert verdict["exact_answer_agreement"] is False + assert any("non-finite probability" in problem for problem in verdict["problems"]) + + +def test_compare_case_matching_infinities_fail(): + """Identical infinities on both sides would delta to NaN or agree on the + argmax; the case still fails because the probabilities are non-finite.""" + + infinite = case_side([float("inf"), 0.0]) + verdict = tool.compare_case("text", infinite, case_side([float("inf"), 0.0])) + assert verdict["passed"] is False + assert verdict["exact_answer_agreement"] is False + assert any("non-finite probability" in problem for problem in verdict["problems"]) + + +def test_compare_case_missing_question_fails(): + original = case_side([0.5159631, 0.4840369]) + emptied = case_side([0.5159631, 0.4840369]) + emptied["questions"] = {} + verdict = tool.compare_case("text", original, emptied) + assert verdict["passed"] is False + + +def test_compare_case_token_accounting_must_agree(): + original = case_side([0.5159631, 0.4840369]) + retokenized = case_side([0.5159631, 0.4840369], compute=49) + verdict = tool.compare_case("text", original, retokenized) + assert verdict["tokens_agree"] is False + assert verdict["passed"] is False + + +# --- the manifest parity slot --- + +def test_build_parity_record_covers_the_documented_slot_shape(): + record = tool.build_parity_record( + status="passed", + device="cuda:0", + hardware="NVIDIA GeForce RTX 5060 Laptop GPU (sm_120)", + cases=[{"case": "text", "passed": True}], + exact_answer_agreement=True, + max_probability_delta=0.0, + tolerance=1e-6, + evidence="recipe/valen/reference.md export-parity block", + ) + # Every key the manifest's parity_slot_shape documents must be present. + assert set(record) == { + "status", "device", "hardware", "tolerance", "cases", + "exact_answer_agreement", "max_probability_delta", "evidence", + } + assert record["status"] == "passed" + assert record["max_probability_delta"] == 0.0 + assert record["tolerance"] == 1e-6 + + +def test_build_parity_record_rejects_unknown_status(): + with pytest.raises(ValueError, match="status"): + tool.build_parity_record( + status="maybe", + device="cuda:0", + hardware="gpu", + cases=[], + exact_answer_agreement=True, + max_probability_delta=0.0, + tolerance=1e-6, + evidence="", + ) + + +def minimal_manifest(path, parity=None): + manifest = { + "format": "valen-checkpoint-export/2", + "parity": parity, + "parity_slot_shape": {"status": "passed | failed"}, + } + Path(path).write_text( + json.dumps(manifest, indent=2, sort_keys=True) + "\n", encoding="utf-8" + ) + return manifest + + +def parity_record(status="passed", delta=1e-8): + return tool.build_parity_record( + status=status, + device="cuda:0", + hardware="gpu", + cases=[{"case": "text", "passed": status == "passed"}], + exact_answer_agreement=status == "passed", + max_probability_delta=delta, + tolerance=1e-6, + evidence="recipe/valen/reference.md", + ) + + +def test_fill_parity_slot_replaces_null(tmp_path): + manifest_path = tmp_path / "valen_export.json" + minimal_manifest(manifest_path) + record = parity_record() + updated = tool.fill_parity_slot(manifest_path, record) + assert updated["parity"] == record + reread = json.loads(manifest_path.read_text(encoding="utf-8")) + assert reread["parity"] == record + # The documented formatting (sorted, indent 2, trailing newline) is kept. + assert manifest_path.read_text(encoding="utf-8").endswith("}\n") + + +def test_fill_parity_slot_is_idempotent_for_the_same_record(tmp_path): + manifest_path = tmp_path / "valen_export.json" + minimal_manifest(manifest_path) + record = parity_record() + tool.fill_parity_slot(manifest_path, record) + before = manifest_path.read_text(encoding="utf-8") + tool.fill_parity_slot(manifest_path, record) + assert manifest_path.read_text(encoding="utf-8") == before + + +def test_fill_parity_slot_refuses_to_silently_overwrite(tmp_path): + manifest_path = tmp_path / "valen_export.json" + minimal_manifest(manifest_path) + tool.fill_parity_slot(manifest_path, parity_record(delta=1e-8)) + with pytest.raises(ValueError, match="parity slot already holds"): + tool.fill_parity_slot(manifest_path, parity_record(delta=5e-3)) + # The earlier record survives the refusal. + assert json.loads(manifest_path.read_text(encoding="utf-8"))["parity"][ + "max_probability_delta" + ] == 1e-8 + + +# --- the source-pin cross-check before any model is loaded --- + +def pin_args(checkpoint_dir, base_dir): + import argparse + + return argparse.Namespace(checkpoint=str(checkpoint_dir), base=str(base_dir)) + + +def pin_fixture(tmp_path): + """A tiny checkpoint dir + single-shard base and a manifest recording + their true digests, mirroring the real pinned layout.""" + + import hashlib + + from export_checkpoint import sha256_file + + checkpoint_dir = tmp_path / "ckpt" + checkpoint_dir.mkdir() + checkpoint_file = checkpoint_dir / "checkpoint.pt" + checkpoint_file.write_bytes(b"checkpoint-bytes") + base_dir = tmp_path / "base" + base_dir.mkdir() + shard = base_dir / "model.safetensors-00001-of-00001.safetensors" + shard.write_bytes(b"shard-bytes") + (base_dir / "model.safetensors.index.json").write_text( + json.dumps({"weight_map": {"a": shard.name}}), encoding="utf-8" + ) + (base_dir / "valen_manifest.json").write_text( + json.dumps({"revision": "badbee"}), encoding="utf-8" + ) + manifest = { + "sources": { + "checkpoint": {"sha256": sha256_file(checkpoint_file)}, + "base": { + "revision": "badbee", + "weights_sha256": { + shard.name: hashlib.sha256(b"shard-bytes").hexdigest() + }, + }, + } + } + return pin_args(checkpoint_dir, base_dir), manifest + + +def test_verify_sources_match_manifest_accepts_matching_pins(tmp_path): + args, manifest = pin_fixture(tmp_path) + tool.verify_sources_match_manifest(args, manifest) + + +def test_verify_sources_rejects_checkpoint_digest_drift(tmp_path): + import pytest + + args, manifest = pin_fixture(tmp_path) + manifest["sources"]["checkpoint"]["sha256"] = "00" * 32 + with pytest.raises(ValueError, match="checkpoint sha256"): + tool.verify_sources_match_manifest(args, manifest) + + +def test_verify_sources_rejects_base_revision_drift(tmp_path): + import pytest + + args, manifest = pin_fixture(tmp_path) + manifest["sources"]["base"]["revision"] = "deadbee" + with pytest.raises(ValueError, match="base revision"): + tool.verify_sources_match_manifest(args, manifest) + + +def test_verify_sources_rejects_tampered_base_shard_digest(tmp_path): + """The base weights themselves must be re-hashed against the manifest's + recorded per-shard digests: 578 tensors pass through the export byte by + byte, so a silently swapped base would poison the parity verdict.""" + + import pytest + + args, manifest = pin_fixture(tmp_path) + manifest["sources"]["base"]["weights_sha256"][ + "model.safetensors-00001-of-00001.safetensors" + ] = "00" * 32 + with pytest.raises(ValueError, match="base weights sha256"): + tool.verify_sources_match_manifest(args, manifest) + + +def test_verify_sources_rejects_swapped_base_shard_bytes(tmp_path): + import pytest + + args, manifest = pin_fixture(tmp_path) + shard = tmp_path / "base" / "model.safetensors-00001-of-00001.safetensors" + shard.write_bytes(b"different-shard-bytes") + with pytest.raises(ValueError, match="base weights sha256"): + tool.verify_sources_match_manifest(args, manifest) + + +# --- the pinned-source checkout must be clean before evidence is produced --- + +def fresh_checkout(path): + """An empty git repository standing in for the pinned source tree.""" + + import subprocess + + path.mkdir(parents=True) + subprocess.run(["git", "init", "-q", str(path)], check=True) + return path + + +def test_clean_checkout_passes(tmp_path): + repo = fresh_checkout(tmp_path / "valen-src") + # Untracked non-code cannot change what gets imported, so it is allowed. + (repo / "notes.txt").write_text("editor droppings\n", encoding="utf-8") + reference._verify_clean_checkout(repo) + + +def test_untracked_python_file_is_rejected(tmp_path): + """A stray .py on the package path can shadow or extend the pinned code + both stacks import, so it must fail the gate before evidence exists.""" + + import pytest + + repo = fresh_checkout(tmp_path / "valen-src") + (repo / "stray.py").write_text("# not pinned\n", encoding="utf-8") + with pytest.raises(SystemExit, match="stray.py"): + reference._verify_clean_checkout(repo) + + +def test_tracked_modification_is_rejected(tmp_path): + """Both stacks import from the same tree, so a local modification would + make the two agree while neither ran the pinned code.""" + + import pytest + import subprocess + + repo = fresh_checkout(tmp_path / "valen-src") + model = repo / "model.py" + model.write_text("original\n", encoding="utf-8") + subprocess.run(["git", "-C", str(repo), "add", "model.py"], check=True) + model.write_text("locally modified\n", encoding="utf-8") + with pytest.raises(SystemExit, match="model.py"): + reference._verify_clean_checkout(repo) + + +def test_sourceless_bytecode_module_is_rejected(tmp_path): + """A standalone .pyc imports by name exactly like its .py source would, + and the checked-out directory is prepended to sys.path, so a sourceless + module can shadow a dependency while the gate still reports a clean + pinned checkout unless bytecode suffixes are swept too.""" + + import py_compile + + import pytest + + repo = fresh_checkout(tmp_path / "valen-src") + source = repo / "peft.py" + source.write_text( + "REVIEW_MARKER = 'unverified module imported'\n", encoding="utf-8" + ) + py_compile.compile(str(source), cfile=str(repo / "peft.pyc"), doraise=True) + source.unlink() + with pytest.raises(SystemExit, match=r"peft\.pyc"): + reference._verify_clean_checkout(repo) + + +def test_ignored_importable_module_is_rejected(tmp_path): + """A git-ignored module hides from plain `git status`, so the sweep must + include ignored entries or a checkout-local ignore rule reopens the + shadowing path the untracked check closes.""" + + import subprocess + + import pytest + + repo = fresh_checkout(tmp_path / "valen-src") + (repo / ".gitignore").write_text("*.pyc\n", encoding="utf-8") + (repo / "README.md").write_text("fixture\n", encoding="utf-8") + subprocess.run( + ["git", "-C", str(repo), "add", ".gitignore", "README.md"], check=True + ) + subprocess.run( + [ + "git", + "-C", + str(repo), + "-c", + "user.name=Fixture", + "-c", + "user.email=fixture@example.invalid", + "commit", + "-qm", + "fixture", + ], + check=True, + ) + (repo / "peft.pyc").write_bytes(b"bytecode placeholder\n") + assert ( + subprocess.run( + ["git", "-C", str(repo), "check-ignore", "-q", "peft.pyc"] + ).returncode + == 0 + ), "fixture setup: peft.pyc must be ignored" + with pytest.raises(SystemExit, match=r"ignored importable file peft\.pyc"): + reference._verify_clean_checkout(repo) + + +def test_extension_module_is_rejected(tmp_path): + """An extension module (`peft.so` and its interpreter-tagged variants) + imports like `peft.py`, so it falls under the same gate.""" + + import pytest + + repo = fresh_checkout(tmp_path / "valen-src") + (repo / "peft.cpython-312-x86_64-linux-gnu.so").write_bytes(b"stub\n") + with pytest.raises(SystemExit, match=r"peft\.cpython-312"): + reference._verify_clean_checkout(repo) + + +def test_bytecode_cache_directory_is_allowed(tmp_path): + """`__pycache__` bytecode is the cache of a real import and is not + importable on its own, so a re-run after a first import must stay + clean instead of failing on its own cache.""" + + repo = fresh_checkout(tmp_path / "valen-src") + cache = repo / "__pycache__" + cache.mkdir() + (cache / "peft.cpython-312.pyc").write_bytes(b"cache placeholder\n") + reference._verify_clean_checkout(repo) + + +def test_export_parity_shares_the_clean_checkout_gate(): + """Both evidence entrypoints must run the same corrected gate: the + export harness stages its environment through the comparison tool's + `_verify_clean_checkout`, not a private copy.""" + + import inspect + + assert "reference._verify_clean_checkout" in inspect.getsource( + tool.stage_environment + ) + + +# --- the full parity gate against the real pinned artifacts --- +# Set VALEN_CKPT, VALEN_BASE and VALEN_SRC (the recipe reference.md artifact +# exports) on a CUDA host to run: it exports a fresh artifact set, then runs +# the two serial model loads (original checkpoint stack, exported stack) and +# fills the manifest parity slot. The unmerged export is expected to pass the +# declared gate; the test asserts the verdict and its internal consistency. + +def test_real_artifact_export_parity_gate(tmp_path): + import subprocess + import torch + + env_vars = ("VALEN_CKPT", "VALEN_BASE", "VALEN_SRC") + if not all(os.environ.get(name) for name in env_vars): + pytest.skip("set VALEN_CKPT, VALEN_BASE and VALEN_SRC to run") + if not torch.cuda.is_available(): + pytest.skip("needs a CUDA device") + + import export_checkpoint as exporter + + export_dir = tmp_path / "export" + exporter.export( + Path(os.environ["VALEN_CKPT"]), Path(os.environ["VALEN_BASE"]), export_dir + ) + + result = subprocess.run( + [ + sys.executable, + str(RECIPE_VALEN / "verify_export_parity.py"), + "--valen-source", os.environ["VALEN_SRC"], + "--checkpoint", os.environ["VALEN_CKPT"], + "--base", os.environ["VALEN_BASE"], + "--export", str(export_dir), + "--device", "cuda", + ], + capture_output=True, + text=True, + timeout=3600, + ) + assert result.returncode == 0, result.stdout[-4000:] + result.stderr[-4000:] + + manifest = exporter.validate_export(export_dir) + parity = manifest["parity"] + assert parity["status"] == "passed" + assert parity["device"].startswith("cuda") + assert parity["tolerance"] == 1e-6 + assert {case["case"] for case in parity["cases"]} == {"text", "image"} + assert parity["exact_answer_agreement"] is True + assert all(case["exact_answer_agreement"] for case in parity["cases"]) + assert parity["max_probability_delta"] == max( + case["max_probability_delta"] for case in parity["cases"] + ) + assert parity["max_probability_delta"] <= parity["tolerance"] + for case in parity["cases"]: + assert case["passed"] is True + assert case["problems"] == [] diff --git a/tests/valen/test_postprocess.py b/tests/valen/test_postprocess.py index 837d420e..40dcc54d 100644 --- a/tests/valen/test_postprocess.py +++ b/tests/valen/test_postprocess.py @@ -48,3 +48,83 @@ def test_build_response_preserves_question_order_and_usage(): "usage": {"input_tokens": 17, "output_tokens": 0}, "internal_usage": {"compute_tokens": 34}, } + + +NOUL = Question("ask", "Was it duplicated?", ("true", "false"), (None, None), "noul") + + +def test_noul_answer_reports_true_probability_and_full_pair(): + result = answer(NOUL, [0.0, math.log(3.0)]) + + assert result["type"] == "noul" + assert result["noul"] == pytest.approx(0.25) + assert result["probabilities"] == { + "true": pytest.approx(0.25), + "false": pytest.approx(0.75), + } + # Valen's choice confidence formula evaluated on the two candidates. + assert result["confidence"] == pytest.approx(0.5) + + flipped = answer(NOUL, [math.log(3.0), 0.0]) + assert flipped["noul"] == pytest.approx(0.75) + + +def test_noul_answer_honors_temperature_and_rejects_bad_logits(): + sharp = answer(NOUL, [1.0, 0.0], temperature=0.5) + plain = answer(NOUL, [1.0, 0.0], temperature=1.0) + assert sharp["noul"] > plain["noul"] > 0.5 + + with pytest.raises(ValueError, match="one logit"): + answer(NOUL, [0.0]) + with pytest.raises(ValueError, match="invalid logits"): + answer(NOUL, [float("nan"), 0.0]) + + +SCORE = Question( + "grade", + "Grade the severity.", + ("0", "1", "2"), + ("No error", "A minor error", "A major error"), + "score", +) + + +def test_score_answer_reports_expected_level_and_distribution(): + result = answer(SCORE, [0.0, 0.0, 0.0]) + + assert result["type"] == "score" + assert result["probabilities"] == {"0": pytest.approx(1 / 3), "1": pytest.approx(1 / 3), "2": pytest.approx(1 / 3)} + assert result["legend"] == {"0": "No error", "1": "A minor error", "2": "A major error"} + assert result["score"] == pytest.approx(1.0) + # A uniform distribution has no concentration. + assert result["confidence"] == pytest.approx(0.0) + + peaked = answer(SCORE, [-100.0, 3.0, -100.0]) + assert peaked["score"] == pytest.approx(1.0) + assert peaked["confidence"] == pytest.approx(1.0) + + +def test_score_answer_confidence_penalizes_split_mass(): + # Mass split between the two endpoints: mode is 0 (ties go to the + # earliest level), distance = 0.6 * 0 + 0.4 * 2 = 0.8 against a uniform + # mean absolute deviation of 2/3, so confidence is 0 (clamped). + result = answer(SCORE, [math.log(1.5), -50.0, 0.0]) + assert result["probabilities"]["0"] == pytest.approx(0.6) + assert result["probabilities"]["2"] == pytest.approx(0.4, abs=1e-6) + assert result["score"] == pytest.approx(0.8) + assert result["confidence"] == 0.0 + + # A tilted but concentrated distribution: p = [0.7, 0.2, 0.1] has mode 0, + # distance 0.2*1 + 0.1*2 = 0.4 and uniform MAD 2/3, so confidence is + # 1 - 0.4/(2/3) = 0.4. + tilted = answer(SCORE, [math.log(7.0), math.log(2.0), 0.0]) + assert tilted["probabilities"]["0"] == pytest.approx(0.7) + assert tilted["score"] == pytest.approx(0.4) + assert tilted["confidence"] == pytest.approx(0.4) + + +def test_score_answer_rejects_wrong_logit_counts(): + with pytest.raises(ValueError, match="one logit"): + answer(SCORE, [0.0, 1.0]) + with pytest.raises(RequestError, match="temperature"): + answer(SCORE, [0.0, 1.0, 2.0], temperature=-1.0) diff --git a/tests/valen/test_preprocess.py b/tests/valen/test_preprocess.py index 5f8280dc..accbb65b 100644 --- a/tests/valen/test_preprocess.py +++ b/tests/valen/test_preprocess.py @@ -1,7 +1,12 @@ import hashlib -from models.valen.preprocess import prepare_request -from models.valen.protocol import MODEL_NAME, parse_request +from models.valen.preprocess import prepare_request, question_record +from models.valen.protocol import ( + MODEL_NAME, + NOUL_FALSE_DESCRIPTION, + NOUL_TRUE_DESCRIPTION, + parse_request, +) from .test_protocol import body @@ -52,3 +57,113 @@ def test_prepare_compiles_text_state_without_media(tmp_path): } assert list(prepared.record["request"]["questions"]) == ["move"] assert list(tmp_path.iterdir()) == [] + + +def noul_question(criteria=None): + question = {"type": "noul", "instructions": "Was the card charged twice?"} + if criteria is not None: + question["criteria"] = criteria + request = parse_request( + body(state="The card was charged twice.", questions={"ask": question}) + ) + return request.questions[0] + + +def test_question_record_keeps_reference_noul_wording_native(): + record = question_record(noul_question()) + + assert record == {"type": "noul", "instructions": "Was the card charged twice?"} + + explicit = question_record( + noul_question( + criteria={ + "true": NOUL_TRUE_DESCRIPTION, + "false": NOUL_FALSE_DESCRIPTION, + } + ) + ) + assert explicit == record + + +def test_question_record_carries_custom_noul_wording_as_choice(): + record = question_record( + noul_question(criteria={"true": "It was duplicated.", "false": "It was unique."}) + ) + + assert record == { + "type": "choice", + "instructions": "Was the card charged twice?", + "criteria": {"true": "It was duplicated.", "false": "It was unique."}, + } + + partial = question_record(noul_question(criteria={"false": "It was unique."})) + assert partial["criteria"] == { + "true": NOUL_TRUE_DESCRIPTION, + "false": "It was unique.", + } + + +def test_question_record_passes_score_levels_in_order(): + request = parse_request( + body( + state="Dialog: the update finished.", + questions={ + "grade": { + "type": "score", + "instructions": "Grade the dialog.", + "criteria": ["unclear", "clear"], + } + }, + ) + ) + + assert question_record(request.questions[0]) == { + "type": "score", + "instructions": "Grade the dialog.", + "criteria": ["unclear", "clear"], + } + + +def test_prepare_builds_mixed_type_records(tmp_path): + request = parse_request( + body( + state="The card was charged twice for one order.", + questions={ + "ask": { + "type": "noul", + "instructions": "Was the card charged twice?", + }, + "grade": { + "type": "score", + "instructions": "Grade the severity.", + "criteria": ["No error", "A minor error", "A major error"], + }, + "act": { + "type": "choice", + "instructions": "Decide the refund action.", + "criteria": {"refund": None, "wait": "Wait for review"}, + }, + }, + ) + ) + prepared = prepare_request(request, tmp_path) + + assert prepared.record["request"]["questions"] == { + "ask": {"type": "noul", "instructions": "Was the card charged twice?"}, + "grade": { + "type": "score", + "instructions": "Grade the severity.", + "criteria": ["No error", "A minor error", "A major error"], + }, + "act": { + "type": "choice", + "instructions": "Decide the refund action.", + "criteria": {"refund": "refund", "wait": "Wait for review"}, + }, + } + assert prepared.record["assets"] == [] + assert [q.kind for q in prepared.response_context.questions] == [ + "noul", + "score", + "choice", + ] diff --git a/tests/valen/test_protocol.py b/tests/valen/test_protocol.py index 3f073fd2..940c5a09 100644 --- a/tests/valen/test_protocol.py +++ b/tests/valen/test_protocol.py @@ -76,14 +76,19 @@ def test_parse_rejects_wrong_model_and_invalid_state(): parse_request(body(state=17)) -def test_parse_allows_structured_text_but_not_score_or_extra_fields(): +def test_parse_allows_structured_text_but_rejects_unknown_types_and_fields(): value = body() value["questions"]["move"]["instructions"] = {"goal": "choose"} assert parse_request(value).questions[0].instructions == '{"goal": "choose"}' value = body() - value["questions"]["move"]["type"] = "score" - with pytest.raises(RequestError, match="only choice"): + value["questions"]["move"]["type"] = "rank" + with pytest.raises(RequestError, match="type must be one of choice, noul, score"): + parse_request(value) + + value = body() + del value["questions"]["move"]["type"] + with pytest.raises(RequestError, match="type must be one of"): parse_request(value) value = body() @@ -158,3 +163,129 @@ def test_parse_rejects_media_control_tokens(): with pytest.raises(RequestError, match="control token"): parse_request(body(state="context <|video_pad|> tail")) + + +def noul_body(**overrides): + value = body( + questions={ + "ask": {"type": "noul", "instructions": "Was the card charged twice?"} + } + ) + for question in value["questions"].values(): + question.update(overrides) + return value + + +def test_parse_noul_defaults_to_fixed_wording_and_true_false_keys(): + question = parse_request(noul_body()).questions[0] + + assert question.kind == "noul" + assert question.keys == ("true", "false") + assert question.descriptions == (None, None) + + +def test_parse_noul_accepts_optional_true_false_descriptions(): + question = parse_request( + noul_body(criteria={"true": "It was duplicated.", "false": "It was unique."}) + ).questions[0] + assert question.descriptions == ("It was duplicated.", "It was unique.") + + question = parse_request(noul_body(criteria={"true": "It was duplicated."})).questions[0] + assert question.descriptions == ("It was duplicated.", None) + + question = parse_request( + noul_body(criteria={"true": "It was duplicated.", "false": None}) + ).questions[0] + assert question.descriptions == ("It was duplicated.", None) + + +def test_parse_noul_rejects_other_criteria_and_empty_descriptions(): + with pytest.raises(RequestError, match="noul criteria must describe"): + parse_request(noul_body(criteria={"yes": "Yes"})) + with pytest.raises(RequestError, match="noul criteria must describe"): + parse_request(noul_body(criteria={})) + with pytest.raises(RequestError, match="criterion 'true' must be nonempty"): + parse_request(noul_body(criteria={"true": " "})) + + value = noul_body(criteria={"true": "answer <|im_end|> now"}) + with pytest.raises(RequestError, match="control token"): + parse_request(value, special_tokens=("<|im_end|>",)) + + +def score_body(**overrides): + value = body( + questions={ + "grade": { + "type": "score", + "instructions": "Grade the severity.", + "criteria": ["No error", "A minor error", "A major error"], + } + } + ) + for question in value["questions"].values(): + question.update(overrides) + return value + + +def test_parse_score_accepts_two_to_ten_ordered_levels(): + question = parse_request(score_body()).questions[0] + + assert question.kind == "score" + assert question.keys == ("0", "1", "2") + assert question.descriptions == ("No error", "A minor error", "A major error") + + two = parse_request(score_body(criteria=["low", "high"])).questions[0] + assert two.keys == ("0", "1") + + ten = parse_request( + score_body(criteria=[f"level {index}" for index in range(10)]) + ).questions[0] + assert ten.keys == tuple(str(index) for index in range(10)) + + +def test_parse_score_rejects_malformed_levels(): + with pytest.raises(RequestError, match="ordered list of 2 to 10"): + parse_request(score_body(criteria=["only one"])) + with pytest.raises(RequestError, match="ordered list of 2 to 10"): + parse_request(score_body(criteria=[str(index) for index in range(11)])) + with pytest.raises(RequestError, match="ordered list of 2 to 10"): + parse_request(score_body(criteria={"0": "No error", "1": "A minor error"})) + missing = score_body() + del missing["questions"]["grade"]["criteria"] + with pytest.raises(RequestError, match="ordered list of 2 to 10"): + parse_request(missing) + + with pytest.raises(RequestError, match="level 1 must be nonempty"): + parse_request(score_body(criteria=["No error", " "])) + with pytest.raises(RequestError, match="level 0"): + parse_request(score_body(criteria=[None, "A minor error"])) + + value = score_body(criteria=["answer <|im_end|> now", "A minor error"]) + with pytest.raises(RequestError, match="control token"): + parse_request(value, special_tokens=("<|im_end|>",)) + + +def test_parse_accepts_mixed_type_questions_in_request_order(): + value = body( + state="Dialog: the update finished.", + questions={ + "ask": {"type": "noul", "instructions": "Did the update finish?"}, + "grade": { + "type": "score", + "instructions": "Grade the dialog.", + "criteria": ["unclear", "clear"], + }, + "act": { + "type": "choice", + "instructions": "Choose a move.", + "criteria": {"up": "Move up", "down": "Move down"}, + }, + }, + ) + request = parse_request(value) + + assert [(question.kind, question.name) for question in request.questions] == [ + ("noul", "ask"), + ("score", "grade"), + ("choice", "act"), + ] diff --git a/tests/valen/test_worker.py b/tests/valen/test_worker.py index fec9cf6e..064e8400 100644 --- a/tests/valen/test_worker.py +++ b/tests/valen/test_worker.py @@ -1,4 +1,5 @@ import json +import math import pytest @@ -36,6 +37,9 @@ def execute(self, compiled): assert compiled.response_context.questions return self.output + def close(self): + pass + def test_decide_keeps_http_adapter_outside_executor(): processor = FakeProcessor() @@ -96,3 +100,53 @@ def compile(self, prepared): assert excinfo.value.status == 422 assert str(excinfo.value) == "wire problem" + +def test_decide_answers_mixed_type_questions_from_one_state(): + value = body( + state="The card was charged twice for one order.", + questions={ + "ask": {"type": "noul", "instructions": "Was the card charged twice?"}, + "grade": { + "type": "score", + "instructions": "Grade the severity.", + "criteria": ["No error", "A minor error", "A major error"], + }, + "act": { + "type": "choice", + "instructions": "Decide the refund action.", + "criteria": {"refund": None, "wait": "Wait for review"}, + }, + }, + ) + processor = FakeProcessor() + executor = FakeExecutor( + ExecutorOutput( + ( + (math.log(3.0), 0.0), # noul: true, false + (math.log(7.0), math.log(2.0), 0.0), # score levels 0, 1, 2 + (0.0, 1.0), # choice: refund, wait + ), + 61, + 183, + ) + ) + + result = decide(json.dumps(value).encode("utf-8"), processor, executor) + + noul = result["answers"]["ask"] + assert noul["type"] == "noul" + assert noul["noul"] == pytest.approx(0.75) + assert noul["probabilities"]["true"] == pytest.approx(0.75) + assert noul["probabilities"]["false"] == pytest.approx(0.25) + + score = result["answers"]["grade"] + assert score["type"] == "score" + assert score["probabilities"]["0"] == pytest.approx(0.7) + assert score["score"] == pytest.approx(0.4) + assert score["legend"] == {"0": "No error", "1": "A minor error", "2": "A major error"} + assert score["confidence"] == pytest.approx(0.4) + + assert result["answers"]["act"]["choice"] == "wait" + assert result["usage"] == {"input_tokens": 61, "output_tokens": 0} + assert result["internal_usage"] == {"compute_tokens": 183} + diff --git a/tests/valen/test_worker_server.py b/tests/valen/test_worker_server.py index 66615a11..c71e9313 100644 --- a/tests/valen/test_worker_server.py +++ b/tests/valen/test_worker_server.py @@ -11,6 +11,7 @@ from __future__ import annotations import json +import math import threading from http.client import HTTPConnection @@ -164,6 +165,43 @@ def test_client_errors_map_to_their_status_and_detail(server): assert status == 422 and "nonempty" in json.loads(raw)["detail"] +def test_noul_and_score_questions_are_served_over_http(): + executor = FakeExecutor( + ExecutorOutput(((0.0, 1.0), (math.log(7.0), math.log(2.0), 0.0)), 5, 11) + ) + mixed = start(executor) + try: + value = body( + state="The card was charged twice for one order.", + questions={ + "ask": {"type": "noul", "instructions": "Was the card charged twice?"}, + "grade": { + "type": "score", + "instructions": "Grade the severity.", + "criteria": ["No error", "A minor error", "A major error"], + }, + }, + ) + status, _, raw = post(mixed, json.dumps(value).encode("utf-8")) + assert status == 200 + result = json.loads(raw) + assert result["answers"]["ask"]["type"] == "noul" + assert result["answers"]["grade"]["type"] == "score" + assert result["internal_usage"] == {"compute_tokens": 11} + finally: + stop(mixed) + + +def test_invalid_score_levels_map_to_422(server): + value = body( + questions={ + "grade": {"type": "score", "instructions": "Grade.", "criteria": ["only"]} + } + ) + status, _, raw = post(server, json.dumps(value).encode("utf-8")) + assert status == 422 and "2 to 10" in json.loads(raw)["detail"] + + def test_busy_worker_returns_503(server): server.inference_lock.acquire() try: