diff --git a/README.md b/README.md index 07815a5c..b154cdcf 100644 --- a/README.md +++ b/README.md @@ -172,7 +172,7 @@ pinned runtime for text state: | Cua-S1 4B 0.2 (`text` adapter) | [Python worker](recipe/cua_s1/text.md); [native worker](recipe/cua_s1/native.md), CUDA; unmerged profile validated on the cumulative A800 sm_80 build, earlier merged path on sm_89 | | Cua-S1 4B 0.2 (`multimodal` adapter) | [Python CUDA reference](src/frontend/cua_s1.py) and [native Rust/CUDA screenshot worker](recipe/cua_s1/native_screenshot.md); one PNG/JPEG image, `choice`; scoped RTX 4090 validation | | JEMM (Qwen3.8-27B) | [Native Rust/CUDA worker](recipe/jemm/README.md); [text and 1–4-image frozen-corpus parity on A800](src/models/jemm/README.md#validation) | -| Valen-Preview-0923 | [Python reference worker](recipe/valen/reference.md); text state or one inline PNG/JPEG image, `choice`; author-run on RTX 5060 Laptop GPU (sm_120, WSL2), owner: community | +| Valen-Preview-0923 | [Python reference worker](recipe/valen/reference.md); text state or one inline PNG/JPEG image, `choice`/`noul`/`score`; author-run on RTX 5060 Laptop GPU (sm_120, WSL2), owner: community | | Open-Jev-27B-v1.1 | [Native Rust/CUDA worker](recipe/open_jev/native.md); eager independent text candidates; [H200 validation](recipe/open_jev/validation.md) | | Open-Jev-9B | The same [native Rust/CUDA worker](recipe/open_jev/native.md); eager independent text candidates; [reference comparison on sm_89](recipe/open_jev/validation-9b.md) | | autotrust/JEV-27B-VL | [Experimental Rust/CUDA worker](recipe/jev_vl/README.md); single-question text and offline-preencoded image inputs; [bounded H800 validation and limits](recipe/jev_vl/validation.md) | diff --git a/docs/supported-models.md b/docs/supported-models.md index aecd6670..9eaebb8b 100644 --- a/docs/supported-models.md +++ b/docs/supported-models.md @@ -16,7 +16,7 @@ Models that are being added are also tracked in issues labeled [new model](https | Cua-S1 4B 0.2, `text` adapter | [Native Rust worker](../recipe/cua_s1/native.md) on the [Qwen3.5 CUDA kernels](../src/backends/cuda/qwen3_5/README.md) | Not supported | Validated on compute capability 8.9 ([#19](https://github.com/ThinkFlowLab/system1-omni/pull/19), [#52](https://github.com/ThinkFlowLab/system1-omni/pull/52)) | Not supported | Compute capability 8.0 or newer, the CUDA toolkit to build, weights merged with `export_text_merged.py` | | Cua-S1 4B 0.2, `multimodal` adapter | Reference worker on Transformers and PEFT, [`src/frontend/cua_s1.py`](../src/frontend/cua_s1.py); no recipe yet | Not supported | Validated ([#17](https://github.com/ThinkFlowLab/system1-omni/pull/17), [#18](https://github.com/ThinkFlowLab/system1-omni/pull/18)) | Not supported | The state is one PNG or JPEG image; upstream's `weights.lock.json` next to the base weights | | Cua-S1 4B 0.2, `multimodal` adapter | [Native Rust/CUDA screenshot worker](../recipe/cua_s1/native_screenshot.md) | Preparation checks only; no CPU inference | RTX 4090 / sm_89, native vision and language Graph regressions; [scoped evidence and controls](../recipe/cua_s1/native_vision_graph.md) | Not supported | Verified BF16 language export, original vision checkpoint and FP32 vision adapter, rebuilt ABI 9 CUDA library; one PNG/JPEG and `choice` | -| Valen-Preview-0923 | [Python reference worker](../recipe/valen/reference.md); text state or one inline PNG/JPEG image; `choice` questions | Not supported | Author-run on RTX 5060 Laptop (sm_120, WSL2): one text and one 2×2 PNG request; input and readout parity ≤1e-6 against reference (model outputs not compared). Owner: community. | Not supported | Python 3.12, pinned Valen source/checkpoint, Qwen3.5-2B base, BF16 CUDA | +| Valen-Preview-0923 | [Python reference worker](../recipe/valen/reference.md); text state or one inline PNG/JPEG image; `choice`, `noul`, and `score` questions | Not supported | Author-run on RTX 5060 Laptop (sm_120, WSL2): one text and one 2×2 PNG request; input and readout parity ≤1e-6 against reference (model outputs not compared); `noul`/`score` parity ≤1e-6 with exact legend and token agreement on the recipe's text fixtures. Owner: community. | Not supported | Python 3.12, pinned Valen source/checkpoint, Qwen3.5-2B base, BF16 CUDA | | Open-Jev-27B-v1.1 | [Native Rust/CUDA worker](../recipe/open_jev/native.md) on the shared Qwen3.5/3.8 executor | Not supported | Validated on H200 (sm_90) for the [74 single-candidate workload](../recipe/open_jev/validation.md) | Not supported | Compute capability 8.0 or newer, CUDA toolkit to build, exported merged weights and trained head | | Open-Jev-9B | The same [native Rust/CUDA worker](../recipe/open_jev/native.md) | Not supported | Validated on compute capability 8.9 against the reference for [253 requests](../recipe/open_jev/validation-9b.md) | Not supported | Compute capability 8.0 or newer, CUDA toolkit to build, exported merged weights and trained head | | JEMM / Qwen3.8-27B | [Native Rust/CUDA worker](../recipe/jemm/README.md) | Host checks only; no CPU inference | A800 (`sm_80`) [text and 1–4-image frozen corpus](../src/models/jemm/README.md#validation) | Not supported | Pinned `jemm-native/2` export with BF16 base and unmerged FP32 LoRA, selected untied LM head, cuDNN 9 and additive reference CUDA symbols | diff --git a/recipe/README.md b/recipe/README.md index a1a77b0f..a90b02db 100644 --- a/recipe/README.md +++ b/recipe/README.md @@ -12,7 +12,8 @@ For a first real decision, follow the [complete CPU walkthrough](../docs/getting - [Cua-S1 4B 0.2 text worker](cua_s1/text.md): download the pinned weights, start the worker and connect the Rust frontend. - [Valen-Preview-0923 reference worker](valen/reference.md): prepare the pinned - Preview checkpoint and Qwen base, start the text-or-image `choice` worker and verify the frontend. + Preview checkpoint and Qwen base, start the text-or-image `choice`/`noul`/`score` + worker and verify the frontend. - [Cua-S1 4B 0.2 native text worker](cua_s1/native.md): build the CUDA library and the Rust worker, export BF16 weights with the original FP32 adapter and start the worker. diff --git a/recipe/valen/compare_reference.py b/recipe/valen/compare_reference.py index 3e355924..2021d847 100644 --- a/recipe/valen/compare_reference.py +++ b/recipe/valen/compare_reference.py @@ -4,17 +4,22 @@ Runs each request through two paths with the same pinned artifacts: - reference: a Valen-native record (a plain-string text state, or a local - image with a SHA-256 asset) compiled by the pinned ``Compiler`` and decoded - with a plain softmax plus Valen's confidence formula; + image with a SHA-256 asset) compiled by the pinned ``Compiler``; ``choice`` + cases are decoded with a plain softmax plus Valen's confidence formula, + while ``noul`` and ``score`` cases are decoded by the pinned reference + ``valen.evaluation.inference.answer`` module itself; - worker: the production ``frontend.valen.decide`` pipeline (protocol, preprocessing, executor, postprocessing). Both paths share one loaded model instance, so the comparison isolates input construction, candidate ordering, decision-head readout and probability normalization instead of checkpoint loading. The declared criteria are exact -decision agreement, equal token accounting and a probability tolerance of -1e-6 (``--tolerance``). Requires the pinned Valen source, the Preview -checkpoint, the Qwen3.5-2B base and a working torch device; see +discrete agreement (candidate order, the requested answer type, the chosen +key, the score legend), equal token accounting and a tolerance of 1e-6 +(``--tolerance``) on probabilities, confidences and the scalar answer fields +(``noul`` P(true), ``score`` expected level), which the two softmax +implementations never match bit-exactly. Requires the pinned Valen source, +the Preview checkpoint, the Qwen3.5-2B base and a working torch device; see recipe/valen/reference.md. """ @@ -24,6 +29,7 @@ import base64 import hashlib import json +import math import sys from pathlib import Path from tempfile import TemporaryDirectory @@ -51,7 +57,11 @@ def parse_args(argv=None): parser.add_argument("--base", required=True, help="Qwen3.5-2B base directory") parser.add_argument("--device", default="cuda") parser.add_argument("--dtype", choices=("bf16", "fp32"), default="bf16") - parser.add_argument("--case", choices=("image", "text", "both"), default="both") + parser.add_argument( + "--case", + choices=("image", "text", "noul", "score", "both", "all"), + default="both", + ) parser.add_argument( "--fixture", default=str(HERE / "example-request.json"), @@ -62,6 +72,16 @@ def parse_args(argv=None): default=None, help="text-case request body JSON file (default: the recipe text example)", ) + parser.add_argument( + "--noul-body", + default=str(HERE / "example-noul-request.json"), + help="noul-case request body JSON file (default: the recipe noul example)", + ) + parser.add_argument( + "--score-body", + default=str(HERE / "example-score-request.json"), + help="score-case request body JSON file (default: the recipe score example)", + ) parser.add_argument("--tolerance", type=float, default=PROBABILITY_TOLERANCE) return parser.parse_args(argv) @@ -78,20 +98,40 @@ def decode_image_data_url(url): return base64.b64decode(encoded, validate=True), expected[prefix] -def reference_questions(body): - """Rebuild the Valen question mapping straight from the fixture body.""" +def reference_questions(body, special_tokens=None): + """Rebuild the Valen question mapping straight from the fixture body. - return { - name: { - "type": "choice", - "instructions": question.get("instructions") or "", - "criteria": { - key: (label if label is not None else key) - for key, label in question["criteria"].items() - }, - } - for name, question in body["questions"].items() - } + The questions go through the worker's own ``parse_request`` + + ``question_record`` path, so the reference record cannot drift from the + one the worker compiles: custom ``noul`` wording becomes the same + ``true``/``false`` choice record, and unspecified sides get the same + fixed reference wording. + """ + + from models.valen.preprocess import question_record + from models.valen.protocol import DEFAULT_SPECIAL_TOKENS, parse_request + + request = parse_request( + body, special_tokens=special_tokens or DEFAULT_SPECIAL_TOKENS + ) + return {question.name: question_record(question) for question in request.questions} + + +def expected_answer_types(body, special_tokens=None): + """The requested public answer type per question, from the wire body. + + The worker's public ``type`` field mirrors the request's question type, + so parity dispatches on the requested type: an answer that drifts to + another shape (for example a ``noul`` answered as a plain choice) fails + the comparison instead of silently skipping the noul/score checks. + """ + + from models.valen.protocol import DEFAULT_SPECIAL_TOKENS, parse_request + + request = parse_request( + body, special_tokens=special_tokens or DEFAULT_SPECIAL_TOKENS + ) + return {question.name: question.kind for question in request.questions} def reference_image_record(image_bytes, image_format, questions, media_dir): @@ -137,33 +177,144 @@ def reference_confidence(probabilities): return max(0.0, (max(probabilities) - 1 / count) / (1 - 1 / count)) -def compare_case(name, worker_response, reference, tolerance=PROBABILITY_TOLERANCE): +def scalar_answer_fields(answer, expected_type, ref_answer): + """Every scalar the per-type branches fold into a running maximum. + + max() keeps its first argument against a NaN difference, so each field + here is swept for finiteness before those branches run. Reference values + derived from the probabilities (the choice-shaped noul decode, the + fallback score) are already covered by the probability gate. + """ + + fields = [answer["confidence"]] + if expected_type == "noul": + fields.append(answer["noul"]) + elif expected_type == "score": + fields.append(answer["score"]) + if ref_answer is not None: + if expected_type == "noul" and "noul" in ref_answer: + fields.append(ref_answer["noul"]) + elif expected_type == "score" and "score" in ref_answer: + fields.append(ref_answer["score"]) + if "confidence" in ref_answer: + fields.append(ref_answer["confidence"]) + return fields + + +def compare_case( + name, worker_response, reference, expected_types, tolerance=PROBABILITY_TOLERANCE +): """Compare one worker response against one reference result. - Returns (report lines, passed). + ``expected_types`` maps each question id to its requested public answer + type; every answer must keep that type before any per-type comparison + runs. ``choice`` answers must agree on the chosen key; ``noul`` answers + on P(true); ``score`` answers on the expected level and the level legend. + All scalar comparisons — probabilities, confidences and the scalar answer + fields — use the declared tolerance because the worker and the pinned + reference normalize probabilities through different softmax + implementations. Returns (report lines, passed). """ lines = [f"case: {name}", "reference model: Valen", f"worker model: {worker_response['model']}"] problems = [] + legend_problems = [] decision_exact = True + legend_exact = True max_prob_delta = 0.0 max_conf_delta = 0.0 + max_answer_delta = 0.0 + scalar_answers = False for qid, ref_keys in reference["keys"].items(): answer = worker_response["answers"][qid] + expected_type = expected_types[qid] + if answer["type"] != expected_type: + problems.append( + f"{qid}: answer type {answer['type']!r} != requested {expected_type!r}" + ) + decision_exact = False + continue worker_keys = list(answer["probabilities"]) if worker_keys != list(ref_keys): problems.append(f"{qid}: candidate order {worker_keys} != {list(ref_keys)}") decision_exact = False continue ref_probs = reference["probabilities"][qid] + # A NaN difference keeps the previous finite maximum and a NaN-free + # argmax list silently keeps its first candidate, so a non-finite + # probability on either side fails the case before any comparison. + if any( + not math.isfinite(value) + for value in list(answer["probabilities"].values()) + list(ref_probs) + ): + problems.append( + f"{qid}: non-finite probability (NaN or infinity) on the " + "worker or reference side" + ) + decision_exact = False + continue for key, worker_p, ref_p in zip(ref_keys, answer["probabilities"].values(), ref_probs): max_prob_delta = max(max_prob_delta, abs(worker_p - ref_p)) - decision = ref_keys[max(range(len(ref_probs)), key=ref_probs.__getitem__)] - if answer["choice"] != decision: - problems.append(f"{qid}: worker chose {answer['choice']!r}, reference {decision!r}") + # The pinned reference decode when the harness ran with the pinned + # source; synthetic references may omit it and fall back to the + # shared probabilities and formulas. + ref_answer = reference.get("answers", {}).get(qid) + # The scalar deltas below fold into running maxima, and max() keeps + # its first argument against a NaN difference, so a non-finite + # scalar field on either side fails the case before any tolerance + # verdict. + if any( + not math.isfinite(value) + for value in scalar_answer_fields(answer, expected_type, ref_answer) + ): + problems.append( + f"{qid}: non-finite scalar answer or confidence (NaN or " + "infinity) on the worker or reference side" + ) decision_exact = False - ref_confidence = reference_confidence(ref_probs) - max_conf_delta = max(max_conf_delta, abs(answer["confidence"] - ref_confidence)) + continue + if expected_type == "noul": + scalar_answers = True + if ref_answer is not None and "noul" in ref_answer: + ref_p_true = ref_answer["noul"] + elif ref_answer is not None: + # Custom-wording noul compiles as a choice record, so the + # pinned decoder answers it in choice shape. + ref_p_true = ref_answer["probabilities"]["true"] + else: + ref_p_true = ref_probs[ref_keys.index("true")] + max_answer_delta = max(max_answer_delta, abs(answer["noul"] - ref_p_true)) + if ref_answer is not None and "confidence" in ref_answer: + ref_confidence = ref_answer["confidence"] + else: + # The reference noul answer carries no confidence; the worker + # exposes Valen's choice formula evaluated on the pair. + ref_confidence = reference_confidence(ref_probs) + max_conf_delta = max(max_conf_delta, abs(answer["confidence"] - ref_confidence)) + elif expected_type == "score": + scalar_answers = True + ref_score = ( + ref_answer["score"] + if ref_answer is not None + else sum(index * p for index, p in enumerate(ref_probs)) + ) + max_answer_delta = max(max_answer_delta, abs(answer["score"] - ref_score)) + if ref_answer is not None and "legend" in ref_answer: + if answer["legend"] != ref_answer["legend"]: + legend_problems.append(f"{qid}: score legend mismatch") + legend_exact = False + if ref_answer is not None and "confidence" in ref_answer: + ref_confidence = ref_answer["confidence"] + else: + ref_confidence = reference_confidence(ref_probs) + max_conf_delta = max(max_conf_delta, abs(answer["confidence"] - ref_confidence)) + else: + decision = ref_keys[max(range(len(ref_probs)), key=ref_probs.__getitem__)] + if answer["choice"] != decision: + problems.append(f"{qid}: worker chose {answer['choice']!r}, reference {decision!r}") + decision_exact = False + ref_confidence = reference_confidence(ref_probs) + max_conf_delta = max(max_conf_delta, abs(answer["confidence"] - ref_confidence)) usage = worker_response["usage"] compute = worker_response["internal_usage"]["compute_tokens"] @@ -184,25 +335,48 @@ def compare_case(name, worker_response, reference, tolerance=PROBABILITY_TOLERAN f" {reference['compute_tokens']}" ) lines.append(f"max probability delta: {max_prob_delta:.10e}") + if scalar_answers: + lines.append(f"max answer delta: {max_answer_delta:.10e}") lines.append(f"confidence delta: {max_conf_delta:.10e}") lines.append( "decision agreement: " + ("PASS" if decision_exact else "FAIL: " + "; ".join(problems)) ) prob_ok = max_prob_delta <= tolerance + confidence_ok = max_conf_delta <= tolerance lines.append( f"probability tolerance <= {tolerance:g}: " + ("PASS" if prob_ok else "FAIL") ) + lines.append( + f"confidence tolerance <= {tolerance:g}: " + + ("PASS" if confidence_ok else "FAIL") + ) + if scalar_answers: + answer_ok = max_answer_delta <= tolerance + lines.append( + f"answer tolerance <= {tolerance:g}: " + ("PASS" if answer_ok else "FAIL") + ) + if any("legend" in answer for answer in worker_response["answers"].values()): + lines.append( + "legend agreement: " + + ("PASS" if legend_exact else "FAIL: " + "; ".join(legend_problems)) + ) lines.append("token accounting: " + ("PASS" if tokens_exact else "FAIL")) - return lines, decision_exact and prob_ok and tokens_exact + passed = ( + decision_exact and legend_exact and prob_ok and confidence_ok and tokens_exact + ) + if scalar_answers: + passed = passed and max_answer_delta <= tolerance + return lines, passed def run_reference(model, base_processor, record, media_root, max_length, media_kwargs): - """Compile a Valen-native record and decode it with a plain softmax.""" + """Compile a Valen-native record and decode it with the pinned modules.""" import torch from valen.data.compiler import Compiler + from valen.evaluation.inference import answer as reference_answer compiler = Compiler( base_processor, @@ -213,14 +387,19 @@ def run_reference(model, base_processor, record, media_root, max_length, media_k state = compiler.compile(record) keys = {} probabilities = {} + answers = {} with torch.no_grad(): for question in state.questions: logits = model(question).float().cpu() probabilities[question.qid] = torch.softmax(logits, dim=-1).tolist() keys[question.qid] = list(question.keys) + # The pinned decoder is the answer-field authority, so every + # question is decoded by it as well as by the plain softmax. + answers[question.qid] = reference_answer(question, logits) return { "keys": keys, "probabilities": probabilities, + "answers": answers, "logical_tokens": state.logical_tokens, "compute_tokens": state.compute_tokens, } @@ -245,19 +424,24 @@ def main(argv=None): executor = ValenExecutor(checkpoint, Path(args.base), args.device, args.dtype) cases = [] - if args.case in ("image", "both"): + if args.case in ("image", "both", "all"): cases.append(("image", json.loads(Path(args.fixture).read_text(encoding="utf-8")))) - if args.case in ("text", "both"): + if args.case in ("text", "both", "all"): if args.text_body: cases.append(("text", json.loads(Path(args.text_body).read_text(encoding="utf-8")))) else: cases.append(("text", DEFAULT_TEXT_BODY)) + if args.case in ("noul", "all"): + cases.append(("noul", json.loads(Path(args.noul_body).read_text(encoding="utf-8")))) + if args.case in ("score", "all"): + cases.append(("score", json.loads(Path(args.score_body).read_text(encoding="utf-8")))) failed = 0 try: for name, body in cases: worker_response = decide(json.dumps(body).encode("utf-8"), processor, executor) - questions = reference_questions(body) + questions = reference_questions(body, processor.special_tokens) + expected = expected_answer_types(body, processor.special_tokens) with TemporaryDirectory(prefix="valen-parity-") as directory: media_root = Path(directory) if name == "image": @@ -268,7 +452,9 @@ def main(argv=None): reference = run_reference( executor.model, processor.processor, record, media_root, max_length, media_kwargs ) - lines, ok = compare_case(name, worker_response, reference, args.tolerance) + lines, ok = compare_case( + name, worker_response, reference, expected, args.tolerance + ) print("\n".join(lines)) print() if not ok: diff --git a/recipe/valen/example-noul-request.json b/recipe/valen/example-noul-request.json new file mode 100644 index 00000000..48d57fdb --- /dev/null +++ b/recipe/valen/example-noul-request.json @@ -0,0 +1,18 @@ +{ + "model": "valen-preview-0923", + "state": "The card was charged twice for one order.", + "questions": { + "duplicate": { + "type": "noul", + "instructions": "Was the card charged twice for one order?" + }, + "authorized": { + "type": "noul", + "instructions": "Did the customer authorize both charges?", + "criteria": { + "true": "Both charges were authorized by the customer.", + "false": "At least one charge was unauthorized." + } + } + } +} diff --git a/recipe/valen/example-score-request.json b/recipe/valen/example-score-request.json new file mode 100644 index 00000000..e077afab --- /dev/null +++ b/recipe/valen/example-score-request.json @@ -0,0 +1,15 @@ +{ + "model": "valen-preview-0923", + "state": "The card was charged twice for one order.", + "questions": { + "severity": { + "type": "score", + "instructions": "How severe is the billing error?", + "criteria": [ + "No billing error: every charge is correct.", + "Minor error: a single duplicated charge.", + "Major error: repeated duplicated charges." + ] + } + } +} diff --git a/recipe/valen/reference.md b/recipe/valen/reference.md index 863d07a0..c773c1b6 100644 --- a/recipe/valen/reference.md +++ b/recipe/valen/reference.md @@ -17,19 +17,21 @@ The validated slice is: - the Python reference worker behind the Rust frontend; - text state (a non-empty string; objects and arrays are serialized to JSON text) or one inline PNG or JPEG image in `state.image`; -- textual instructions and criteria for `choice` questions; +- textual instructions and criteria for `choice`, `noul`, and `score` + questions, freely mixed in one request against one shared state; - BF16 CUDA inference with the pinned Preview checkpoint. Image and text state are serving-validated on the GPU host, with recorded reference parity - for both. + for all three question types. -The current worker does not implement `noul`, `score`, video, multiple images, -remote image URLs, or a native Rust/CUDA Valen executor. +The current worker does not implement video, multiple images, remote image +URLs, or a native Rust/CUDA Valen executor. The current protocol limits requests to an 8 MiB JSON body, one PNG/JPEG image of at most 4 MiB, 2048 pixels per side, 1,048,576 pixels total, and a 200:1 maximum aspect ratio. Text state is limited to 16,384 characters, the same limit as combined instruction and criterion text. A request may contain at -most 8 questions and 255 candidates per question. +most 8 questions, 255 named candidates per `choice` question, and an ordered +list of 2 to 10 level descriptions per `score` question. The first slice follows [issue #84](https://github.com/ThinkFlowLab/system1-omni/issues/84): establish @@ -99,7 +101,7 @@ Pillow: 12.3.0 av: 16.1.0 GPU capability: sm_120 dtype: bfloat16 -system1-omni: 2fd8ce9 (GPU serving and smoke records); parity blocks re-verified verbatim at c9fb2d2 +system1-omni: 2fd8ce9 (GPU serving and smoke records); parity blocks re-verified verbatim at 507c448 ~~~ FP32 and BF16 small-operator checks passed. Startup memory was approximately @@ -250,6 +252,60 @@ Observed response, same host as the image fixture: } ~~~ +A yes/no judgment uses `noul`, optionally replacing the reference's fixed +true/false wording through `criteria`. The complete runnable request is in +[`example-noul-request.json`](example-noul-request.json); it asks one +fixed-wording and one custom-wording question against one state: + +~~~sh +curl -sS http://127.0.0.1:8000/v1/systemone \ + -H 'Content-Type: application/json' \ + --data-binary @recipe/valen/example-noul-request.json +~~~ + +Expected response shape (the observed response for this fixture is recorded +in the integration pull request): + +~~~json +{ + "model": "valen-preview-0923", + "answers": { + "duplicate": {"type": "noul", "noul": 0.0, "probabilities": {"true": 0.0, "false": 0.0}, "confidence": 0.0}, + "authorized": {"type": "noul", "noul": 0.0, "probabilities": {"true": 0.0, "false": 0.0}, "confidence": 0.0} + }, + "usage": {"input_tokens": 0, "output_tokens": 0}, + "internal_usage": {"compute_tokens": 0} +} +~~~ + +A graded assessment uses `score` with an ordered list of 2 to 10 level +descriptions. The complete runnable request is in +[`example-score-request.json`](example-score-request.json): + +~~~sh +curl -sS http://127.0.0.1:8000/v1/systemone \ + -H 'Content-Type: application/json' \ + --data-binary @recipe/valen/example-score-request.json +~~~ + +Expected response shape (the observed response for this fixture is recorded +in the integration pull request): + +~~~json +{ + "model": "valen-preview-0923", + "answers": {"severity": { + "type": "score", + "probabilities": {"0": 0.0, "1": 0.0, "2": 0.0}, + "score": 0.0, + "legend": {"0": "level 0 text", "1": "level 1 text", "2": "level 2 text"}, + "confidence": 0.0 + }}, + "usage": {"input_tokens": 0, "output_tokens": 0}, + "internal_usage": {"compute_tokens": 0} +} +~~~ + ## Start and verify the Rust frontend ~~~sh @@ -267,8 +323,8 @@ curl -sS http://127.0.0.1:8080/v1/systemone \ --data-binary @recipe/valen/example-request.json ~~~ -The generic `recipe/compare_with_backend.py` also sends `score`/`noul` cases, -which this worker rejects, so compare the Valen fixture directly: +The generic `recipe/compare_with_backend.py` also covers `score`/`noul` +cases; the byte-identical comparison below uses the Valen fixture directly: ~~~sh curl -fsS http://127.0.0.1:8000/v1/systemone \ @@ -282,7 +338,9 @@ cmp /tmp/valen-worker.json /tmp/valen-frontend.json The validated runs returned 200 from both endpoints with matching JSON content types and byte-identical response bodies, for the image fixture above and for -the text-only example. +the text-only example. The noul and score fixtures were re-checked the same +way on the same host: byte-identical through the frontend for all three +fixtures. ## Error behavior @@ -309,7 +367,7 @@ python recipe/valen/test_compare_reference.py Validated results: ~~~text -pytest tests/valen: 29 passed in 0.67s +pytest tests/valen: 47 passed in 1.21s compileall: passed cargo fmt --all --check: passed frontend integration: 12 passed; 0 failed @@ -335,6 +393,7 @@ max probability delta: 2.7810645453e-08 confidence delta: 5.5621290906e-08 decision agreement: PASS probability tolerance <= 1e-06: PASS +confidence tolerance <= 1e-06: PASS token accounting: PASS ~~~ @@ -351,18 +410,28 @@ max probability delta: 2.6866340819e-08 confidence delta: 5.3732681637e-08 decision agreement: PASS probability tolerance <= 1e-06: PASS +confidence tolerance <= 1e-06: PASS token accounting: PASS ~~~ The reference model field is `Valen`; the system1-omni worker deliberately uses `valen-preview-0923` as its stable serving identity. -Reproduce with [`compare_reference.py`](compare_reference.py). The script loads -one pinned model instance and drives each request through two paths: a -Valen-native record compiled by the pinned `Compiler` and decoded with a plain -softmax, and the production `frontend.valen.decide` pipeline. The decision -criteria are exact choice agreement, equal token accounting and the declared -probability tolerance (default `1e-6`, `--tolerance` to override): +Reproduce with [`compare_reference.py`](compare_reference.py). Both paths +share one pinned model instance: a Valen-native record compiled by the pinned +`Compiler` from the worker's own record rules, against the production +`frontend.valen.decide` pipeline; `noul` and `score` answers are decoded by +the pinned reference answer module (`valen.evaluation.inference.answer`). The +criteria are exact discrete agreement (candidate order, the requested answer +type, chosen key, score legend), equal token accounting, and the declared +tolerance (default `1e-6`, `--tolerance` to override) on probabilities, +confidences and the scalar answer fields (`noul` P(true), `score` expected +level). A non-finite probability, scalar answer or confidence on either side +fails the run before any tolerance verdict: `max()` keeps its previous finite +maximum against a NaN difference, so a NaN delta would otherwise leave every +tolerance check passing. An answer whose public `type` drifts from the +requested question type fails the run instead of skipping the noul/score +checks: ~~~sh PYTHONPATH=src .venv/bin/python recipe/valen/compare_reference.py \ @@ -372,11 +441,23 @@ PYTHONPATH=src .venv/bin/python recipe/valen/compare_reference.py \ PYTHONPATH=src .venv/bin/python recipe/valen/compare_reference.py \ --valen-source "$VALEN_SRC" --checkpoint "$VALEN_CKPT" \ --base "$VALEN_BASE" --device cuda --dtype bf16 --case text + +PYTHONPATH=src .venv/bin/python recipe/valen/compare_reference.py \ + --valen-source "$VALEN_SRC" --checkpoint "$VALEN_CKPT" \ + --base "$VALEN_BASE" --device cuda --dtype bf16 --case noul + +PYTHONPATH=src .venv/bin/python recipe/valen/compare_reference.py \ + --valen-source "$VALEN_SRC" --checkpoint "$VALEN_CKPT" \ + --base "$VALEN_BASE" --device cuda --dtype bf16 --case score ~~~ The image case uses [`example-request.json`](example-request.json); the text case uses the recipe's text example unless `--text-body` points elsewhere. -Both blocks above are verbatim output of the committed tool. +The `noul` and `score` cases use [`example-noul-request.json`](example-noul-request.json) +and [`example-score-request.json`](example-score-request.json) unless +`--noul-body`/`--score-body` point elsewhere; `--case all` runs all four in +one model load. Both blocks above are verbatim output of the committed tool; +the `noul` and `score` blocks are recorded in the integration pull request. ## Troubleshooting and limits @@ -392,4 +473,4 @@ Both blocks above are verbatim output of the committed tool. - `/health` is exposed only after model loading and real warmup complete. - The memory and smoke-response observations are host-specific; they are not general accuracy or performance claims. -- Video, native execution, `noul`, and `score` remain outside this recipe. +- Video and native execution remain outside this recipe. diff --git a/recipe/valen/test_compare_reference.py b/recipe/valen/test_compare_reference.py index d124f09d..dc449cf9 100644 --- a/recipe/valen/test_compare_reference.py +++ b/recipe/valen/test_compare_reference.py @@ -3,7 +3,8 @@ Checks the pure comparison helpers without the pinned Valen source, model weights or a GPU: the reference confidence formula, data-URL decoding, the -reference record builders, and the PASS/FAIL verdicts of compare_case. +reference record builders (including the noul/score record rules), and the +PASS/FAIL verdicts of compare_case. python recipe/valen/test_compare_reference.py """ @@ -12,32 +13,36 @@ import base64 import hashlib +import json import sys import tempfile from pathlib import Path HERE = Path(__file__).resolve().parent sys.path.insert(0, str(HERE)) +sys.path.insert(0, str(HERE.parents[1] / "src")) import compare_reference as tool -def worker_response(probabilities, choice, confidence, input_tokens=17, compute_tokens=17): +def worker_response(answers, input_tokens=17, compute_tokens=17): return { "model": "valen-preview-0923", - "answers": { - "move": { - "type": "choice", - "choice": choice, - "probabilities": probabilities, - "confidence": confidence, - } - }, + "answers": answers, "usage": {"input_tokens": input_tokens, "output_tokens": 0}, "internal_usage": {"compute_tokens": compute_tokens}, } +def choice_answer(probabilities, choice, confidence): + return { + "type": "choice", + "choice": choice, + "probabilities": probabilities, + "confidence": confidence, + } + + REFERENCE = { "keys": {"move": ["up", "down"]}, "probabilities": {"move": [0.25, 0.75]}, @@ -102,19 +107,78 @@ def check(name: str, condition: bool, detail: str = "") -> None: "text record uses the compiler's plain-string state", record["request"]["state"] == "plain context" and record["request"]["questions"] is questions - and "assets" not in record, + and "assets" not in record ) + + # 3b. Reference question records reuse the worker's own record rules. + def request_body(question): + return { + "model": "valen-preview-0923", + "state": "The card was charged twice for one order.", + "questions": {"q": question}, + } + check( "reference questions map null labels to their keys", - tool.reference_questions( - {"questions": {"q": {"instructions": None, - "criteria": {"a": None, "b": "B"}}}} - )["q"]["criteria"] == {"a": "a", "b": "B"}, + tool.reference_questions(request_body( + {"type": "choice", "instructions": "Choose.", + "criteria": {"a": None, "b": "B"}}))["q"]["criteria"] == {"a": "a", "b": "B"}, + ) + check( + "noul without wording stays the reference's native noul record", + tool.reference_questions(request_body( + {"type": "noul", "instructions": "Was it duplicated?"}))["q"] + == {"type": "noul", "instructions": "Was it duplicated?"}, + ) + custom = tool.reference_questions(request_body( + {"type": "noul", "instructions": "Was it duplicated?", + "criteria": {"true": "Duplicated.", "false": None}}))["q"] + check( + "custom noul wording becomes a true/false choice record", + custom["type"] == "choice" + and custom["criteria"]["true"] == "Duplicated." + and custom["criteria"]["false"] == "False / 否：不满足问题中的条件。", + ) + check( + "score levels pass through as an ordered list", + tool.reference_questions(request_body( + {"type": "score", "instructions": "Grade.", + "criteria": ["low", "high"]}))["q"] + == {"type": "score", "instructions": "Grade.", "criteria": ["low", "high"]}, + ) + + # 3c. The committed noul and score fixtures build valid records. + noul_fixture = json.loads( + (HERE / "example-noul-request.json").read_text(encoding="utf-8") + ) + noul_records = tool.reference_questions(noul_fixture) + check( + "noul fixture builds one native and one worded record", + noul_records["duplicate"] == { + "type": "noul", + "instructions": "Was the card charged twice for one order?", + } + and noul_records["authorized"]["type"] == "choice" + and set(noul_records["authorized"]["criteria"]) == {"true", "false"}, + ) + score_fixture = json.loads( + (HERE / "example-score-request.json").read_text(encoding="utf-8") + ) + score_records = tool.reference_questions(score_fixture) + check( + "score fixture builds one three-level score record", + score_records["severity"]["type"] == "score" + and len(score_records["severity"]["criteria"]) == 3, ) # 4. A matching worker response passes every verdict. lines, ok = tool.compare_case( - "text", worker_response({"up": 0.25, "down": 0.75}, "down", 0.5), REFERENCE + "text", + worker_response( + {"move": choice_answer({"up": 0.25, "down": 0.75}, "down", 0.5)} + ), + REFERENCE, + {"move": "choice"}, ) report = "\n".join(lines) check( @@ -122,6 +186,7 @@ def check(name: str, condition: bool, detail: str = "") -> None: ok and "decision agreement: PASS" in report and "probability tolerance <= 1e-06: PASS" in report + and "confidence tolerance <= 1e-06: PASS" in report and "token accounting: PASS" in report and "input_tokens: 17 == 17" in report, report, @@ -130,14 +195,20 @@ def check(name: str, condition: bool, detail: str = "") -> None: # 5. A tiny delta stays inside the declared tolerance. lines, ok = tool.compare_case( "text", - worker_response({"up": 0.25 + 1e-9, "down": 0.75 - 1e-9}, "down", 0.5), + worker_response( + {"move": choice_answer({"up": 0.25 + 1e-9, "down": 0.75 - 1e-9}, "down", 0.5)} + ), REFERENCE, + {"move": "choice"}, ) check("delta within tolerance passes", ok, "\n".join(lines)) # 6. Each failure mode fails on its own. lines, ok = tool.compare_case( - "text", worker_response({"up": 0.6, "down": 0.4}, "up", 0.2), REFERENCE + "text", + worker_response({"move": choice_answer({"up": 0.6, "down": 0.4}, "up", 0.2)}), + REFERENCE, + {"move": "choice"}, ) check( "probability and decision mismatch fails", @@ -148,8 +219,25 @@ def check(name: str, condition: bool, detail: str = "") -> None: ) lines, ok = tool.compare_case( - "text", worker_response({"up": 0.25, "down": 0.75}, "down", 0.5, input_tokens=18), + "text", + worker_response({"move": choice_answer({"up": 0.25, "down": 0.75}, "down", 0.9)}), + REFERENCE, + {"move": "choice"}, + ) + check( + "confidence drift beyond tolerance fails", + not ok and "confidence tolerance <= 1e-06: FAIL" in "\n".join(lines), + "\n".join(lines), + ) + + lines, ok = tool.compare_case( + "text", + worker_response( + {"move": choice_answer({"up": 0.25, "down": 0.75}, "down", 0.5)}, + input_tokens=18, + ), REFERENCE, + {"move": "choice"}, ) check( "token mismatch fails", @@ -159,7 +247,12 @@ def check(name: str, condition: bool, detail: str = "") -> None: ) lines, ok = tool.compare_case( - "text", worker_response({"down": 0.75, "up": 0.25}, "down", 0.5), REFERENCE + "text", + worker_response( + {"move": choice_answer({"down": 0.75, "up": 0.25}, "down", 0.5)} + ), + REFERENCE, + {"move": "choice"}, ) check( "candidate order mismatch fails", @@ -167,6 +260,304 @@ def check(name: str, condition: bool, detail: str = "") -> None: "\n".join(lines), ) + # 6b. Non-finite probabilities fail before any delta or argmax work. + # A NaN difference would keep the previous finite maximum and the argmax + # would silently keep a NaN-free first candidate, so both sides are + # rejected up front instead of compared. + lines, ok = tool.compare_case( + "text", + worker_response( + {"move": choice_answer({"up": float("nan"), "down": 0.4}, "up", 0.5)} + ), + REFERENCE, + {"move": "choice"}, + ) + check( + "worker NaN probability fails", + not ok + and "non-finite probability" in "\n".join(lines) + and "decision agreement: FAIL" in "\n".join(lines), + "\n".join(lines), + ) + + nan_reference = { + "keys": {"move": ["up", "down"]}, + "probabilities": {"move": [float("nan"), 0.75]}, + "logical_tokens": 17, + "compute_tokens": 17, + } + lines, ok = tool.compare_case( + "text", + worker_response({"move": choice_answer({"up": 0.25, "down": 0.75}, "down", 0.5)}), + nan_reference, + {"move": "choice"}, + ) + check( + "reference NaN probability fails", + not ok and "non-finite probability" in "\n".join(lines), + "\n".join(lines), + ) + + inf_reference = { + "keys": {"move": ["up", "down"]}, + "probabilities": {"move": [float("inf"), 0.0]}, + "logical_tokens": 17, + "compute_tokens": 17, + } + lines, ok = tool.compare_case( + "text", + worker_response( + {"move": choice_answer({"up": float("inf"), "down": 0.0}, "up", 0.5)} + ), + inf_reference, + {"move": "choice"}, + ) + check( + "matching infinities on both sides still fail", + not ok and "non-finite probability" in "\n".join(lines), + "\n".join(lines), + ) + + # 6c. Non-finite scalar answers and confidences fail the same way: the + # per-type branches fold them into running maxima, and max() keeps its + # first argument against a NaN difference, so a NaN delta would + # otherwise leave the tolerance verdicts passing. + lines, ok = tool.compare_case( + "text", + worker_response( + {"move": choice_answer({"up": 0.25, "down": 0.75}, "down", float("nan"))} + ), + REFERENCE, + {"move": "choice"}, + ) + check( + "worker NaN confidence fails", + not ok + and "non-finite scalar answer or confidence" in "\n".join(lines) + and "decision agreement: FAIL" in "\n".join(lines), + "\n".join(lines), + ) + + # 7. noul answers compare P(true) with tolerance. + noul_reference = { + "keys": {"ask": ["true", "false"]}, + "probabilities": {"ask": [0.75, 0.25]}, + # The pinned decoder answers native noul with P(true) only; custom + # wording decodes as a choice pair instead. + "answers": {"ask": {"type": "noul", "noul": 0.75}}, + "logical_tokens": 17, + "compute_tokens": 17, + } + + def noul_answer(p_true, confidence): + return { + "type": "noul", + "noul": p_true, + "probabilities": {"true": p_true, "false": 1.0 - p_true}, + "confidence": confidence, + } + + lines, ok = tool.compare_case( + "noul", + worker_response({"ask": noul_answer(0.75, 0.5)}), + noul_reference, + {"ask": "noul"}, + ) + report = "\n".join(lines) + check( + "noul exact match passes with the answer tolerance", + ok + and "answer tolerance <= 1e-06: PASS" in report + and "legend agreement" not in report, + report, + ) + + lines, ok = tool.compare_case( + "noul", + worker_response({"ask": noul_answer(0.75 + 1e-4, 0.5)}), + noul_reference, + {"ask": "noul"}, + ) + check( + "noul answer drift beyond tolerance fails", + not ok and "answer tolerance <= 1e-06: FAIL" in "\n".join(lines), + "\n".join(lines), + ) + + # A NaN P(true) with finite probabilities must fail instead of being + # swallowed by the answer-delta maximum. + nan_noul = { + "type": "noul", + "noul": float("nan"), + "probabilities": {"true": 0.75, "false": 0.25}, + "confidence": 0.5, + } + lines, ok = tool.compare_case( + "noul", worker_response({"ask": nan_noul}), noul_reference, {"ask": "noul"} + ) + check( + "worker NaN noul answer fails", + not ok and "non-finite scalar answer or confidence" in "\n".join(lines), + "\n".join(lines), + ) + + # A noul request answered in choice shape must fail even when the + # candidate probabilities, decision and tokens all match. + drifted = { + "type": "choice", + "choice": "true", + "probabilities": {"true": 0.75, "false": 0.25}, + "confidence": 0.5, + } + lines, ok = tool.compare_case( + "noul", worker_response({"ask": drifted}), noul_reference, {"ask": "noul"} + ) + check( + "noul answered as choice fails", + not ok + and "decision agreement: FAIL" in "\n".join(lines) + and "answer type 'choice' != requested 'noul'" in "\n".join(lines), + "\n".join(lines), + ) + + choice_shaped = dict(noul_reference) + choice_shaped["answers"] = { + "ask": {"type": "choice", "probabilities": {"true": 0.75, "false": 0.25}, + "choice": "true", "confidence": 0.5} + } + lines, ok = tool.compare_case( + "noul", + worker_response({"ask": noul_answer(0.75, 0.5)}), + choice_shaped, + {"ask": "noul"}, + ) + check( + "custom-wording noul compares against the choice-shaped decode", + ok and "confidence delta: 0.0000000000e+00" in "\n".join(lines), + "\n".join(lines), + ) + + # 8. score answers compare the expected level, legend and confidence. + score_reference = { + "keys": {"grade": ["0", "1", "2"]}, + "probabilities": {"grade": [0.7, 0.2, 0.1]}, + "answers": { + "grade": { + "type": "score", + "probabilities": {"0": 0.7, "1": 0.2, "2": 0.1}, + "score": 0.4, + "legend": {"0": "No error", "1": "A minor error", "2": "A major error"}, + "confidence": 0.4, + } + }, + "logical_tokens": 17, + "compute_tokens": 51, + } + + def score_answer(score, legend, confidence): + return { + "type": "score", + "probabilities": {"0": 0.7, "1": 0.2, "2": 0.1}, + "score": score, + "legend": legend, + "confidence": confidence, + } + + legend = {"0": "No error", "1": "A minor error", "2": "A major error"} + lines, ok = tool.compare_case( + "score", + worker_response({"grade": score_answer(0.4, legend, 0.4)}, compute_tokens=51), + score_reference, + {"grade": "score"}, + ) + report = "\n".join(lines) + check( + "score exact match passes with legend agreement", + ok + and "answer tolerance <= 1e-06: PASS" in report + and "legend agreement: PASS" in report, + report, + ) + + wrong_legend = dict(legend, **{"2": "A catastrophic error"}) + lines, ok = tool.compare_case( + "score", + worker_response( + {"grade": score_answer(0.4, wrong_legend, 0.4)}, compute_tokens=51 + ), + score_reference, + {"grade": "score"}, + ) + check( + "score legend mismatch fails", + not ok + and "legend agreement: FAIL" in "\n".join(lines) + and "score legend mismatch" in "\n".join(lines), + "\n".join(lines), + ) + + lines, ok = tool.compare_case( + "score", + worker_response({"grade": score_answer(0.5, legend, 0.4)}, compute_tokens=51), + score_reference, + {"grade": "score"}, + ) + check( + "score expected-level drift fails the answer tolerance", + not ok and "answer tolerance <= 1e-06: FAIL" in "\n".join(lines), + "\n".join(lines), + ) + + # A score request answered in choice shape must fail even when the + # candidate probabilities, decision and tokens all match. + drifted_score = { + "type": "choice", + "choice": "1", + "probabilities": {"0": 0.7, "1": 0.2, "2": 0.1}, + "confidence": 0.4, + } + lines, ok = tool.compare_case( + "score", + worker_response({"grade": drifted_score}, compute_tokens=51), + score_reference, + {"grade": "score"}, + ) + check( + "score answered as choice fails", + not ok + and "decision agreement: FAIL" in "\n".join(lines) + and "answer type 'choice' != requested 'score'" in "\n".join(lines), + "\n".join(lines), + ) + + # A NaN confidence carried by the reference decode must fail instead of + # being swallowed by the confidence-delta maximum. + nan_conf_reference = { + "keys": {"grade": ["0", "1", "2"]}, + "probabilities": {"grade": [0.7, 0.2, 0.1]}, + "answers": { + "grade": { + "type": "score", + "score": 0.4, + "legend": legend, + "confidence": float("nan"), + } + }, + "logical_tokens": 17, + "compute_tokens": 51, + } + lines, ok = tool.compare_case( + "score", + worker_response({"grade": score_answer(0.4, legend, 0.4)}, compute_tokens=51), + nan_conf_reference, + {"grade": "score"}, + ) + check( + "reference NaN confidence fails", + not ok and "non-finite scalar answer or confidence" in "\n".join(lines), + "\n".join(lines), + ) + failed = 0 for name, passed, detail in results: print(f" {'ok ' if passed else 'FAIL'} {name}") diff --git a/src/models/valen/README.md b/src/models/valen/README.md index 8d107ce5..d94585ed 100644 --- a/src/models/valen/README.md +++ b/src/models/valen/README.md @@ -2,8 +2,9 @@ This directory contains the first System1-Omni integration slice for Valen-Preview-0923. It is a reference-serving boundary only: the Python worker, -processor, executor, and postprocessor are implemented for the pinned text and -single-image `choice` slice. Native CUDA execution is not part of this slice. +processor, executor, and postprocessor serve the pinned text and single-image +`choice`, `noul`, and `score` questions. Native CUDA execution is not part of +this slice. ## Pinned artifacts @@ -29,9 +30,14 @@ does not contain the frozen Qwen base model. The published config uses criteria are limited to 16,384 characters; - `instructions` must be non-empty after trimming; criterion keys and texts must be non-empty after trimming (a null criterion text defaults to its key); -- `choice` questions only, at most 8 per request; -- up to 255 named candidates per question; -- no `noul`, `score`, video, multi-image, or remote URL inputs; +- `choice`, `noul` and `score` questions, at most 8 per request, freely mixed + against one shared state; +- `choice`: up to 255 named candidates per question; +- `noul`: fixed `true`/`false` candidates; `criteria` is optional and may + replace either side's wording (`{"true": ..., "false": ...}`, null keeps + the reference's fixed wording); +- `score`: an ordered list of 2 to 10 level descriptions in `criteria`; +- no video, multi-image, or remote URL inputs; - no token generation: the decision head returns candidate logits directly. The public request keeps the Cua-S1-compatible shape: @@ -69,12 +75,30 @@ A text-only request replaces the image object with plain text: Preprocessing validates and decodes the data URL, writes it to a request-scoped directory, hashes the bytes, and converts it to Valen's local-media `messages` record. Text state skips materialization and compiles a text-only -user message with no assets. The worker owns that directory's lifetime. Postprocessing -uses Valen's original choice confidence formula for this reference slice: +user message with no assets. The worker owns that directory's lifetime. + +The pinned compiler builds every question type from the same record family: + +- `choice` and `noul` compile to one branch holding all candidates; +- `score` compiles one isolated branch per level — each branch sees only its + own level description, never the level ID or the other levels — and the K + per-level decision logits are softmaxed across branches; +- `noul` without custom wording is the reference's native noul record with + the schema's fixed candidate wording; custom wording cannot flow through + it, so it compiles as the equivalent `choice` record keyed `true`/`false`, + with unspecified sides filled from the same fixed wording — `P(true)` still + comes from the two-candidate softmax. + +Postprocessing keeps Valen's original formulas for this reference slice: ~~~text -K = 1: confidence = 1 -K > 1: confidence = max(0, (max(p) - 1/K) / (1 - 1/K)) +choice/noul confidence (n candidates): + n = 1: 1 + n > 1: max(0, (max(p) - 1/n) / (1 - 1/n)) # noul is the n = 2 case +score confidence (K ordered levels, mode = argmax p): + distance = sum_i p_i * |i - mode| + uniform_mad = (sum_i |i - (K-1)/2|) / K + confidence = max(0, 1 - distance / uniform_mad) ~~~ This confidence choice is intentionally temporary. The shared `/v1/systemone` @@ -82,17 +106,29 @@ contract, including confidence semantics, error details, busy behavior, model identity, and usage fields, remains coordinated with [system1-omni#61](https://github.com/ThinkFlowLab/system1-omni/issues/61). -Each answer is `{"type": "choice", "choice": ..., "probabilities": ..., -"confidence": ...}`: `choice` is the candidate with the highest probability, -ties go to the earliest candidate in request order, and `probabilities` maps -every candidate key to its probability in request order. +Each `choice` answer is `{"type": "choice", "choice": ..., +"probabilities": ..., "confidence": ...}`: `choice` is the candidate with the +highest probability, ties go to the earliest candidate in request order, and +`probabilities` maps every candidate key to its probability in request order. + +Each `noul` answer is `{"type": "noul", "noul": ..., +"probabilities": ..., "confidence": ...}`: `noul` is `P(true)`, the reference +answer module's own field; the pair and confidence are this worker's +extension (the reference answer carries only `P(true)`). + +Each `score` answer is `{"type": "score", "probabilities": ..., +"score": ..., "legend": ..., "confidence": ...}`: `probabilities` maps the +level keys `"0"`… to their cross-branch softmax probability, `score` is the +expected level `sum(i * p_i)`, and `legend` maps each level key back to its +request-order description. The response `model` is `valen-preview-0923`, the worker's stable serving identity. `usage.input_tokens` counts the compiled prompt's logical tokens: the shared state tokens plus each question's task suffix, counted once. `usage.output_tokens` is 0; the decision head returns logits without generation. `internal_usage.compute_tokens` counts the full per-question -branch lengths the executor runs, with the shared state repeated per branch. +branch lengths the executor runs, with the shared state repeated per branch — +a K-level `score` question therefore counts K branches. ## Errors @@ -111,14 +147,17 @@ and `503` that another inference is running. The status is `422` when a well-formed request cannot be evaluated: - a `model` other than `valen-preview-0923`; -- a `score` or `noul` question, or unsupported request or question fields; +- a question whose `type` is not `choice`, `noul` or `score`, or unsupported + request or question fields; - text over its limit, or containing a tokenizer control token — the media tokens such as `<|image_pad|>` at minimum, plus the loaded tokenizer's full special-token set (for example `<|im_end|>`) when the worker is serving; - this guard covers text state, instructions, criterion texts and option keys - (candidate names enter the prompt); -- empty or whitespace-only `instructions`, option keys, criterion texts, or - text state; + this guard covers text state, instructions, criterion texts, level + descriptions and option keys (candidate names enter the prompt); +- empty or whitespace-only `instructions`, option keys, criterion texts, + noul descriptions, score level descriptions, or text state; +- `noul` criteria that name anything other than `true`/`false`, or `score` + criteria that are not an ordered list of 2 to 10 level descriptions; - an empty text state, or a state that is neither an image data URL nor text; - an image that is not a single-frame PNG/JPEG matching its MIME type and limits; diff --git a/src/models/valen/postprocess.py b/src/models/valen/postprocess.py index 08d0c6f7..c739d0d5 100644 --- a/src/models/valen/postprocess.py +++ b/src/models/valen/postprocess.py @@ -23,28 +23,67 @@ def _probabilities(logits: Sequence[float], temperature: float) -> list[float]: return [value / total for value in exponentials] +def _choice_confidence(probabilities: Sequence[float]) -> float: + """Valen's original choice confidence formula (reference answer module).""" + + n = len(probabilities) + if n == 1: + return 1.0 + return max(0.0, (max(probabilities) - 1 / n) / (1 - 1 / n)) + + +def _noul_answer( + question: Question, probabilities: Sequence[float] +) -> dict[str, Any]: + """The reference's noul answer (P(true)) plus the full candidate pair.""" + + pair = dict(zip(question.keys, probabilities)) + return { + "type": "noul", + "noul": pair["true"], + "probabilities": pair, + "confidence": _choice_confidence(probabilities), + } + + +def _score_answer( + question: Question, probabilities: Sequence[float] +) -> dict[str, Any]: + """Valen's pinned score decoding: expected level and concentration.""" + + mode = max(range(len(probabilities)), key=probabilities.__getitem__) + count = len(probabilities) + distance = sum(p * abs(i - mode) for i, p in enumerate(probabilities)) + uniform_mad = sum(abs(i - (count - 1) / 2) for i in range(count)) / count + return { + "type": "score", + "probabilities": dict(zip(question.keys, probabilities)), + "score": sum(i * p for i, p in enumerate(probabilities)), + "legend": dict(zip(question.keys, question.descriptions)), + "confidence": max(0.0, 1.0 - distance / uniform_mad), + } + + def answer( question: Question, logits: Sequence[float], temperature: float = 1.0, ) -> dict[str, Any]: - """Match Valen's pinned choice decoding and confidence formula.""" + """Match Valen's pinned decoding and confidence formulas per type.""" if len(logits) != len(question.keys): - raise ValueError("model returned one logit per candidate") + raise ValueError("model must return one logit per candidate") probabilities = _probabilities(logits, temperature) + if question.kind == "noul": + return _noul_answer(question, probabilities) + if question.kind == "score": + return _score_answer(question, probabilities) mode = max(range(len(probabilities)), key=probabilities.__getitem__) - n = len(probabilities) - confidence = ( - 1.0 - if n == 1 - else max(0.0, (max(probabilities) - 1 / n) / (1 - 1 / n)) - ) return { "type": "choice", "choice": question.keys[mode], "probabilities": dict(zip(question.keys, probabilities)), - "confidence": confidence, + "confidence": _choice_confidence(probabilities), } diff --git a/src/models/valen/preprocess.py b/src/models/valen/preprocess.py index e0fec410..38e60363 100644 --- a/src/models/valen/preprocess.py +++ b/src/models/valen/preprocess.py @@ -13,7 +13,14 @@ from pathlib import Path from typing import Any -from .protocol import DEFAULT_SPECIAL_TOKENS, Question, Request, TextState +from .protocol import ( + DEFAULT_SPECIAL_TOKENS, + NOUL_FALSE_DESCRIPTION, + NOUL_TRUE_DESCRIPTION, + Question, + Request, + TextState, +) @dataclass(frozen=True) @@ -83,6 +90,47 @@ def compile(self, prepared: PreparedInput) -> CompiledInput: ) +def question_record(question: Question) -> dict[str, Any]: + """Build the pinned compiler's question record for one wire question. + + ``noul`` without custom wording compiles as the reference's native noul + record, whose candidate descriptions the pinned schema fixes. Custom + true/false wording cannot flow through a noul record (the pinned + ``candidates`` ignores noul criteria), so it compiles as a choice record + keyed ``true``/``false`` with the unspecified sides filled from the + reference's fixed wording — the same single-branch, two-candidate + structure. ``score`` passes its ordered level descriptions straight + through; the pinned schema assigns the level keys ``"0"``… + """ + + if question.kind == "noul": + descriptions = [ + description if description is not None else fixed + for description, fixed in zip( + question.descriptions, + (NOUL_TRUE_DESCRIPTION, NOUL_FALSE_DESCRIPTION), + ) + ] + if tuple(descriptions) == (NOUL_TRUE_DESCRIPTION, NOUL_FALSE_DESCRIPTION): + return {"type": "noul", "instructions": question.instructions} + return { + "type": "choice", + "instructions": question.instructions, + "criteria": dict(zip(question.keys, descriptions)), + } + if question.kind == "score": + return { + "type": "score", + "instructions": question.instructions, + "criteria": list(question.descriptions), + } + return { + "type": "choice", + "instructions": question.instructions, + "criteria": dict(zip(question.keys, question.descriptions)), + } + + def prepare_request(request: Request, media_root: Path) -> PreparedInput: """Materialize one validated request into the pinned Valen record format.""" @@ -122,12 +170,7 @@ def prepare_request(request: Request, media_root: Path) -> PreparedInput: } assets = [{"path": image_path.name, "sha256": image_sha256}] criteria = { - question.name: { - "type": "choice", - "instructions": question.instructions, - "criteria": dict(zip(question.keys, question.descriptions)), - } - for question in request.questions + question.name: question_record(question) for question in request.questions } record = { "request": {"state": state, "questions": criteria}, diff --git a/src/models/valen/protocol.py b/src/models/valen/protocol.py index 512b6190..7122104b 100644 --- a/src/models/valen/protocol.py +++ b/src/models/valen/protocol.py @@ -25,6 +25,17 @@ MAX_QUESTIONS = 8 MAX_OPTIONS = 255 MAX_TEXT = 16384 +MIN_SCORE_LEVELS = 2 +MAX_SCORE_LEVELS = 10 + +QUESTION_TYPES = ("choice", "noul", "score") + +# Verbatim from the pinned valen.data.schema.candidates noul wording at +# revision 750bfcfbb48a5275534a9c912257ebe83ca57a97; the compiler emits these +# descriptions for every noul question, so the worker reuses them whenever the +# wire leaves a side unspecified. +NOUL_TRUE_DESCRIPTION = "True / 是：满足问题中的条件。" +NOUL_FALSE_DESCRIPTION = "False / 否：不满足问题中的条件。" DEFAULT_SPECIAL_TOKENS = ( "<|image_pad|>", @@ -68,10 +79,19 @@ class TextState: @dataclass(frozen=True) class Question: + """One validated wire question. + + ``keys`` are the candidate names the pinned compiler assigns: wire option + keys for ``choice``, ``true``/``false`` for ``noul``, and the level + indices ``"0"``.. for ``score``. A ``None`` description marks a ``noul`` + side left to the reference's fixed wording. + """ + name: str instructions: str keys: tuple[str, ...] - descriptions: tuple[str, ...] + descriptions: tuple[str | None, ...] + kind: str = "choice" @dataclass(frozen=True) @@ -225,8 +245,11 @@ def parse_request(value: dict[str, Any], special_tokens=DEFAULT_SPECIAL_TOKENS) "criteria", }: raise RequestError(f"question {name!r} contains unsupported fields") - if question.get("type") != "choice": - raise RequestError(f"question {name!r}: only choice is supported") + kind = question.get("type") + if kind not in QUESTION_TYPES: + raise RequestError( + f"question {name!r}: type must be one of {', '.join(QUESTION_TYPES)}" + ) if "instructions" not in question: raise RequestError(f"question {name!r}: instructions is required") instructions = _text( @@ -234,34 +257,83 @@ def parse_request(value: dict[str, Any], special_tokens=DEFAULT_SPECIAL_TOKENS) ) if not instructions.strip(): raise RequestError(f"question {name!r}: instructions must be nonempty") - criteria = question.get("criteria") - if not isinstance(criteria, dict) or not 1 <= len(criteria) <= MAX_OPTIONS: - raise RequestError( - f"question {name!r}: choice requires 1 to {MAX_OPTIONS} options" - ) keys: list[str] = [] - descriptions: list[str] = [] - for key, label in criteria.items(): - if not isinstance(key, str) or not key.strip() or len(key) > 256: - raise RequestError("option keys must be nonempty and at most 256 characters") - if any(token in key for token in special_tokens): - # Candidate names enter the prompt, so keys get the same - # control-token guard as instructions and descriptions. + descriptions: list[str | None] = [] + if kind == "noul": + keys = ["true", "false"] + criteria = question.get("criteria") + if criteria is not None: + if not isinstance(criteria, dict) or not criteria: + raise RequestError( + f"question {name!r}: noul criteria must describe true and/or false" + ) + if set(criteria) - {"true", "false"}: + raise RequestError( + f"question {name!r}: noul criteria must describe true and/or false" + ) + for key in ("true", "false"): + label = (criteria or {}).get(key) + # None keeps the reference's fixed candidate wording; the + # preprocess layer substitutes it when building the record. + description = None + if label is not None: + description = _text( + label, f"question {name!r} criteria {key!r}", special_tokens + ) + if not description.strip(): + raise RequestError( + f"question {name!r}: criterion {key!r} must be nonempty" + ) + descriptions.append(description) + elif kind == "score": + criteria = question.get("criteria") + if ( + not isinstance(criteria, list) + or not MIN_SCORE_LEVELS <= len(criteria) <= MAX_SCORE_LEVELS + ): raise RequestError( - f"question {name!r}: option key {key!r} contains an" - " unsupported tokenizer control token" + f"question {name!r}: score requires an ordered list of" + f" {MIN_SCORE_LEVELS} to {MAX_SCORE_LEVELS} level descriptions" ) - keys.append(key) - description = ( - key - if label is None - else _text(label, f"question {name!r} criteria {key!r}", special_tokens) - ) - if not description.strip(): - raise RequestError(f"question {name!r}: criterion {key!r} must be nonempty") - descriptions.append(description) - if len(instructions) + sum(map(len, descriptions)) > MAX_TEXT: + keys = [str(index) for index in range(len(criteria))] + for index, label in enumerate(criteria): + description = _text( + label, f"question {name!r} level {index}", special_tokens + ) + if not description.strip(): + raise RequestError( + f"question {name!r}: level {index} must be nonempty" + ) + descriptions.append(description) + else: + criteria = question.get("criteria") + if not isinstance(criteria, dict) or not 1 <= len(criteria) <= MAX_OPTIONS: + raise RequestError( + f"question {name!r}: choice requires 1 to {MAX_OPTIONS} options" + ) + for key, label in criteria.items(): + if not isinstance(key, str) or not key.strip() or len(key) > 256: + raise RequestError("option keys must be nonempty and at most 256 characters") + if any(token in key for token in special_tokens): + # Candidate names enter the prompt, so keys get the same + # control-token guard as instructions and descriptions. + raise RequestError( + f"question {name!r}: option key {key!r} contains an" + " unsupported tokenizer control token" + ) + keys.append(key) + description = ( + key + if label is None + else _text(label, f"question {name!r} criteria {key!r}", special_tokens) + ) + if not description.strip(): + raise RequestError(f"question {name!r}: criterion {key!r} must be nonempty") + descriptions.append(description) + if len(instructions) + sum(len(d) for d in descriptions if d is not None) > MAX_TEXT: raise RequestError("combined question text exceeds 16384 characters") - parsed.append(Question(name, instructions, tuple(keys), tuple(descriptions))) + parsed.append( + Question(name, instructions, tuple(keys), tuple(descriptions), kind) + ) return Request(MODEL_NAME, _state(value["state"], special_tokens), tuple(parsed)) diff --git a/tests/valen/test_postprocess.py b/tests/valen/test_postprocess.py index 837d420e..40dcc54d 100644 --- a/tests/valen/test_postprocess.py +++ b/tests/valen/test_postprocess.py @@ -48,3 +48,83 @@ def test_build_response_preserves_question_order_and_usage(): "usage": {"input_tokens": 17, "output_tokens": 0}, "internal_usage": {"compute_tokens": 34}, } + + +NOUL = Question("ask", "Was it duplicated?", ("true", "false"), (None, None), "noul") + + +def test_noul_answer_reports_true_probability_and_full_pair(): + result = answer(NOUL, [0.0, math.log(3.0)]) + + assert result["type"] == "noul" + assert result["noul"] == pytest.approx(0.25) + assert result["probabilities"] == { + "true": pytest.approx(0.25), + "false": pytest.approx(0.75), + } + # Valen's choice confidence formula evaluated on the two candidates. + assert result["confidence"] == pytest.approx(0.5) + + flipped = answer(NOUL, [math.log(3.0), 0.0]) + assert flipped["noul"] == pytest.approx(0.75) + + +def test_noul_answer_honors_temperature_and_rejects_bad_logits(): + sharp = answer(NOUL, [1.0, 0.0], temperature=0.5) + plain = answer(NOUL, [1.0, 0.0], temperature=1.0) + assert sharp["noul"] > plain["noul"] > 0.5 + + with pytest.raises(ValueError, match="one logit"): + answer(NOUL, [0.0]) + with pytest.raises(ValueError, match="invalid logits"): + answer(NOUL, [float("nan"), 0.0]) + + +SCORE = Question( + "grade", + "Grade the severity.", + ("0", "1", "2"), + ("No error", "A minor error", "A major error"), + "score", +) + + +def test_score_answer_reports_expected_level_and_distribution(): + result = answer(SCORE, [0.0, 0.0, 0.0]) + + assert result["type"] == "score" + assert result["probabilities"] == {"0": pytest.approx(1 / 3), "1": pytest.approx(1 / 3), "2": pytest.approx(1 / 3)} + assert result["legend"] == {"0": "No error", "1": "A minor error", "2": "A major error"} + assert result["score"] == pytest.approx(1.0) + # A uniform distribution has no concentration. + assert result["confidence"] == pytest.approx(0.0) + + peaked = answer(SCORE, [-100.0, 3.0, -100.0]) + assert peaked["score"] == pytest.approx(1.0) + assert peaked["confidence"] == pytest.approx(1.0) + + +def test_score_answer_confidence_penalizes_split_mass(): + # Mass split between the two endpoints: mode is 0 (ties go to the + # earliest level), distance = 0.6 * 0 + 0.4 * 2 = 0.8 against a uniform + # mean absolute deviation of 2/3, so confidence is 0 (clamped). + result = answer(SCORE, [math.log(1.5), -50.0, 0.0]) + assert result["probabilities"]["0"] == pytest.approx(0.6) + assert result["probabilities"]["2"] == pytest.approx(0.4, abs=1e-6) + assert result["score"] == pytest.approx(0.8) + assert result["confidence"] == 0.0 + + # A tilted but concentrated distribution: p = [0.7, 0.2, 0.1] has mode 0, + # distance 0.2*1 + 0.1*2 = 0.4 and uniform MAD 2/3, so confidence is + # 1 - 0.4/(2/3) = 0.4. + tilted = answer(SCORE, [math.log(7.0), math.log(2.0), 0.0]) + assert tilted["probabilities"]["0"] == pytest.approx(0.7) + assert tilted["score"] == pytest.approx(0.4) + assert tilted["confidence"] == pytest.approx(0.4) + + +def test_score_answer_rejects_wrong_logit_counts(): + with pytest.raises(ValueError, match="one logit"): + answer(SCORE, [0.0, 1.0]) + with pytest.raises(RequestError, match="temperature"): + answer(SCORE, [0.0, 1.0, 2.0], temperature=-1.0) diff --git a/tests/valen/test_preprocess.py b/tests/valen/test_preprocess.py index 5f8280dc..accbb65b 100644 --- a/tests/valen/test_preprocess.py +++ b/tests/valen/test_preprocess.py @@ -1,7 +1,12 @@ import hashlib -from models.valen.preprocess import prepare_request -from models.valen.protocol import MODEL_NAME, parse_request +from models.valen.preprocess import prepare_request, question_record +from models.valen.protocol import ( + MODEL_NAME, + NOUL_FALSE_DESCRIPTION, + NOUL_TRUE_DESCRIPTION, + parse_request, +) from .test_protocol import body @@ -52,3 +57,113 @@ def test_prepare_compiles_text_state_without_media(tmp_path): } assert list(prepared.record["request"]["questions"]) == ["move"] assert list(tmp_path.iterdir()) == [] + + +def noul_question(criteria=None): + question = {"type": "noul", "instructions": "Was the card charged twice?"} + if criteria is not None: + question["criteria"] = criteria + request = parse_request( + body(state="The card was charged twice.", questions={"ask": question}) + ) + return request.questions[0] + + +def test_question_record_keeps_reference_noul_wording_native(): + record = question_record(noul_question()) + + assert record == {"type": "noul", "instructions": "Was the card charged twice?"} + + explicit = question_record( + noul_question( + criteria={ + "true": NOUL_TRUE_DESCRIPTION, + "false": NOUL_FALSE_DESCRIPTION, + } + ) + ) + assert explicit == record + + +def test_question_record_carries_custom_noul_wording_as_choice(): + record = question_record( + noul_question(criteria={"true": "It was duplicated.", "false": "It was unique."}) + ) + + assert record == { + "type": "choice", + "instructions": "Was the card charged twice?", + "criteria": {"true": "It was duplicated.", "false": "It was unique."}, + } + + partial = question_record(noul_question(criteria={"false": "It was unique."})) + assert partial["criteria"] == { + "true": NOUL_TRUE_DESCRIPTION, + "false": "It was unique.", + } + + +def test_question_record_passes_score_levels_in_order(): + request = parse_request( + body( + state="Dialog: the update finished.", + questions={ + "grade": { + "type": "score", + "instructions": "Grade the dialog.", + "criteria": ["unclear", "clear"], + } + }, + ) + ) + + assert question_record(request.questions[0]) == { + "type": "score", + "instructions": "Grade the dialog.", + "criteria": ["unclear", "clear"], + } + + +def test_prepare_builds_mixed_type_records(tmp_path): + request = parse_request( + body( + state="The card was charged twice for one order.", + questions={ + "ask": { + "type": "noul", + "instructions": "Was the card charged twice?", + }, + "grade": { + "type": "score", + "instructions": "Grade the severity.", + "criteria": ["No error", "A minor error", "A major error"], + }, + "act": { + "type": "choice", + "instructions": "Decide the refund action.", + "criteria": {"refund": None, "wait": "Wait for review"}, + }, + }, + ) + ) + prepared = prepare_request(request, tmp_path) + + assert prepared.record["request"]["questions"] == { + "ask": {"type": "noul", "instructions": "Was the card charged twice?"}, + "grade": { + "type": "score", + "instructions": "Grade the severity.", + "criteria": ["No error", "A minor error", "A major error"], + }, + "act": { + "type": "choice", + "instructions": "Decide the refund action.", + "criteria": {"refund": "refund", "wait": "Wait for review"}, + }, + } + assert prepared.record["assets"] == [] + assert [q.kind for q in prepared.response_context.questions] == [ + "noul", + "score", + "choice", + ] diff --git a/tests/valen/test_protocol.py b/tests/valen/test_protocol.py index 3f073fd2..940c5a09 100644 --- a/tests/valen/test_protocol.py +++ b/tests/valen/test_protocol.py @@ -76,14 +76,19 @@ def test_parse_rejects_wrong_model_and_invalid_state(): parse_request(body(state=17)) -def test_parse_allows_structured_text_but_not_score_or_extra_fields(): +def test_parse_allows_structured_text_but_rejects_unknown_types_and_fields(): value = body() value["questions"]["move"]["instructions"] = {"goal": "choose"} assert parse_request(value).questions[0].instructions == '{"goal": "choose"}' value = body() - value["questions"]["move"]["type"] = "score" - with pytest.raises(RequestError, match="only choice"): + value["questions"]["move"]["type"] = "rank" + with pytest.raises(RequestError, match="type must be one of choice, noul, score"): + parse_request(value) + + value = body() + del value["questions"]["move"]["type"] + with pytest.raises(RequestError, match="type must be one of"): parse_request(value) value = body() @@ -158,3 +163,129 @@ def test_parse_rejects_media_control_tokens(): with pytest.raises(RequestError, match="control token"): parse_request(body(state="context <|video_pad|> tail")) + + +def noul_body(**overrides): + value = body( + questions={ + "ask": {"type": "noul", "instructions": "Was the card charged twice?"} + } + ) + for question in value["questions"].values(): + question.update(overrides) + return value + + +def test_parse_noul_defaults_to_fixed_wording_and_true_false_keys(): + question = parse_request(noul_body()).questions[0] + + assert question.kind == "noul" + assert question.keys == ("true", "false") + assert question.descriptions == (None, None) + + +def test_parse_noul_accepts_optional_true_false_descriptions(): + question = parse_request( + noul_body(criteria={"true": "It was duplicated.", "false": "It was unique."}) + ).questions[0] + assert question.descriptions == ("It was duplicated.", "It was unique.") + + question = parse_request(noul_body(criteria={"true": "It was duplicated."})).questions[0] + assert question.descriptions == ("It was duplicated.", None) + + question = parse_request( + noul_body(criteria={"true": "It was duplicated.", "false": None}) + ).questions[0] + assert question.descriptions == ("It was duplicated.", None) + + +def test_parse_noul_rejects_other_criteria_and_empty_descriptions(): + with pytest.raises(RequestError, match="noul criteria must describe"): + parse_request(noul_body(criteria={"yes": "Yes"})) + with pytest.raises(RequestError, match="noul criteria must describe"): + parse_request(noul_body(criteria={})) + with pytest.raises(RequestError, match="criterion 'true' must be nonempty"): + parse_request(noul_body(criteria={"true": " "})) + + value = noul_body(criteria={"true": "answer <|im_end|> now"}) + with pytest.raises(RequestError, match="control token"): + parse_request(value, special_tokens=("<|im_end|>",)) + + +def score_body(**overrides): + value = body( + questions={ + "grade": { + "type": "score", + "instructions": "Grade the severity.", + "criteria": ["No error", "A minor error", "A major error"], + } + } + ) + for question in value["questions"].values(): + question.update(overrides) + return value + + +def test_parse_score_accepts_two_to_ten_ordered_levels(): + question = parse_request(score_body()).questions[0] + + assert question.kind == "score" + assert question.keys == ("0", "1", "2") + assert question.descriptions == ("No error", "A minor error", "A major error") + + two = parse_request(score_body(criteria=["low", "high"])).questions[0] + assert two.keys == ("0", "1") + + ten = parse_request( + score_body(criteria=[f"level {index}" for index in range(10)]) + ).questions[0] + assert ten.keys == tuple(str(index) for index in range(10)) + + +def test_parse_score_rejects_malformed_levels(): + with pytest.raises(RequestError, match="ordered list of 2 to 10"): + parse_request(score_body(criteria=["only one"])) + with pytest.raises(RequestError, match="ordered list of 2 to 10"): + parse_request(score_body(criteria=[str(index) for index in range(11)])) + with pytest.raises(RequestError, match="ordered list of 2 to 10"): + parse_request(score_body(criteria={"0": "No error", "1": "A minor error"})) + missing = score_body() + del missing["questions"]["grade"]["criteria"] + with pytest.raises(RequestError, match="ordered list of 2 to 10"): + parse_request(missing) + + with pytest.raises(RequestError, match="level 1 must be nonempty"): + parse_request(score_body(criteria=["No error", " "])) + with pytest.raises(RequestError, match="level 0"): + parse_request(score_body(criteria=[None, "A minor error"])) + + value = score_body(criteria=["answer <|im_end|> now", "A minor error"]) + with pytest.raises(RequestError, match="control token"): + parse_request(value, special_tokens=("<|im_end|>",)) + + +def test_parse_accepts_mixed_type_questions_in_request_order(): + value = body( + state="Dialog: the update finished.", + questions={ + "ask": {"type": "noul", "instructions": "Did the update finish?"}, + "grade": { + "type": "score", + "instructions": "Grade the dialog.", + "criteria": ["unclear", "clear"], + }, + "act": { + "type": "choice", + "instructions": "Choose a move.", + "criteria": {"up": "Move up", "down": "Move down"}, + }, + }, + ) + request = parse_request(value) + + assert [(question.kind, question.name) for question in request.questions] == [ + ("noul", "ask"), + ("score", "grade"), + ("choice", "act"), + ] diff --git a/tests/valen/test_worker.py b/tests/valen/test_worker.py index fec9cf6e..064e8400 100644 --- a/tests/valen/test_worker.py +++ b/tests/valen/test_worker.py @@ -1,4 +1,5 @@ import json +import math import pytest @@ -36,6 +37,9 @@ def execute(self, compiled): assert compiled.response_context.questions return self.output + def close(self): + pass + def test_decide_keeps_http_adapter_outside_executor(): processor = FakeProcessor() @@ -96,3 +100,53 @@ def compile(self, prepared): assert excinfo.value.status == 422 assert str(excinfo.value) == "wire problem" + +def test_decide_answers_mixed_type_questions_from_one_state(): + value = body( + state="The card was charged twice for one order.", + questions={ + "ask": {"type": "noul", "instructions": "Was the card charged twice?"}, + "grade": { + "type": "score", + "instructions": "Grade the severity.", + "criteria": ["No error", "A minor error", "A major error"], + }, + "act": { + "type": "choice", + "instructions": "Decide the refund action.", + "criteria": {"refund": None, "wait": "Wait for review"}, + }, + }, + ) + processor = FakeProcessor() + executor = FakeExecutor( + ExecutorOutput( + ( + (math.log(3.0), 0.0), # noul: true, false + (math.log(7.0), math.log(2.0), 0.0), # score levels 0, 1, 2 + (0.0, 1.0), # choice: refund, wait + ), + 61, + 183, + ) + ) + + result = decide(json.dumps(value).encode("utf-8"), processor, executor) + + noul = result["answers"]["ask"] + assert noul["type"] == "noul" + assert noul["noul"] == pytest.approx(0.75) + assert noul["probabilities"]["true"] == pytest.approx(0.75) + assert noul["probabilities"]["false"] == pytest.approx(0.25) + + score = result["answers"]["grade"] + assert score["type"] == "score" + assert score["probabilities"]["0"] == pytest.approx(0.7) + assert score["score"] == pytest.approx(0.4) + assert score["legend"] == {"0": "No error", "1": "A minor error", "2": "A major error"} + assert score["confidence"] == pytest.approx(0.4) + + assert result["answers"]["act"]["choice"] == "wait" + assert result["usage"] == {"input_tokens": 61, "output_tokens": 0} + assert result["internal_usage"] == {"compute_tokens": 183} + diff --git a/tests/valen/test_worker_server.py b/tests/valen/test_worker_server.py index 66615a11..c71e9313 100644 --- a/tests/valen/test_worker_server.py +++ b/tests/valen/test_worker_server.py @@ -11,6 +11,7 @@ from __future__ import annotations import json +import math import threading from http.client import HTTPConnection @@ -164,6 +165,43 @@ def test_client_errors_map_to_their_status_and_detail(server): assert status == 422 and "nonempty" in json.loads(raw)["detail"] +def test_noul_and_score_questions_are_served_over_http(): + executor = FakeExecutor( + ExecutorOutput(((0.0, 1.0), (math.log(7.0), math.log(2.0), 0.0)), 5, 11) + ) + mixed = start(executor) + try: + value = body( + state="The card was charged twice for one order.", + questions={ + "ask": {"type": "noul", "instructions": "Was the card charged twice?"}, + "grade": { + "type": "score", + "instructions": "Grade the severity.", + "criteria": ["No error", "A minor error", "A major error"], + }, + }, + ) + status, _, raw = post(mixed, json.dumps(value).encode("utf-8")) + assert status == 200 + result = json.loads(raw) + assert result["answers"]["ask"]["type"] == "noul" + assert result["answers"]["grade"]["type"] == "score" + assert result["internal_usage"] == {"compute_tokens": 11} + finally: + stop(mixed) + + +def test_invalid_score_levels_map_to_422(server): + value = body( + questions={ + "grade": {"type": "score", "instructions": "Grade.", "criteria": ["only"]} + } + ) + status, _, raw = post(server, json.dumps(value).encode("utf-8")) + assert status == 422 and "2 to 10" in json.loads(raw)["detail"] + + def test_busy_worker_returns_503(server): server.inference_lock.acquire() try: