From 19b262e544de2c9e2d0b23f865204a6ae2372038 Mon Sep 17 00:00:00 2001 From: levius <2114377220@qq.com> Date: Wed, 7 Oct 2026 21:05:39 +0800 Subject: [PATCH 1/4] feat(decider): add bounded request-local CUDA batching --- docs/architecture.md | 3 + recipe/decider/README.md | 12 +++ recipe/decider/validation.md | 22 +++++ src/models/decider/README.md | 10 ++- src/models/decider/native/Cargo.toml | 4 + src/models/decider/native/src/batching.rs | 82 +++++++++++++++++ src/models/decider/native/src/engine.rs | 18 +++- src/models/decider/native/src/executor.rs | 98 ++++++++++++++------ src/models/decider/native/src/lib.rs | 1 + tests/decider/batching.rs | 37 ++++++++ tests/decider/executor.rs | 35 +++++++- tests/decider/fail_head.c | 4 +- tests/decider/gpu.rs | 44 +++++++++ tests/decider/verify_http.py | 8 +- tests/decider/verify_modes.py | 104 ++++++++++++++++++++++ tests/decider/verify_retirement.py | 5 ++ 16 files changed, 450 insertions(+), 37 deletions(-) create mode 100644 src/models/decider/native/src/batching.rs create mode 100644 tests/decider/batching.rs create mode 100644 tests/decider/verify_modes.py diff --git a/docs/architecture.md b/docs/architecture.md index 2e5b0ab4..e56fc79d 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -50,6 +50,9 @@ and candidate identity, usage, and response metadata outside the executor. Cua-S1 input collections are serial work. Open-Jev's model-specific batch adapter packs up to 16 independent candidates and 4096 tokens per group; longer prompts execute alone. It restores question/candidate grouping before normalization. +Decider optionally packs bounded contiguous independent complete rows within one +request, preserving Score-level reconstruction and unique-prefix usage. The +selected-label projection is batched; request admission remains serial. Laya batches questions within one request. Shared runtime admission precedes blocking dispatch: Cua-S1 admits one question forward at a time; Open-Jev and Laya admit one complete request. Cua-S1's CPU letter projection diff --git a/recipe/decider/README.md b/recipe/decider/README.md index ec5512e3..554ac082 100644 --- a/recipe/decider/README.md +++ b/recipe/decider/README.md @@ -123,3 +123,15 @@ CUA_S1_GRAPH=0 ./target/release/decider-run \ See [validation.md](validation.md) for full-checkpoint reference comparison, HTTP/frontend checks, prerequisites and the recorded scope. + +## Optional request-local packing + +Set `DECIDER_BATCH_MAX_ROWS=4 DECIDER_BATCH_MAX_TOKENS=4096` on `omni-decider` +to pack independent rows within one admitted request. Rows default to 1; accepted +limits are 1–4 rows and 1–4096 packed tokens. Longer complete rows execute alone. +Malformed or out-of-range settings fail startup. These limits do not change HTTP +admission, token usage, or the maximum complete-row length. + +The selected-label head uses persistent buffers sized to the row limit and projects +each batch in one BF16 GEMM. No cross-request batching is introduced. Keep the +single-row default until numerical and performance results fit your workload. diff --git a/recipe/decider/validation.md b/recipe/decider/validation.md index 891a325b..e0042dba 100644 --- a/recipe/decider/validation.md +++ b/recipe/decider/validation.md @@ -144,3 +144,25 @@ Generated run output and binaries are kept out of the source tree; only maintain fixtures and reproduction helpers are committed. The implementation PR links the immutable evidence archive and its SHA-256. All requests are synthetic and carry no private/customer content. A video is inapplicable to this text API integration. + +## Request-local batching comparison + +The followup runner `tests/decider/verify_modes.py` accepts a frozen JSON plan with +`model`, `library`, `parity_output` (the pinned reference run above), +`repetitions: 2`, `timed_cases` and `modes`. Each mode supplies `name`, an absolute +`binary` path and an `env` mapping. It hashes binaries, library, reference records +and comparison scripts before execution; retains complete records; checks prepared +rows exactly and the same response gates. Time is the diagnostic JSON-lines +roundtrip at concurrency 1, including processing, synchronization, serialization +and IPC. Startup to the empty diagnostic response and one workload warmup are +reported/excluded separately. These measurements do not establish HTTP latency, +peak memory or production throughput. Device memory is a whole-device sample at +the end of each mode, not a peak measurement. + +On the RTX 4090 followup campaign, row limits 2 and 4 passed all 18 reference +requests. Row limit 8 failed the unchanged probability gate on structured Unicode +input (0.0208 > 0.02), so supported packing is capped at **4 rows**. The failed +campaign and narrowed-protocol rerun are retained in the PR evidence archive. +Defaults remain one row; do not extrapolate these small workloads to arbitrary +requests or hardware. Batch fault injection also verifies a two-row head failure +retires the complete worker after real startup warmup. diff --git a/src/models/decider/README.md b/src/models/decider/README.md index 4e215dea..b0f02b05 100644 --- a/src/models/decider/README.md +++ b/src/models/decider/README.md @@ -46,8 +46,14 @@ are loaded from the pinned tied input embedding; all 255 rows are retained, with `executor.rs` owns the shared Qwen model and a separate CUDA projection stream, GEMM handle and persistent head buffers. Each complete request gets one shared -`SerialScheduler` permit. Rows run eagerly in order, resetting the backbone state -for each row. The BF16 selected projection uses existing backend GEMM, then converts +`SerialScheduler` permit. Default execution runs rows eagerly in order. Optional request-local packing +uses `DECIDER_BATCH_MAX_ROWS` (1–4, default 1) and +`DECIDER_BATCH_MAX_TOKENS` (1–4096, default 4096). Contiguous complete rows are +packed with the shared backbone `forward_batch`; each sequence resets its own +GDN/attention state. A row above the packing budget runs alone under the unchanged +complete-row limit. A persistent head projects all batch hidden rows in one GEMM. +Score levels can cross batch boundaries; response normalization still uses the +complete question. This does not combine separate requests or change admission. The BF16 selected projection uses existing backend GEMM, then converts its BF16 outputs to FP32 before CPU temperature scaling and normalization. Padding never participates in softmax or token accounting. Current GEMM/reduction algorithms can differ from Transformers; numerical agreement is measured rather than assumed. diff --git a/src/models/decider/native/Cargo.toml b/src/models/decider/native/Cargo.toml index 80f7125f..70706eb6 100644 --- a/src/models/decider/native/Cargo.toml +++ b/src/models/decider/native/Cargo.toml @@ -33,3 +33,7 @@ path = "../../../../tests/decider/contract.rs" [[test]] name = "config" path = "../../../../tests/decider/config.rs" + +[[test]] +name = "batching" +path = "../../../../tests/decider/batching.rs" diff --git a/src/models/decider/native/src/batching.rs b/src/models/decider/native/src/batching.rs new file mode 100644 index 00000000..8204f639 --- /dev/null +++ b/src/models/decider/native/src/batching.rs @@ -0,0 +1,82 @@ +//! Bounded packing of independent complete rows within a single admitted request. +use anyhow::{Context, Result, ensure}; +use std::ops::Range; + +#[derive(Clone, Copy, Debug)] +pub struct BatchLimits { + max_rows: usize, + max_tokens: usize, +} +impl Default for BatchLimits { + fn default() -> Self { + Self { + max_rows: 1, + max_tokens: 4096, + } + } +} +impl BatchLimits { + pub fn new(max_rows: usize, max_tokens: usize) -> Result { + ensure!( + (1..=4).contains(&max_rows), + "batch rows must be within 1..=4" + ); + ensure!( + (1..=4096).contains(&max_tokens), + "batch tokens must be within 1..=4096" + ); + Ok(Self { + max_rows, + max_tokens, + }) + } + pub fn from_values(rows: Option<&str>, tokens: Option<&str>) -> Result { + let rows = rows + .map_or(Ok(1), str::parse) + .context("DECIDER_BATCH_MAX_ROWS must be an integer")?; + let tokens = tokens + .map_or(Ok(4096), str::parse) + .context("DECIDER_BATCH_MAX_TOKENS must be an integer")?; + Self::new(rows, tokens) + } + pub fn from_env() -> Result { + let get = |name| match std::env::var(name) { + Ok(value) => Ok(Some(value)), + Err(std::env::VarError::NotPresent) => Ok(None), + Err(e) => Err(e), + }; + let rows = get("DECIDER_BATCH_MAX_ROWS")?; + let tokens = get("DECIDER_BATCH_MAX_TOKENS")?; + Self::from_values(rows.as_deref(), tokens.as_deref()) + } + pub fn max_rows(&self) -> usize { + self.max_rows + } + pub fn max_tokens(&self) -> usize { + self.max_tokens + } + /// Retain original row order; a row above the packing token budget runs alone. + /// Total request/row admission remains the processor/executor's separate limit. + pub fn ranges(&self, lengths: &[usize]) -> Vec> { + let mut ranges = Vec::new(); + let mut start = 0; + let mut tokens = 0usize; + for (index, &length) in lengths.iter().enumerate() { + if index > start + && (index - start == self.max_rows + || tokens + .checked_add(length) + .is_none_or(|total| total > self.max_tokens)) + { + ranges.push(start..index); + start = index; + tokens = 0; + } + tokens = tokens.saturating_add(length); + } + if start < lengths.len() { + ranges.push(start..lengths.len()); + } + ranges + } +} diff --git a/src/models/decider/native/src/engine.rs b/src/models/decider/native/src/engine.rs index f843c6a4..09ae64ad 100644 --- a/src/models/decider/native/src/engine.rs +++ b/src/models/decider/native/src/engine.rs @@ -1,5 +1,8 @@ //! Assemble verified artifacts, model-owned processing, and serial eager execution. -use crate::{Config, Kind, Limits, Processor, checkpoint::Checkpoint, executor::Executor}; +use crate::{ + Config, Kind, Limits, Processor, batching::BatchLimits, checkpoint::Checkpoint, + executor::Executor, +}; use anyhow::Result; use omni_runtime::SerialScheduler; use serde_json::{Value, json}; @@ -13,8 +16,15 @@ pub struct Engine { } impl Engine { pub async fn load(dir: &Path, library: &Path) -> Result { + Self::load_with_batch(dir, library, BatchLimits::from_env()?).await + } + pub async fn load_with_batch( + dir: &Path, + library: &Path, + batching: BatchLimits, + ) -> Result { let directory = dir.to_owned(); - let (processor,checkpoint,metadata) = tokio::task::spawn_blocking(move || -> Result<_> { + let (processor,checkpoint,mut metadata) = tokio::task::spawn_blocking(move || -> Result<_> { let processor = Processor::load(&directory,Limits::default())?; let labels: Vec = processor.labels().iter().map(|label| label.id).collect(); let checkpoint = Checkpoint::load(&directory,&labels)?; @@ -23,7 +33,9 @@ impl Engine { Ok((processor,checkpoint,metadata)) }).await??; let labels = processor.labels().iter().map(|label| label.id).collect(); - let executor = Executor::load(dir, library, checkpoint, labels).await?; + let executor = Executor::load(dir, library, checkpoint, labels, batching).await?; + metadata["batch_limits"] = + json!({"rows":batching.max_rows(),"tokens":batching.max_tokens()}); let engine = Self { processor: Arc::new(processor), executor, diff --git a/src/models/decider/native/src/executor.rs b/src/models/decider/native/src/executor.rs index 3868fe0e..bd832fce 100644 --- a/src/models/decider/native/src/executor.rs +++ b/src/models/decider/native/src/executor.rs @@ -1,6 +1,7 @@ //! Eager complete-row prefills and a selected tied-embedding CUDA readout. use crate::{ Limits, RowInput, + batching::BatchLimits, checkpoint::{Checkpoint, HIDDEN, LABELS, PADDED_LABELS, VOCAB}, }; use anyhow::{Context, Result, ensure}; @@ -45,6 +46,7 @@ struct Head { weights: DeviceBuffer, input: DeviceBuffer, output: DeviceBuffer, + capacity: usize, } impl Drop for Head { fn drop(&mut self) { @@ -53,7 +55,8 @@ impl Drop for Head { } } impl Head { - fn new(weights: &[u8]) -> Result { + fn new(weights: &[u8], capacity: usize) -> Result { + ensure!((1..=4).contains(&capacity), "invalid head batch capacity"); ensure!( weights.len() == PADDED_LABELS * HIDDEN * 2, "invalid selected head buffer" @@ -66,8 +69,9 @@ impl Head { stream, gemm, weights: DeviceBuffer::new(weights.len())?, - input: DeviceBuffer::new(HIDDEN * 2)?, - output: DeviceBuffer::new(PADDED_LABELS * 2)?, + input: DeviceBuffer::new(capacity * HIDDEN * 2)?, + output: DeviceBuffer::new(capacity * PADDED_LABELS * 2)?, + capacity, }; // SAFETY: the device allocation has exactly weights.len() bytes. unsafe { @@ -76,18 +80,27 @@ impl Head { Ok(head) } fn project(&self, hidden: &[f32], count: usize) -> Result> { + Ok(self.project_batch(&[hidden], &[count])?.pop().unwrap()) + } + fn project_batch(&self, hidden: &[&[f32]], counts: &[usize]) -> Result>> { + let rows = hidden.len(); ensure!( - hidden.len() == HIDDEN - && hidden.iter().all(|x| x.is_finite()) - && (2..=LABELS).contains(&count), - "invalid hidden state/readout" + rows > 0 + && rows <= self.capacity + && rows == counts.len() + && hidden + .iter() + .all(|h| h.len() == HIDDEN && h.iter().all(|x| x.is_finite())) + && counts.iter().all(|n| (2..=LABELS).contains(n)), + "invalid hidden batch/readout" ); let input: Vec = hidden .iter() + .flat_map(|h| h.iter()) .flat_map(|&x| bf16::from_f32(x).to_le_bytes()) .collect(); - // SAFETY: [1,HIDDEN] * [256,HIDDEN]^T -> [1,256]; allocated buffers match. - // The last weight row is zero padding and is never returned or normalized. + // SAFETY: [rows,HIDDEN] * [256,HIDDEN]^T -> [rows,256]; rows <= capacity. + // The final zero weight row is padding and never participates in normalization. unsafe { cuda::upload(self.input.at(0), &input, self.stream.0)?; cuda::check( @@ -96,7 +109,7 @@ impl Head { self.input.at(0), self.weights.at(0), self.output.at(0), - 1, + rows as i32, PADDED_LABELS as i32, HIDDEN as i32, PADDED_LABELS as i32, @@ -105,36 +118,60 @@ impl Head { "Decider label projection", )?; } - let mut output = vec![0; PADDED_LABELS * 2]; - // SAFETY: download waits for GEMM and copies exactly the output allocation. + let mut output = vec![0; rows * PADDED_LABELS * 2]; + // SAFETY: every returned row is inside the allocation; download synchronizes GEMM. unsafe { cuda::download(&mut output, self.output.at(0), self.stream.0)?; } - let logits: Vec = output.as_chunks::<2>().0[..count] + output + .as_chunks::<{ PADDED_LABELS * 2 }>() + .0 .iter() - .map(|&b| bf16::from_le_bytes(b).to_f32()) - .collect(); - ensure!( - logits.iter().all(|x| x.is_finite()), - "nonfinite candidate logits" - ); - Ok(logits) + .zip(counts) + .map(|(row, &count)| { + let logits: Vec = row.as_chunks::<2>().0[..count] + .iter() + .map(|&b| bf16::from_le_bytes(b).to_f32()) + .collect(); + ensure!( + logits.iter().all(|x| x.is_finite()), + "nonfinite candidate logits" + ); + Ok(logits) + }) + .collect() } } struct Loaded { model: Model, head: Head, + batching: BatchLimits, } impl Loaded { fn execute(&mut self, rows: &[RowInput]) -> Result>> { cuda::set_device(0)?; let result = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { - rows.iter() - .map(|row| { - let hidden = self.model.forward(&row.ids)?; - self.head.project(&hidden, row.candidate_ids.len()) - }) - .collect::>>() + let lengths: Vec = rows.iter().map(|row| row.ids.len()).collect(); + let mut logits = Vec::with_capacity(rows.len()); + for range in self.batching.ranges(&lengths) { + let batch = &rows[range]; + if batch.len() == 1 { + let hidden = self.model.forward(&batch[0].ids)?; + logits.push(self.head.project(&hidden, batch[0].candidate_ids.len())?); + } else { + let inputs: Vec<&[u32]> = batch.iter().map(|row| row.ids.as_slice()).collect(); + let hidden = self.model.forward_batch(&inputs)?; + ensure!( + hidden.len() == batch.len(), + "backbone batch output count mismatch" + ); + let hidden: Vec<&[f32]> = hidden.iter().map(Vec::as_slice).collect(); + let counts: Vec = + batch.iter().map(|row| row.candidate_ids.len()).collect(); + logits.extend(self.head.project_batch(&hidden, &counts)?); + } + } + Ok(logits) })) .map_err(|_| anyhow::anyhow!("Decider execution panicked")) .and_then(|result| result); @@ -158,6 +195,7 @@ impl Executor { library: &Path, checkpoint: Checkpoint, labels: Vec, + batching: BatchLimits, ) -> Result { ensure!( std::env::var("CUA_S1_GRAPH").as_deref() != Ok("1"), @@ -166,8 +204,12 @@ impl Executor { let (dir, library) = (dir.to_owned(), library.to_owned()); let loaded = tokio::task::spawn_blocking(move || -> Result { let model = Model::load(&dir, &library)?; - let head = Head::new(&checkpoint.head)?; - Ok(Loaded { model, head }) + let head = Head::new(&checkpoint.head, batching.max_rows())?; + Ok(Loaded { + model, + head, + batching, + }) }) .await??; Ok(Self { diff --git a/src/models/decider/native/src/lib.rs b/src/models/decider/native/src/lib.rs index d51f1455..52000fe8 100644 --- a/src/models/decider/native/src/lib.rs +++ b/src/models/decider/native/src/lib.rs @@ -31,6 +31,7 @@ pub const MODEL_ID: &str = "decider-2b-v11"; pub const RUNTIME_REVISION: &str = "50d0be0d7cb43d2066965ce5fa7f3fe4e489a60f"; pub const CHECKPOINT_REVISION: &str = "533964dae8be954c5b5e19fa4948e48408094c1e"; +pub mod batching; mod checkpoint; pub mod engine; pub mod executor; diff --git a/tests/decider/batching.rs b/tests/decider/batching.rs new file mode 100644 index 00000000..becc198b --- /dev/null +++ b/tests/decider/batching.rs @@ -0,0 +1,37 @@ +use omni_decider_native::batching::BatchLimits; +#[test] +fn bounded_contiguous_ranges_preserve_order_and_long_rows() { + let limits = BatchLimits::new(3, 8).unwrap(); + assert_eq!( + limits.ranges(&[2, 3, 3, 1, 9, 2, 2, 2, 2]), + vec![0..3, 3..4, 4..5, 5..8, 8..9] + ); + assert!(limits.ranges(&[]).is_empty()); + assert_eq!(limits.ranges(&[9, 9]), vec![0..1, 1..2]); +} +#[test] +fn row_limit_and_exact_token_boundary_are_independent() { + assert_eq!( + BatchLimits::new(2, 4096).unwrap().ranges(&[1, 1, 1]), + vec![0..2, 2..3] + ); + assert_eq!( + BatchLimits::new(4, 4096).unwrap().ranges(&[2048, 2048, 1]), + vec![0..2, 2..3] + ); + assert_eq!( + BatchLimits::default().ranges(&[1, 2, 3]), + vec![0..1, 1..2, 2..3] + ); +} +#[test] +fn startup_limits_reject_unbounded_or_malformed_settings() { + for (rows, tokens) in [(0, 1), (5, 1), (8, 1), (16, 1), (1, 0), (1, 4097)] { + assert!(BatchLimits::new(rows, tokens).is_err()); + } + assert!(BatchLimits::from_values(Some("wrong"), None).is_err()); + assert!(BatchLimits::from_values(None, Some("-1")).is_err()); + let limits = BatchLimits::from_values(Some("4"), Some("4096")).unwrap(); + assert_eq!(limits.max_rows(), 4); + assert_eq!(limits.max_tokens(), 4096); +} diff --git a/tests/decider/executor.rs b/tests/decider/executor.rs index d5116dce..c5a73532 100644 --- a/tests/decider/executor.rs +++ b/tests/decider/executor.rs @@ -44,7 +44,7 @@ fn bf16_head_rounds_before_fp32_calibration() { ] { weights[offset..offset + 2].copy_from_slice(&bf16::from_f32(value).to_le_bytes()); } - let head = Head::new(&weights).unwrap(); + let head = Head::new(&weights, 1).unwrap(); let mut hidden = vec![0.0; HIDDEN]; hidden[..2].copy_from_slice(&[1.0, 1.0]); assert_eq!(head.project(&hidden, 2).unwrap(), vec![1.0, 1.015625]); @@ -52,3 +52,36 @@ fn bf16_head_rounds_before_fp32_calibration() { hidden[0] = f32::NAN; assert!(head.project(&hidden, 2).is_err()); } + +#[test] +#[ignore = "requires the built Qwen CUDA library and an available NVIDIA GPU"] +fn batched_head_preserves_rows_counts_and_bf16_rounding() { + cuda::load(Path::new(&std::env::var("DECIDER_CUDA_LIB").unwrap())).unwrap(); + cuda::set_device(0).unwrap(); + let mut weights = vec![0; PADDED_LABELS * HIDDEN * 2]; + for (offset, value) in [ + (0, 1.0), + (2, 0.00390625), + (HIDDEN * 2, 1.0), + (HIDDEN * 2 + 2, 0.015625), + ] { + weights[offset..offset + 2].copy_from_slice(&bf16::from_f32(value).to_le_bytes()); + } + let head = Head::new(&weights, 2).unwrap(); + let mut a = vec![0.0; HIDDEN]; + a[..2].copy_from_slice(&[1.0, 1.0]); + let mut b = vec![0.0; HIDDEN]; + b[..2].copy_from_slice(&[2.0, 1.0]); + let first = head.project_batch(&[&a, &b], &[2, 255]).unwrap(); + assert_eq!(first[0], vec![1.0, 1.015625]); + assert_eq!(&first[1][..2], &[2.0, 2.015625]); + assert_eq!(first[1].len(), 255); + assert!(first[1][2..].iter().all(|&x| x == 0.0)); + let reversed = head.project_batch(&[&b, &a], &[255, 2]).unwrap(); + assert_eq!(reversed[0], first[1]); + assert_eq!(reversed[1], first[0]); + assert!(head.project_batch(&[&a, &b, &a], &[2, 2, 2]).is_err()); + assert!(head.project_batch(&[&a, &b], &[2]).is_err()); + b[0] = f32::NAN; + assert!(head.project_batch(&[&a, &b], &[2, 2]).is_err()); +} diff --git a/tests/decider/fail_head.c b/tests/decider/fail_head.c index 73c9fb87..07418ff3 100644 --- a/tests/decider/fail_head.c +++ b/tests/decider/fail_head.c @@ -10,8 +10,8 @@ int cs1_gemm(void* handle, const void* x, const void* w, void* y, static unsigned projection_calls; gemm_fn original = (gemm_fn)dlsym(RTLD_NEXT, "cs1_gemm"); if (!original) return 999; - if (m == 1 && n == 256 && k == 2048) { - fprintf(stderr, "Decider test projection %u\n", ++projection_calls); + if (m >= 1 && m <= 16 && n == 256 && k == 2048) { + fprintf(stderr, "Decider test projection %u rows %d\n", ++projection_calls, m); if (projection_calls == 2) return 2; } return original(handle, x, w, y, m, n, k, ldy, stream); diff --git a/tests/decider/gpu.rs b/tests/decider/gpu.rs index 65532b8e..8bec730e 100644 --- a/tests/decider/gpu.rs +++ b/tests/decider/gpu.rs @@ -91,3 +91,47 @@ async fn maximum_complete_row_grows_scratch_without_leaking_state() { assert_eq!(engine.predict(short).await.unwrap(), baseline); assert!(engine.executor.is_ready()); } + +#[tokio::test] +#[ignore = "requires pinned Decider-2B checkpoint and NVIDIA CUDA; request packing"] +async fn packed_mixed_rows_preserve_order_usage_and_recovery() { + use omni_decider_native::batching::BatchLimits; + use serde_json::json; + let dir = std::env::var("DECIDER_MODEL").unwrap(); + let library = std::env::var("DECIDER_CUDA_LIB").unwrap(); + let engine = Engine::load_with_batch( + Path::new(&dir), + Path::new(&library), + BatchLimits::new(4, 4096).unwrap(), + ) + .await + .unwrap(); + let raw=json!({"state":"A customer requests a refund for a damaged item.","questions":{"first":{"instructions":"Choose the team.","criteria":["refund","shipping"]},"score":{"type":"score","instructions":"Severity.","criteria":["low","medium","high"]},"last":{"type":"noul","instructions":"The item is damaged."}}}).to_string(); + let first = engine.predict(raw.as_bytes()).await.unwrap(); + assert_eq!( + first["answers"] + .as_object() + .unwrap() + .keys() + .map(String::as_str) + .collect::>(), + vec!["first", "score", "last"] + ); + assert_eq!(engine.health()["batch_limits"]["rows"], 4); + let long=json!({"state":"The item is damaged. ".repeat(1000),"questions":{"q":{"instructions":"Choose the team.","criteria":["refund","shipping"]}}}).to_string(); + assert!( + engine.processor.prepare(long.as_bytes()).unwrap().rows[0] + .ids + .len() + > 4096 + ); + assert!(engine.predict(long.as_bytes()).await.is_ok()); + assert_eq!(engine.predict(raw.as_bytes()).await.unwrap(), first); + assert!( + engine + .predict(br#"{"state":0,"questions":{"q":{"criteria":["only"]}}}"#) + .await + .is_err() + ); + assert_eq!(engine.predict(raw.as_bytes()).await.unwrap(), first); +} diff --git a/tests/decider/verify_http.py b/tests/decider/verify_http.py index 3fcef853..595f6c64 100644 --- a/tests/decider/verify_http.py +++ b/tests/decider/verify_http.py @@ -44,6 +44,8 @@ def main(): parser.add_argument("--library", type=Path, required=True) parser.add_argument("--parity-output", type=Path, required=True) parser.add_argument("--output", type=Path, required=True) + parser.add_argument("--execution", default="eager") + parser.add_argument("--native-records", type=Path, help="verify_modes records.jsonl for the selected mode") args = parser.parse_args() args.output.mkdir(parents=True, exist_ok=True) worker_base, frontend_base = "http://127.0.0.1:18110", "http://127.0.0.1:18111" @@ -57,7 +59,7 @@ def main(): processes.append(worker) metadata = wait_ready(worker_base, worker, path) assert metadata["checkpoint_revision"] == "533964dae8be954c5b5e19fa4948e48408094c1e" - assert metadata["execution"] == "eager" and metadata["dtype"] == "bfloat16" + assert metadata["execution"] == args.execution and metadata["dtype"] == "bfloat16" environment.update(OMNI_JEV_BIND="127.0.0.1:18111", OMNI_JEV_BACKEND_URL=worker_base) path = args.output / "frontend.log" logs.append(path.open("w")) @@ -66,6 +68,10 @@ def main(): wait_ready(frontend_base, frontend, path) protocol = json.loads((args.parity_output / "protocol.json").read_text()) native_records = [json.loads(line) for line in (args.parity_output / "native.jsonl").read_text().splitlines()] + if args.native_records: + native_records = [item["record"] for line in args.native_records.read_text().splitlines() + for item in [json.loads(line)] if item["phase"] == "parity"] + assert len(native_records) == len(protocol["cases"]) for case, native in zip(protocol["cases"], native_records): body = json.dumps(case["request"], ensure_ascii=False).encode() direct, proxy = request(worker_base, "/v1/systemone", body), request(frontend_base, "/v1/systemone", body) diff --git a/tests/decider/verify_modes.py b/tests/decider/verify_modes.py new file mode 100644 index 00000000..bb5d439b --- /dev/null +++ b/tests/decider/verify_modes.py @@ -0,0 +1,104 @@ +#!/usr/bin/env python3 +"""Sequential matched diagnostic-runner comparisons; raw output and frozen plan retained.""" +import argparse +import hashlib +import json +import os +from pathlib import Path +import statistics +import subprocess +import time + +from verify_reference import GATES, compare + + +def digest(path): + return hashlib.sha256(Path(path).read_bytes()).hexdigest() + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--plan", type=Path, required=True) + parser.add_argument("--output", type=Path, required=True) + args = parser.parse_args() + plan = json.loads(args.plan.read_text()) + assert plan["repetitions"] == 2 + parity = Path(plan["parity_output"]) + protocol = json.loads((parity / "protocol.json").read_text()) + reference = [json.loads(line) for line in (parity / "reference.jsonl").read_text().splitlines()] + baseline = [json.loads(line) for line in (parity / "native.jsonl").read_text().splitlines()] + cases = protocol["cases"] + assert len(cases) == len(reference) == len(baseline) + selected = [i for i, case in enumerate(cases) if case["name"] in plan["timed_cases"]] + assert len(selected) == len(plan["timed_cases"]) + args.output.mkdir(parents=True, exist_ok=False) + frozen = {**plan, "gates": GATES, "baseline_protocol": protocol, + "input_hashes": {name: digest(parity / name) for name in ("protocol.json", "reference.jsonl", "native.jsonl")}, + "runner_sha256": digest(__file__), "comparison_sha256": digest(Path(__file__).with_name("verify_reference.py")), "library_sha256": digest(plan["library"]), + "binary_hashes": {mode["name"]: digest(mode["binary"]) for mode in plan["modes"]}, + "timing": "wall roundtrip: request JSON encoding/write through diagnostic stdout JSON decode; includes CPU preparation, GPU synchronization and serialization; excludes startup and one workload warmup; concurrency=1"} + (args.output / "protocol.json").write_text(json.dumps(frozen, indent=2) + "\n") + summaries = [] + for mode in plan["modes"]: + directory = args.output / mode["name"] + directory.mkdir() + env = dict(os.environ) + for key in ("CUA_S1_GRAPH", "DECIDER_GRAPH", "DECIDER_PREFIX", "DECIDER_FIXED", "DECIDER_BATCH_MAX_ROWS", "DECIDER_BATCH_MAX_TOKENS"): + env.pop(key, None) + env.update(mode["env"]) + with (directory / "stderr.log").open("w") as stderr, (directory / "records.jsonl").open("w") as records: + start = time.perf_counter() + child = subprocess.Popen([mode["binary"], plan["model"], plan["library"]], env=env, + stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=stderr, text=True) + def run(request): + begun = time.perf_counter() + child.stdin.write(json.dumps(request, ensure_ascii=False) + "\n") + child.stdin.flush() + line = child.stdout.readline() + if not line: + raise RuntimeError(f"{mode['name']} terminated: {child.poll()}; see stderr.log") + value = json.loads(line) + return value, (time.perf_counter() - begun) * 1000 + try: + run({"state": "", "questions": {}}) + startup_ms = (time.perf_counter() - start) * 1000 + checks, samples = [], [] + for i, case in enumerate(cases): + got, ms = run(case["request"]) + records.write(json.dumps({"phase": "parity", "name": case["name"], "elapsed_ms": ms, "record": got}, ensure_ascii=False) + "\n") + records.flush() + assert got["rows"] == baseline[i]["rows"], (mode["name"], case["name"], "prepared rows") + checks.append({"name": case["name"], **compare(reference[i]["response"], got["response"])}) + for i in selected: + got, ms = run(cases[i]["request"]) + records.write(json.dumps({"phase": "warmup", "name": cases[i]["name"], "elapsed_ms": ms, "record": got}, ensure_ascii=False) + "\n") + for repetition in range(plan["repetitions"]): + for i in selected: + got, ms = run(cases[i]["request"]) + sample = {"name": cases[i]["name"], "repetition": repetition, "elapsed_ms": ms} + samples.append(sample) + records.write(json.dumps({"phase": "measured", **sample, "record": got}, ensure_ascii=False) + "\n") + records.flush() + assert got["rows"] == baseline[i]["rows"], (mode["name"], cases[i]["name"], "measured rows") + compare(reference[i]["response"], got["response"]) + memory = subprocess.check_output(["nvidia-smi", "--query-gpu=memory.used", "--format=csv,noheader,nounits"], text=True).strip() + summary = {"name": mode["name"], "startup_to_empty_diagnostic_ms": startup_ms, + "checks": checks, "samples": samples, "device_used_mib_at_end": int(memory), + "warm_ms": {name: {"median": statistics.median(values), "min": min(values), "max": max(values)} + for name in plan["timed_cases"] + for values in [[s["elapsed_ms"] for s in samples if s["name"] == name]]}} + summaries.append(summary) + (directory / "summary.json").write_text(json.dumps(summary, indent=2) + "\n") + print(json.dumps(summary), flush=True) + finally: + child.stdin.close() + try: + child.wait(timeout=30) + except subprocess.TimeoutExpired: + child.kill() + child.wait() + (args.output / "summary.json").write_text(json.dumps({"modes": summaries}, indent=2) + "\n") + + +if __name__ == "__main__": + main() diff --git a/tests/decider/verify_retirement.py b/tests/decider/verify_retirement.py index 84357b55..8bb5e125 100644 --- a/tests/decider/verify_retirement.py +++ b/tests/decider/verify_retirement.py @@ -24,6 +24,7 @@ def main(): parser.add_argument("--model", type=Path, required=True) parser.add_argument("--wrapper", type=Path, required=True) parser.add_argument("--output", type=Path, required=True) + parser.add_argument("--batched", action="store_true") args = parser.parse_args() args.output.mkdir(parents=True, exist_ok=True) log = args.output / "worker.log" @@ -37,6 +38,8 @@ def main(): assert health["status"] == "ready" before = memory(child.pid) raw = b'{"state":"The item is damaged.","questions":{"q":{"instructions":"Choose a category.","criteria":["refund","shipping"]}}}' + if args.batched: + raw = b'{"state":"The item is damaged.","questions":{"q":{"type":"score","instructions":"Severity.","criteria":["low","high"]}}}' status, body = request(base, "/v1/systemone", raw) assert status == 503 and json.loads(body)["detail"] == "model inference failed" status, body = request(base, "/health") @@ -50,6 +53,8 @@ def main(): text = log.read_text() assert text.count("Decider test projection") == 2 assert "Decider test projection 2" in text + if args.batched: + assert "Decider test projection 2 rows 2" in text result = {"passed": True, "projection_calls": 2, "health_after": "unavailable", "response_status": 503, "sampled_before_mib": before, "sampled_after_mib": after, "later_requests_refused": 2} (args.output / "summary.json").write_text(json.dumps(result, indent=2) + "\n") From 098d3b501631712082c313314d13d159e5c961ed Mon Sep 17 00:00:00 2001 From: levius <2114377220@qq.com> Date: Thu, 8 Oct 2026 14:13:41 +0800 Subject: [PATCH 2/4] chore: retain only core implementation in PR diff --- README.md | 3 +- docs/architecture.md | 26 +-- docs/supported-models.md | 6 +- mkdocs.yml | 2 - recipe/README.md | 2 - recipe/decider/README.md | 137 -------------- recipe/decider/example-request.json | 1 - recipe/decider/validation.md | 168 ----------------- src/backends/cuda/qwen3_5/README.md | 8 +- src/models/decider/README.md | 77 -------- src/models/decider/native/Cargo.toml | 16 -- .../decider/native/src/bin/decider-run.rs | 35 ---- src/models/decider/native/src/checkpoint.rs | 4 - src/models/decider/native/src/executor.rs | 4 - src/runtime/README.md | 3 +- tests/decider/batching.rs | 37 ---- tests/decider/checkpoint.rs | 69 ------- tests/decider/config.rs | 75 -------- tests/decider/contract.rs | 93 --------- tests/decider/data/README.md | 23 --- tests/decider/data/cpu.json | 1 - tests/decider/data/responses.json | 1 - tests/decider/executor.rs | 87 --------- tests/decider/fail_head.c | 18 -- tests/decider/gpu.rs | 137 -------------- tests/decider/test_download.py | 28 --- tests/decider/test_reference.py | 29 --- tests/decider/verify_http.py | 124 ------------ tests/decider/verify_modes.py | 104 ----------- tests/decider/verify_reference.py | 176 ------------------ tests/decider/verify_retirement.py | 68 ------- 31 files changed, 12 insertions(+), 1550 deletions(-) delete mode 100644 recipe/decider/README.md delete mode 100644 recipe/decider/example-request.json delete mode 100644 recipe/decider/validation.md delete mode 100644 src/models/decider/README.md delete mode 100644 src/models/decider/native/src/bin/decider-run.rs delete mode 100644 tests/decider/batching.rs delete mode 100644 tests/decider/checkpoint.rs delete mode 100644 tests/decider/config.rs delete mode 100644 tests/decider/contract.rs delete mode 100644 tests/decider/data/README.md delete mode 100644 tests/decider/data/cpu.json delete mode 100644 tests/decider/data/responses.json delete mode 100644 tests/decider/executor.rs delete mode 100644 tests/decider/fail_head.c delete mode 100644 tests/decider/gpu.rs delete mode 100644 tests/decider/test_download.py delete mode 100644 tests/decider/test_reference.py delete mode 100644 tests/decider/verify_http.py delete mode 100644 tests/decider/verify_modes.py delete mode 100644 tests/decider/verify_reference.py delete mode 100644 tests/decider/verify_retirement.py diff --git a/README.md b/README.md index 67596c77..62c30ed0 100644 --- a/README.md +++ b/README.md @@ -43,7 +43,7 @@ scheduling from model execution; a native Metal backend is planned, while LAYA already has a Python worker for Apple GPUs through PyTorch MPS. The Rust frontend forwards requests to a separately running model worker. The -Cua-S1 4B 0.2 `text` adapter, Open-Jev-27B-v1.1, Open-Jev-9B and Decider-2B v11 have native +Cua-S1 4B 0.2 `text` adapter, Open-Jev-27B-v1.1 and Open-Jev-9B have native workers using shared CUDA kernels in this repository. ## News @@ -168,7 +168,6 @@ Python screenshot worker, and CLM has a stub-encoder contract recipe: | Cua-S1 4B 0.2 (`multimodal` adapter) | [Python CUDA worker](src/frontend/cua_s1.py); one PNG/JPEG screenshot, `choice`; native screenshot execution remains in progress | | Open-Jev-27B-v1.1 | [Native Rust/CUDA worker](recipe/open_jev/native.md); eager independent text candidates; [H200 validation](recipe/open_jev/validation.md) | | Open-Jev-9B | The same [native Rust/CUDA worker](recipe/open_jev/native.md); eager independent text candidates; [reference comparison on sm_89](recipe/open_jev/validation-9b.md) | -| Decider-2B v11 | [Native Rust/CUDA text worker](recipe/decider/README.md); independent Choice/Noul and isolated Score; [RTX 4090 validation](recipe/decider/validation.md) | | CLM-v0.1-8B | [External worker with a CPU stub encoder](recipe/clm/README.md); contract checks only, real Qwen3-8B decisions unverified by this recipe | [Supported models and hardware](docs/supported-models.md) lists the devices diff --git a/docs/architecture.md b/docs/architecture.md index 660a6865..7710c734 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -7,24 +7,22 @@ design. Concrete input/output types follow each executor's supported layout. ## Implementation status The [Rust frontend](../src/frontend/README.md) currently forwards HTTP requests -to separately running workers. Cua-S1, Open-Jev and Decider have native Rust/CUDA workers +to separately running workers. Cua-S1 and Open-Jev have native Rust/CUDA workers that share the [Qwen3.5/3.8 executor](../src/models/qwen3_5/native/), which accepts single prompts and bounded packed prefill. Cua-S1 and Open-Jev-9B use single-prompt calls; Open-Jev-27B-v1.1 packs candidates within one request for input and gate/up GEMMs while preserving per-sequence mixers and output/down GEMM shapes. [Laya's native worker](../src/models/laya/README.md) -uses a separate Hopper CUDA backend for one complete padded request. All four +uses a separate Hopper CUDA backend for one complete padded request. All three coordinate independent processors and executors through `prepare` → `execute` → `finish` and use the [native runtime](../src/runtime/README.md) for FIFO admission and blocking dispatch per loaded executor. Shared processing orchestration, batch budgets, compatibility grouping and dynamic batching are planned. -Cua-S1 and Open-Jev compute their decision heads on the CPU after downloading the -final hidden state. Decider projects selected tied-embedding labels with a BF16 CUDA -GEMM, then calibrates the downloaded logits on the CPU. Laya computes its scorer -and action head on CUDA. Its fixed-shape +Qwen workers compute their decision heads on the CPU after downloading the final +hidden state. Laya computes its scorer and action head on CUDA. Its fixed-shape Graph captures Encoder/Decision; gather, scorer/action head and synchronized readback remain outside capture. Native Metal remains planned; Python workers retain their documented reference/serving roles. @@ -33,11 +31,7 @@ retain their documented reference/serving roles. The native workers separate `processing.rs` from `executor.rs`; their worker assembly owns a `SerialScheduler` per loaded executor, and the HTTP handler -coordinates the three stages. Decider verifies the immutable released checkpoint -and configuration before CUDA initialization, and admits one complete request -including every row and GPU readout. Both streams synchronize before release; -execution failure retires the loaded state and makes health unavailable. -Preparation validates the entire request before +coordinates the three stages. Preparation validates the entire request before any forward call and returns executor inputs plus a response context. The context retains question and candidate identity, usage, and response metadata outside the executor. @@ -45,16 +39,12 @@ and candidate identity, usage, and response metadata outside the executor. | --- | --- | --- | --- | | Cua-S1 | One unpadded token-ID vector and option count per question, in request order. | One FP32 answer-letter logit vector per question. | Per-question softmax, choice/confidence, ordered answers, and token usage. | | Open-Jev | Token-ID vectors grouped by question, then independent candidate, in request order. | One FP32 learned scalar per candidate in the same grouping. | Add the `noul` false logit of zero, calibrate across each complete question, and restore typed answers, usage, and metadata. | -| Decider | Complete independent unpadded rows, including separate no/yes rows for each Score level, with selected-label IDs and final readout positions. | BF16-projection-rounded FP32 candidate logits in row order. | Per-type calibration, whole-question isolated Score assembly, ordered typed answers and unique-prefix usage. | | Laya | One padded request: token IDs, true lengths, question types and ordered option markers; at most 16 questions, 512 tokens per row and 2048 markers. | Per-question FP32 option logits and two action logits copied back after GPU heads. | Calibrate and decode ordered `choice`, `score` and `noul` answers, usage and metadata. | Cua-S1 and Open-Jev-9B input collections are serial work. For Open-Jev-27B-v1.1, Open-Jev's model-specific batch adapter packs up to 16 independent candidates and 4096 tokens per group; longer prompts execute alone. It restores question/candidate grouping before normalization. -Decider optionally packs bounded contiguous independent complete rows within one -request, preserving Score-level reconstruction and unique-prefix usage. The -selected-label projection is batched; request admission remains serial. Laya batches questions within one request. Shared runtime admission precedes blocking dispatch: Cua-S1 admits one question forward at a time; Open-Jev and Laya admit one complete request. Cua-S1's CPU letter projection @@ -66,10 +56,8 @@ dispatch retains the scheduler permit until execution completes. Laya synchronizes and disposes a failed model before returning an inference error and reports unavailable health thereafter. -Cua-S1 and Open-Jev executors own loaded Qwen models and CPU head weights, preserving -FP64 accumulation and the existing FP32 rounding and bias order. Decider retains -selected tied BF16 weights and preserves BF16 projection output rounding before -FP32 calibration. Finishing checks output cardinality +Qwen executors own loaded models and CPU head weights, preserving FP64 accumulation and +the existing FP32 rounding and bias order. Finishing checks output cardinality before reconstruction. HTTP validation, error status/body conventions, and real warmup before readiness remain model-specific and unchanged. diff --git a/docs/supported-models.md b/docs/supported-models.md index 3fdecc73..9aba8d39 100644 --- a/docs/supported-models.md +++ b/docs/supported-models.md @@ -16,7 +16,6 @@ Models that are being added are also tracked in issues labeled [new model](https | Cua-S1 4B 0.2, `multimodal` adapter | Reference worker on Transformers and PEFT, [`src/frontend/cua_s1.py`](../src/frontend/cua_s1.py); no recipe yet | Not supported | Validated ([#17](https://github.com/ThinkFlowLab/system1-omni/pull/17), [#18](https://github.com/ThinkFlowLab/system1-omni/pull/18)) | Not supported | The state is one PNG or JPEG image; upstream's `weights.lock.json` next to the base weights | | Open-Jev-27B-v1.1 | [Native Rust/CUDA worker](../recipe/open_jev/native.md) on the shared Qwen3.5/3.8 executor | Not supported | Validated on H200 (sm_90) for the [74 single-candidate workload](../recipe/open_jev/validation.md) | Not supported | Compute capability 8.0 or newer, CUDA toolkit to build, exported merged weights and trained head | | Open-Jev-9B | The same [native Rust/CUDA worker](../recipe/open_jev/native.md) | Not supported | Validated on compute capability 8.9 against the reference for [253 requests](../recipe/open_jev/validation-9b.md) | Not supported | Compute capability 8.0 or newer, CUDA toolkit to build, exported merged weights and trained head | -| Decider-2B v11 | [Native Rust/CUDA text worker](../recipe/decider/README.md), independent `choice`/`noul` and isolated `score` | CPU processing/checkpoint checks; no CPU inference | BF16 RTX 4090 (sm_89); [validation scope](../recipe/decider/validation.md) | Not supported | Pinned immutable single-file checkpoint, CUDA toolkit/cuBLASLt and existing Qwen backend | | CLM-v0.1-8B | [External `clm-serve` recipe](../recipe/clm/README.md) with a CPU stub embeddings server | **Stub-encoder contract checks only** ([#23](https://github.com/ThinkFlowLab/system1-omni/pull/23)); not real Qwen3-8B decisions | Real encoder unverified by the merged recipe | Unverified | Python, upstream CLM and head checkpoint; a real encoder requires a separate embeddings server | - **Validated:** covered by the recipe on `main` or by the checks in the linked merged pull request. @@ -24,7 +23,7 @@ Models that are being added are also tracked in issues labeled [new model](https - **Planned:** not implemented yet; the linked issue tracks it. The Cua-S1 workers answer `choice` questions only. -LAYA's English worker, Open-Jev and Decider support `choice`, `score`, and `noul` text questions. +LAYA's English worker and Open-Jev support `choice`, `score`, and `noul` text questions. CLM's merged recipe exercises these answer shapes with stub embeddings; it does not validate decision quality. MPS validation above is for a Python/PyTorch worker, not a native Metal backend. @@ -32,7 +31,6 @@ worker, not a native Metal backend. The [architecture contracts](architecture.md) describe the native target. Shared processing orchestration and dynamic batching remain planned. Native workers reuse serial admission. Qwen workers run independent single-prompt -prefills; Cua-S1/Open-Jev use CPU heads and Decider uses a BF16 CUDA selected-label -projection. Laya packs questions within one request and runs its +prefills with CPU heads; Laya packs questions within one request and runs its scorer/action head on CUDA. These target layers do not expand the validated model or hardware coverage above. diff --git a/mkdocs.yml b/mkdocs.yml index b685a3d0..e276a583 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -68,7 +68,5 @@ nav: - Open-Jev native text worker: recipe/open_jev/native.md - Open-Jev-27B H200 validation: recipe/open_jev/validation.md - Open-Jev-9B validation: recipe/open_jev/validation-9b.md - - Decider native text worker: recipe/decider/README.md - - Decider RTX 4090 validation: recipe/decider/validation.md - CLM stub-encoder contract: recipe/clm/README.md - Contributing: CONTRIBUTING.md diff --git a/recipe/README.md b/recipe/README.md index a52ba6d2..8909eb94 100644 --- a/recipe/README.md +++ b/recipe/README.md @@ -17,8 +17,6 @@ For a first real decision, follow the [complete CPU walkthrough](../docs/getting backbone and trained decision head of Open-Jev-27B-v1.1 or Open-Jev-9B, then serve with Rust and CUDA. [Open-Jev-9B validation](open_jev/validation-9b.md) compares the 9B worker with the reference. -- [Decider-2B v11 native text worker](decider/README.md): verify the released - checkpoint and serve independent Choice/Noul and isolated Score with Rust/CUDA. - [CLM behind the frontend](clm/README.md): run CLM's own server behind the frontend on CPU with a stub encoder, and what the response comparison has to allow for. diff --git a/recipe/decider/README.md b/recipe/decider/README.md deleted file mode 100644 index 554ac082..00000000 --- a/recipe/decider/README.md +++ /dev/null @@ -1,137 +0,0 @@ -# Native Decider-2B v11 text decisions - -Run these commands from the repository root on Linux with Rust, an NVIDIA GPU, -CUDA toolkit/nvcc and cuBLASLt. The existing Qwen backend targets compute capability -8.0 or newer; hardware validation is described in [validation.md](validation.md). -The worker itself uses no Python/PyTorch serving process. Python 3.10+ is used only -for downloading and optional reference validation. Keep at least 8 GB disk free for -weights and Rust build products; reference Python packages need additional space. -Device memory includes about 3.76 GB of BF16 checkpoint data plus model/head buffers -and length-dependent activation/workspace storage. - -## Download and build - -```sh -python3 recipe/decider/download_weights.py /path/to/models/decider-2b-v11 -cargo build --release --locked -p omni-decider-native -p omni-jev -NVCC=/usr/local/cuda/bin/nvcc \ - bash src/backends/cuda/qwen3_5/build.sh target/release 89 -``` - -Replace `89` with the target GPU's compute capability. The download helper pins -`Mapika/decider-2b` revision `533964dae8be954c5b5e19fa4948e48408094c1e` and checks -all five downloaded files; it never downloads unpinned remote Python code. Its -`--endpoint` option can select an accessible Hugging Face mirror; fixed hashes -remain required. Interrupted downloads leave no usable unverified final file. - -Startup independently hashes the required checkpoint/config/calibration/tokenizer -and rejects changed artifacts before loading CUDA. Use the original single -`model.safetensors`, with no `model.safetensors.index.json`; no export or LoRA merge -is needed. Keep files immutable while loading and serving. - -## Start the worker and frontend - -Worker terminal: - -```sh -CUA_S1_GRAPH=0 \ -DECIDER_MODEL=/path/to/models/decider-2b-v11 \ -DECIDER_CUDA_LIB="$PWD/target/release/libqwen3_5_cuda.so" \ -DECIDER_HOST=127.0.0.1 DECIDER_PORT=8000 \ - ./target/release/omni-decider -``` - -Decider's first path is eager. `CUA_S1_GRAPH=1` is rejected at startup so a shared -backbone switch cannot silently change this path. An unset switch is also accepted. -`DECIDER_CUDA_LIB` defaults to the library beside the binary. The worker uses CUDA -device 0; use `CUDA_VISIBLE_DEVICES` to select a physical device. - -The socket binds only after artifact validation, CUDA loading and a real warmup -request. Startup failure does not advertise readiness. In another terminal: - -```sh -curl --fail http://127.0.0.1:8000/health -OMNI_JEV_BIND=127.0.0.1:8080 \ -OMNI_JEV_BACKEND_URL=http://127.0.0.1:8000 \ - ./target/release/omni-jev -``` - -Health returns the model identity, checkpoint/reference revisions, BF16/eager mode -and effective per-type temperatures. A retired executor returns HTTP 503 and -`status: unavailable`. The frontend exposes its existing health behavior; see the -[frontend configuration](../../src/frontend/README.md) for its own limits/timeouts. - -## First decision - -```sh -curl --fail http://127.0.0.1:8080/v1/systemone \ - -H 'Content-Type: application/json' \ - --data-binary @recipe/decider/example-request.json -``` - -The example asks a Choice, Noul and three-level Score question about one text -state. The response has model `decider-2b-v11`, ordered typed answers and usage. -Question identities are omitted from model text. The exact output depends on the -checkpoint computation; this example is not a task-quality guarantee. - -Errors use JSON `detail`: HTTP 415 for unsupported Content-Type, 413 for a body -above 8 MiB, 422 for invalid/unsupported requests, and 503 when the model becomes -unavailable after an execution failure. Request-preparation task failures return -500. Invalid inputs do not retire the executor. Empty questions return HTTP 200 -with empty answers and zero input/output usage. No partial answers are returned. - -Supported inputs are plain text/JSON state, independent Choice (2–255 options), -Noul and isolated Score (2–10 levels). State-prefix truncation is 32,768 tokens; -complete rows can include question suffixes up to 36,864 tokens. At most 1,024 -expanded rows and 1,048,576 processed tokens are accepted per request. These bounds -do not bound all waiting requests together; runtime pending-queue limits and -cross-request batching remain separate work. - -The pinned worker requires released calibration. Images, video, chat/schema-first, -packed questions, shared-prefix caching, quantized/CPU/Metal execution and Graph -optimizations are unsupported. See the [model contract](../../src/models/decider/README.md) -for native JSON restrictions and the processing/execution boundary. - -## Validation and diagnostics - -```sh -cargo fmt --all --check -cargo clippy --workspace --locked --all-targets -- -D warnings -cargo test --workspace --locked -cargo build --workspace --release --locked -DECIDER_MODEL=/path/to/models/decider-2b-v11 \ - cargo test --release --locked -p omni-decider-native --test contract -- --ignored -DECIDER_MODEL=/path/to/models/decider-2b-v11 \ -DECIDER_CUDA_LIB="$PWD/target/release/libqwen3_5_cuda.so" CUA_S1_GRAPH=0 \ - cargo test --release --locked -p omni-decider-native --test gpu -- --ignored -DECIDER_CUDA_LIB="$PWD/target/release/libqwen3_5_cuda.so" \ - cargo test --release --locked -p omni-decider-native --lib \ - bf16_head_rounds_before_fp32_calibration -- --ignored -``` - -The CPU contract target requires only tokenizer/config; the GPU target requires the -complete pinned checkpoint. The projection test needs a built backend and a GPU. - -For exact prepared rows and raw BF16-rounded logits, the diagnostic binary reads -one JSON request per line and emits rows, logits and complete response: - -```sh -CUA_S1_GRAPH=0 ./target/release/decider-run \ - /path/to/models/decider-2b-v11 "$PWD/target/release/libqwen3_5_cuda.so" \ - < recipe/decider/example-request.json -``` - -See [validation.md](validation.md) for full-checkpoint reference comparison, -HTTP/frontend checks, prerequisites and the recorded scope. - -## Optional request-local packing - -Set `DECIDER_BATCH_MAX_ROWS=4 DECIDER_BATCH_MAX_TOKENS=4096` on `omni-decider` -to pack independent rows within one admitted request. Rows default to 1; accepted -limits are 1–4 rows and 1–4096 packed tokens. Longer complete rows execute alone. -Malformed or out-of-range settings fail startup. These limits do not change HTTP -admission, token usage, or the maximum complete-row length. - -The selected-label head uses persistent buffers sized to the row limit and projects -each batch in one BF16 GEMM. No cross-request batching is introduced. Keep the -single-row default until numerical and performance results fit your workload. diff --git a/recipe/decider/example-request.json b/recipe/decider/example-request.json deleted file mode 100644 index fbec40a6..00000000 --- a/recipe/decider/example-request.json +++ /dev/null @@ -1 +0,0 @@ -{"state":"A customer received a damaged item and requests a refund.","questions":{"route":{"type":"choice","instructions":"Choose the responsible team.","criteria":{"billing":"Refund and billing","shipping":"Delivery status","sales":"New purchases"}},"refund":{"type":"noul","instructions":"The customer requests a refund."},"severity":{"type":"score","instructions":"Severity of the damage","criteria":["Minor","Moderate","Severe"]}}} diff --git a/recipe/decider/validation.md b/recipe/decider/validation.md deleted file mode 100644 index e0042dba..00000000 --- a/recipe/decider/validation.md +++ /dev/null @@ -1,168 +0,0 @@ -# Decider-2B v11 validation on RTX 4090 - -Validation date: 2026-10-07. This records one BF16 CUDA device and a frozen synthetic -contract corpus. It does not establish task quality, every supported GPU, performance -improvement or universal bit-identical agreement with Transformers. - -## Controls and fidelity gates - -Checkpoint/tokenizer/calibration: Mapika/decider-2b -`533964dae8be954c5b5e19fa4948e48408094c1e`. Checkpoint SHA-256: -`acaef2228b134dcdc20cad4ee79219482c927ec819aa3687b9b8a575c338817f`. -Native checkpoint/config/tokenizer checks are in `src/models/decider/native/src/checkpoint.rs`. -Reference runtime: Mapika/decider `50d0be0d7cb43d2066965ce5fa7f3fe4e489a60f`, -decider-ai 1.8.1. The comparison harness verifies fixed hashes of all five imported -reference files before running either implementation and saves them in its protocol. - -RTX 4090, sm_89, 24 GiB; driver 595.71.05, CUDA toolkit 13.0.88, Rust 1.98.1, -Python 3.12, Torch 2.14.0+cu130 and Transformers 5.17.0. Native runs one complete -unpadded row at a time, eager with `CUA_S1_GRAPH=0`; it uses the existing ABI5 Qwen -backend, rebuilt from this branch. The reference uses `DecisionModel.slot_logits` -with unpadded independent rows and disabled cache. Its missing causal-conv1d and -flash-linear-attention packages use Transformers' PyTorch reference fallbacks. -No optimized-reference speed comparison is claimed. - -Before execution, the harness fixes these gates: - -- Maximum absolute answer-probability and per-level fit drift: **0.02**. -- Maximum Score expectation drift: **0.1**. -- Identical Choice decisions when reference top-two probability margin is at least - **0.05**; report all lower-margin disagreements. -- Exact prepared tokens, candidate IDs/order, readout positions, usage and answer - identities. Complete responses retain all model-specific fields. - -## Recorded results - -The 18-request corpus has **41 independent inference rows**. It covers Choice with -2/3/10/11/26/255 options, Score with 2/3/10 isolated levels, true/false and -criteria-only Noul, structured Unicode and long arrays, mixed questions, reversed -question order, repeated input, empty questions and long context. The longest -reference-compared row is 4,140 tokens. - -Both the initial run and the source-hash-verified rerun pass the same gates: -maximum probability/per-level-fit drift **0.0119**, maximum Score drift **0.01**, -and **zero Choice disagreements**, including lower-margin cases. Raw logit drift -is reported separately (maximum 0.125); logits are not bit-identical. These are -observed results for this corpus, not a general numerical-error bound. - -CPU opt-in tests check 12 pinned token/row/usage fixtures, every label through JT, -12 response assemblies including zero-fit Score, and state-prefix truncation. -Normal CPU tests cover selected-row ordering/padding, invalid shapes/IDs/nonfinite -weights, file content/size hashes, alternate index rejection and complete-row -validation before dispatch. A real GPU microtest checks the BF16 projection tie -`1 + 0.00390625 -> 1`, before FP32 calibration, and excludes the padded label. - -Real model regression checks verify mixed answer order, exact repeated output, -validation-error recovery and empty requests without inference. The separate native GPU boundary test executes a **36,864-token complete row** -twice, rejects the next token before execution, and then reproduces an earlier short -request exactly after scratch growth. It does not expand the reference-parity claim -beyond 4,140 tokens. - -HTTP testing passes **27 checks**: 18 direct/native diagnostic/frontend comparisons, -six invalid-input cases, the worker body limit, concurrent requests and recovery -health. The concurrent check makes 12 requests across direct and frontend paths -with four client threads; all returned bytes match the serial baseline. The first -HTTP harness run stopped on a Python variable-shadowing error after its earlier -checks; the corrected full run passes. The worker binaries were unchanged. - -The test-only CUDA wrapper lets the genuine warmup projection complete, then fails -only the second `1 x 256 x 2048` head GEMM. The worker returns 503, health becomes -unavailable, two later requests are refused, and exactly two projections are seen. -Sampled per-process GPU residency drops from **4,192 MiB to 394 MiB**, demonstrating -retirement of the model allocation; these are two lifecycle samples, not peak-memory -or performance measurements. The wrapper delegates all other operations to the -real library and is never used by normal serving. - -Cancellation semantics inherit the tested shared SerialScheduler. Decider-specific -caller cancellation and injected Rust panic are not separately hardware-tested. -CPU/Metal inference, other GPUs/precisions, dynamic batching, prefix reuse, Decider -Graph replay, model-quality datasets, throughput and cold/warm latency comparisons -are outside this validation. - -## Reproduce - -Build and prepare the checkpoint using [the native recipe](README.md). For the -reference environment, use Python 3.12 and the recorded Torch/Transformers versions; -install Transformers 5.17.0, tokenizers 0.22.2 and Torch 2.14.0 built for CUDA 13.0. -The native worker's preparation and serving require none of these Python packages. - -Obtain the pinned reference in a separate directory: - -```sh -git clone https://github.com/Mapika/decider.git /path/to/decider-reference -git -C /path/to/decider-reference checkout 50d0be0d7cb43d2066965ce5fa7f3fe4e489a60f -``` - -From the System1-Omni repository root, with a reserved/available NVIDIA GPU: - -```sh -CUA_S1_GRAPH=0 python tests/decider/verify_reference.py \ - --model /path/to/models/decider-2b-v11 \ - --reference /path/to/decider-reference \ - --binary "$PWD/target/release/decider-run" \ - --library "$PWD/target/release/libqwen3_5_cuda.so" \ - --output /path/to/evidence/parity - -python tests/decider/verify_http.py \ - --worker "$PWD/target/release/omni-decider" \ - --frontend "$PWD/target/release/omni-jev" \ - --model /path/to/models/decider-2b-v11 \ - --library "$PWD/target/release/libqwen3_5_cuda.so" \ - --parity-output /path/to/evidence/parity \ - --output /path/to/evidence/http -``` - -The HTTP runner starts local sockets on 18110/18111, checks readiness and shuts its -processes down even on failure. It retains worker/frontend logs and complete results. -Do not run it over unrelated processes already using these ports. - -For the isolated failure test, build a wrapper beside the real library so its -runtime search path resolves the actual backend: - -```sh -gcc -shared -fPIC -Wall -Wextra -I src/backends/cuda/qwen3_5 \ - tests/decider/fail_head.c -L target/release -Wl,--no-as-needed \ - -lqwen3_5_cuda -Wl,-rpath,'$ORIGIN' -ldl \ - -o target/release/decider-fail-head.so -python tests/decider/verify_retirement.py \ - --worker "$PWD/target/release/omni-decider" \ - --model /path/to/models/decider-2b-v11 \ - --wrapper "$PWD/target/release/decider-fail-head.so" \ - --output /path/to/evidence/retirement -``` - -This runner uses port 18112 and `nvidia-smi` per-process memory reporting. The recipe -also lists the explicit CPU and real-GPU Cargo tests. Ordinary workspace tests -compile/discover their ignored cases but do not establish hardware validation. - -## Evidence - -The task evidence archive retains both parity runs, the original HTTP harness -failure and its successful rerun, native/reference rows/logits/responses, protocol, -source/binary/library hashes, environment inventory, GPU tests and retirement logs. -Generated run output and binaries are kept out of the source tree; only maintained -fixtures and reproduction helpers are committed. The implementation PR links the -immutable evidence archive and its SHA-256. All requests are synthetic and carry -no private/customer content. A video is inapplicable to this text API integration. - -## Request-local batching comparison - -The followup runner `tests/decider/verify_modes.py` accepts a frozen JSON plan with -`model`, `library`, `parity_output` (the pinned reference run above), -`repetitions: 2`, `timed_cases` and `modes`. Each mode supplies `name`, an absolute -`binary` path and an `env` mapping. It hashes binaries, library, reference records -and comparison scripts before execution; retains complete records; checks prepared -rows exactly and the same response gates. Time is the diagnostic JSON-lines -roundtrip at concurrency 1, including processing, synchronization, serialization -and IPC. Startup to the empty diagnostic response and one workload warmup are -reported/excluded separately. These measurements do not establish HTTP latency, -peak memory or production throughput. Device memory is a whole-device sample at -the end of each mode, not a peak measurement. - -On the RTX 4090 followup campaign, row limits 2 and 4 passed all 18 reference -requests. Row limit 8 failed the unchanged probability gate on structured Unicode -input (0.0208 > 0.02), so supported packing is capped at **4 rows**. The failed -campaign and narrowed-protocol rerun are retained in the PR evidence archive. -Defaults remain one row; do not extrapolate these small workloads to arbitrary -requests or hardware. Batch fault injection also verifies a two-row head failure -retires the complete worker after real startup warmup. diff --git a/src/backends/cuda/qwen3_5/README.md b/src/backends/cuda/qwen3_5/README.md index e4f5a22c..e142176e 100644 --- a/src/backends/cuda/qwen3_5/README.md +++ b/src/backends/cuda/qwen3_5/README.md @@ -1,6 +1,6 @@ # Qwen3.5/3.8 prefill operations -CUDA kernels for a prefill-only Qwen3.5/3.8 forward pass, built into `libqwen3_5_cuda.so` with a C interface ([`ops.h`](ops.h)), so that a Rust model engine loads it at run time and builds without a CUDA toolkit. The Cua-S1, Open-Jev and Decider native workers share the layer loop and buffers in [`src/models/qwen3_5/native/`](../../../models/qwen3_5/native/). +CUDA kernels for a prefill-only Qwen3.5/3.8 forward pass, built into `libqwen3_5_cuda.so` with a C interface ([`ops.h`](ops.h)), so that a Rust model engine loads it at run time and builds without a CUDA toolkit. The Cua-S1 and Open-Jev native workers share the layer loop and buffers in [`src/models/qwen3_5/native/`](../../../models/qwen3_5/native/). ```sh src/backends/cuda/qwen3_5/build.sh [compute capability, default 89] @@ -20,9 +20,3 @@ The shared Rust model can pack independent sequences for input and gate/up GEMMs Output/down GEMMs retain each prompt's original shape and reduction order; attention, convolution and GDN calls remain sequence-local. The CUDA ABI is unchanged. Open-Jev uses this path within requests; Cua-S1 keeps single-prompt calls. - -Decider-2B v11 reuses these operations for eager independent text rows and its -BF16 selected tied-embedding projection. It pads the 255 retained labels to 256 -GEMM rows and excludes padding from the output; no CUDA ABI or kernel change is -needed. See the [Decider recipe](../../../../recipe/decider/README.md) and -[RTX 4090 full-checkpoint scope](../../../../recipe/decider/validation.md). diff --git a/src/models/decider/README.md b/src/models/decider/README.md deleted file mode 100644 index b0f02b05..00000000 --- a/src/models/decider/README.md +++ /dev/null @@ -1,77 +0,0 @@ -# Decider-2B v11 native worker - -The native Rust/CUDA worker serves text `choice`, `noul` and isolated-level `score` -through `POST /v1/systemone`. Follow the [native recipe](../../../recipe/decider/README.md). -The [validation protocol](../../../recipe/decider/validation.md) distinguishes -CPU contract checks, real checkpoint inference and HTTP checks. - -## Pinned artifacts and contract - -- Model/tokenizer/config: Mapika/decider-2b - `533964dae8be954c5b5e19fa4948e48408094c1e` (released 2B v11). -- Reference: Mapika/decider `50d0be0d7cb43d2066965ce5fa7f3fe4e489a60f`, decider-ai 1.8.1. -- BF16, 24 Qwen3.5 layers, hidden width 2048, 18 Gated DeltaNet and six full-attention - layers. Output embeddings are tied; there is no adapter merge or separate trained head. -- Plain state-first, independent questions; Score levels become separate no/yes rows. - Choice supports 2–255 single-token labels A through JT; Score supports 2–10 levels. -- Per-type temperatures are Choice 1.164, Noul 1.624 and Score 1.124, read from the - verified released configuration. The worker accepts only the original calibration - artifact. The CPU library's configuration API also supports explicit calibration - overrides, but those are outside the pinned worker's supported scope. -- Preserve ordered structured state/question rendering, long-array annotations, - per-question output ordering, confidence/certainty/legend/level-fit fields and - Python-style response rounding. `usage.input_tokens` counts the common prefix once; - `usage.output_tokens` is zero. Empty questions return empty answers without execution. - -State truncation caps the tokenized `Context:` prefix at 32,768 tokens; the question -suffix is additional. Native admission bounds complete rows to 36,864 tokens, -expanded rows to 1,024, total processed row tokens to 1,048,576 and raw bodies to -8 MiB. Unsupported modes, duplicate JSON keys, depth >=128, nonfinite literals -and integers outside i64/u64 are rejected. No image/video, chat/schema-first, -packed-question execution, neutralization, quantization, CPU/Metal inference, -shared-prefix cache or Decider CUDA Graph optimization is included. - -## Ownership and execution - -`processing.rs` compiles the entire request before device work. Prepared rows own -unpadded token IDs, final-position readout, candidate IDs and original identities; -`ResponseContext` owns whole-question calibration and answer reconstruction. - -`checkpoint.rs` verifies fixed sizes and SHA-256 hashes for the checkpoint, -configuration, calibration and tokenizer before CUDA initialization. It rejects -sharded indices that could redirect the shared loader away from the verified file. -Keep artifacts immutable for the entire worker lifetime. Selected BF16 output rows -are loaded from the pinned tied input embedding; all 255 rows are retained, with a -256th zero row for aligned CUDA GEMM storage. - -`executor.rs` owns the shared Qwen model and a separate CUDA projection stream, -GEMM handle and persistent head buffers. Each complete request gets one shared -`SerialScheduler` permit. Default execution runs rows eagerly in order. Optional request-local packing -uses `DECIDER_BATCH_MAX_ROWS` (1–4, default 1) and -`DECIDER_BATCH_MAX_TOKENS` (1–4096, default 4096). Contiguous complete rows are -packed with the shared backbone `forward_batch`; each sequence resets its own -GDN/attention state. A row above the packing budget runs alone under the unchanged -complete-row limit. A persistent head projects all batch hidden rows in one GEMM. -Score levels can cross batch boundaries; response normalization still uses the -complete question. This does not combine separate requests or change admission. The BF16 selected projection uses existing backend GEMM, then converts -its BF16 outputs to FP32 before CPU temperature scaling and normalization. Padding -never participates in softmax or token accounting. Current GEMM/reduction algorithms -can differ from Transformers; numerical agreement is measured rather than assumed. - -Both streams synchronize on completion, errors and caught execution panics before -admission is released. An execution failure retires the loaded model/head and makes -health unavailable. Validation errors preserve readiness. Queued cancellation -removes the caller; after dispatch, resources and admission remain held until work -completes. FIFO waiting is serial and inherits the current runtime's unbounded -pending queue; no new queue/token scheduling policy is claimed. - -`engine.rs` assembles the processor/executor and performs a real warmup before -binding a socket. `serve.rs` handles transport, status codes and worker health. -HTTP CPU preparation uses a blocking task outside GPU admission and the model lock. -The existing Rust frontend forwards bytes to this separately running worker. - -## Tests - -All test bodies and fixtures live under root `tests/decider`. Normal Cargo tests -require neither CUDA nor downloads. Explicit checkpoint/accelerator tests are -registered and ignored until their prerequisites are supplied; see the recipe. diff --git a/src/models/decider/native/Cargo.toml b/src/models/decider/native/Cargo.toml index 70706eb6..5df4bfa6 100644 --- a/src/models/decider/native/Cargo.toml +++ b/src/models/decider/native/Cargo.toml @@ -21,19 +21,3 @@ tokio = { version = "1.49.0", features = ["macros", "net", "rt-multi-thread", "s [[bin]] name = "omni-decider" path = "src/main.rs" - -[[test]] -name = "gpu" -path = "../../../../tests/decider/gpu.rs" - -[[test]] -name = "contract" -path = "../../../../tests/decider/contract.rs" - -[[test]] -name = "config" -path = "../../../../tests/decider/config.rs" - -[[test]] -name = "batching" -path = "../../../../tests/decider/batching.rs" diff --git a/src/models/decider/native/src/bin/decider-run.rs b/src/models/decider/native/src/bin/decider-run.rs deleted file mode 100644 index 4502c5fb..00000000 --- a/src/models/decider/native/src/bin/decider-run.rs +++ /dev/null @@ -1,35 +0,0 @@ -//! Diagnostic JSON-lines runner retaining exact prepared rows and raw logits. -use anyhow::{Context, Result}; -use omni_decider_native::engine::Engine; -use serde_json::json; -use std::{ - io::{self, BufRead}, - path::Path, -}; - -#[tokio::main] -async fn main() -> Result<()> { - let args: Vec = std::env::args().collect(); - let model = args - .get(1) - .context("usage: decider-run MODEL_DIR CUDA_LIBRARY")?; - let library = args - .get(2) - .context("usage: decider-run MODEL_DIR CUDA_LIBRARY")?; - let engine = Engine::load(Path::new(model), Path::new(library)).await?; - for line in io::stdin().lock().lines() { - let line = line?; - let prepared = engine.processor.prepare(line.as_bytes())?; - let rows:Vec<_> = prepared.rows.iter().map(|row| json!({"ids":row.ids,"candidate_ids":row.candidate_ids,"readout_position":row.readout_position,"question_id":row.question_id,"level_index":row.level_index,"temperature":row.temperature})).collect(); - let logits = engine - .executor - .execute(&engine.scheduler, prepared.rows) - .await?; - let response = prepared.context.finish(logits.clone())?; - println!( - "{}", - json!({"rows":rows,"logits":logits,"response":response}) - ); - } - Ok(()) -} diff --git a/src/models/decider/native/src/checkpoint.rs b/src/models/decider/native/src/checkpoint.rs index 05051eac..14bd30da 100644 --- a/src/models/decider/native/src/checkpoint.rs +++ b/src/models/decider/native/src/checkpoint.rs @@ -143,7 +143,3 @@ fn selected_rows( } Ok(rows) } - -#[cfg(test)] -#[path = "../../../../../tests/decider/checkpoint.rs"] -mod tests; diff --git a/src/models/decider/native/src/executor.rs b/src/models/decider/native/src/executor.rs index bd832fce..10d2f8ac 100644 --- a/src/models/decider/native/src/executor.rs +++ b/src/models/decider/native/src/executor.rs @@ -288,7 +288,3 @@ fn validate_rows(rows: &[RowInput], labels: &[u32]) -> Result<()> { } Ok(()) } - -#[cfg(test)] -#[path = "../../../../../tests/decider/executor.rs"] -mod tests; diff --git a/src/runtime/README.md b/src/runtime/README.md index cd1b2a88..bd571b03 100644 --- a/src/runtime/README.md +++ b/src/runtime/README.md @@ -1,6 +1,6 @@ # Native execution runtime -`omni-runtime` provides `SerialScheduler` for the Cua-S1, Open-Jev, Decider and Laya native +`omni-runtime` provides `SerialScheduler` for the Cua-S1, Open-Jev and Laya native workers. Their engines assemble a processor, one scheduler for the loaded executor, and the executor. HTTP handlers still coordinate `prepare` → `execute` → `finish`. @@ -15,7 +15,6 @@ state. Waiting requests no longer occupy blocking threads waiting for that mutex | --- | --- | --- | | Cua-S1 | One question's unpadded Qwen forward. Release before admitting its next question. | Request preparation, CPU letter projection and response finishing. | | Open-Jev | One complete request's independent candidate forwards and CPU scalar heads. | Request preparation and calibrated response finishing. | -| Decider | One complete request of independent rows, including BF16 CUDA selected-label projection and synchronized copyback. | Request preparation and calibrated response finishing. | | Laya | One complete padded request, including GPU scorer/action head and copyback. | Request preparation/padding and calibrated response finishing. | Models retain weights, heads, device state, scratch buffers and graph caches. diff --git a/tests/decider/batching.rs b/tests/decider/batching.rs deleted file mode 100644 index becc198b..00000000 --- a/tests/decider/batching.rs +++ /dev/null @@ -1,37 +0,0 @@ -use omni_decider_native::batching::BatchLimits; -#[test] -fn bounded_contiguous_ranges_preserve_order_and_long_rows() { - let limits = BatchLimits::new(3, 8).unwrap(); - assert_eq!( - limits.ranges(&[2, 3, 3, 1, 9, 2, 2, 2, 2]), - vec![0..3, 3..4, 4..5, 5..8, 8..9] - ); - assert!(limits.ranges(&[]).is_empty()); - assert_eq!(limits.ranges(&[9, 9]), vec![0..1, 1..2]); -} -#[test] -fn row_limit_and_exact_token_boundary_are_independent() { - assert_eq!( - BatchLimits::new(2, 4096).unwrap().ranges(&[1, 1, 1]), - vec![0..2, 2..3] - ); - assert_eq!( - BatchLimits::new(4, 4096).unwrap().ranges(&[2048, 2048, 1]), - vec![0..2, 2..3] - ); - assert_eq!( - BatchLimits::default().ranges(&[1, 2, 3]), - vec![0..1, 1..2, 2..3] - ); -} -#[test] -fn startup_limits_reject_unbounded_or_malformed_settings() { - for (rows, tokens) in [(0, 1), (5, 1), (8, 1), (16, 1), (1, 0), (1, 4097)] { - assert!(BatchLimits::new(rows, tokens).is_err()); - } - assert!(BatchLimits::from_values(Some("wrong"), None).is_err()); - assert!(BatchLimits::from_values(None, Some("-1")).is_err()); - let limits = BatchLimits::from_values(Some("4"), Some("4096")).unwrap(); - assert_eq!(limits.max_rows(), 4); - assert_eq!(limits.max_tokens(), 4096); -} diff --git a/tests/decider/checkpoint.rs b/tests/decider/checkpoint.rs deleted file mode 100644 index a32746f6..00000000 --- a/tests/decider/checkpoint.rs +++ /dev/null @@ -1,69 +0,0 @@ -use super::*; -use safetensors::{Dtype, tensor::TensorView}; -use std::collections::HashMap; - -fn tensor(dtype: Dtype, shape: Vec, bytes: &[u8]) -> Vec { - let view = TensorView::new(dtype, shape, bytes).unwrap(); - safetensors::serialize( - HashMap::from([("model.language_model.embed_tokens.weight", view)]), - None, - ) - .unwrap() -} - -#[test] -fn selects_requested_tied_rows_and_zero_pads_the_head() { - let bytes: Vec = (0..12) - .flat_map(|i| half::bf16::from_f32(i as f32).to_le_bytes()) - .collect(); - let encoded = tensor(Dtype::BF16, vec![3, 4], &bytes); - let st = SafeTensors::deserialize(&encoded).unwrap(); - let selected = selected_rows(&st, &[2, 0], 4, 3).unwrap(); - assert_eq!(&selected[..8], &bytes[16..24]); - assert_eq!(&selected[8..16], &bytes[..8]); - assert_eq!(selected.len(), 8 * 4 * 2); - assert!(selected[16..].iter().all(|&b| b == 0)); -} - -#[test] -fn rejects_invalid_head_layouts_ids_and_nonfinite_weights() { - let bytes = vec![0; 24]; - for (dtype, shape, data) in [ - (Dtype::BF16, vec![3, 4], bytes.clone()), - (Dtype::F32, vec![3, 2], bytes.clone()), - (Dtype::BF16, vec![2, 6], bytes.clone()), - ] { - let encoded = tensor(dtype, shape, &data); - let st = SafeTensors::deserialize(&encoded).unwrap(); - assert!(selected_rows(&st, &[3], 4, 3).is_err()); - } - let mut bytes = bytes; - bytes[..2].copy_from_slice(&half::bf16::NAN.to_le_bytes()); - let encoded = tensor(Dtype::BF16, vec![3, 4], &bytes); - let st = SafeTensors::deserialize(&encoded).unwrap(); - assert!(selected_rows(&st, &[0], 4, 3).is_err()); - assert!(selected_rows(&st, &[], 4, 3).is_err()); - assert!(selected_rows(&st, &[1, 1], 4, 3).is_err()); -} - -#[test] -fn checks_actual_file_content_and_length() { - let path = std::env::temp_dir().join(format!("decider-hash-{}", std::process::id())); - std::fs::write(&path, b"abc").unwrap(); - let digest = "ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad"; - verify_file(&path, 3, digest).unwrap(); - assert!(verify_file(&path, 4, digest).is_err()); - assert!(verify_file(&path, 3, &"0".repeat(64)).is_err()); - std::fs::remove_file(path).unwrap(); -} - -#[test] -fn refuses_an_index_that_would_override_the_verified_single_file() { - let dir = std::env::temp_dir().join(format!("decider-index-{}", std::process::id())); - std::fs::create_dir_all(&dir).unwrap(); - std::fs::write(dir.join("model.safetensors.index.json"), b"{}").unwrap(); - let labels: Vec = (0..255).collect(); - let error = Checkpoint::load(&dir, &labels).err().unwrap(); - std::fs::remove_dir_all(dir).unwrap(); - assert!(error.to_string().contains("sharded"), "{error}"); -} diff --git a/tests/decider/config.rs b/tests/decider/config.rs deleted file mode 100644 index 29c670a2..00000000 --- a/tests/decider/config.rs +++ /dev/null @@ -1,75 +0,0 @@ -use omni_decider_native::{Config, Kind}; -use serde_json::{Value, json}; -use std::{ - path::PathBuf, - sync::atomic::{AtomicU64, Ordering}, -}; - -fn plain() -> Value { - json!({"version":"2b-v11","layout":"plain","max_options":255,"max_state_tokens":32768,"neutralize_none":false,"isolated_levels":true,"temperature":1.0}) -} -struct Directory(PathBuf); -impl Directory { - fn new() -> Self { - static NEXT: AtomicU64 = AtomicU64::new(0); - let path = std::env::temp_dir().join(format!( - "decider-config-{}-{}", - std::process::id(), - NEXT.fetch_add(1, Ordering::Relaxed) - )); - std::fs::create_dir(&path).unwrap(); - std::fs::write(path.join("config.json"),json!({"model_type":"qwen3_5_text","hidden_size":2048,"num_hidden_layers":24,"tie_word_embeddings":true,"vocab_size":248320,"dtype":"bfloat16"}).to_string()).unwrap(); - Self(path) - } - fn calibration(&self, config: &Value) { - std::fs::write(self.0.join("decider_config.json"), config.to_string()).unwrap(); - } -} -impl Drop for Directory { - fn drop(&mut self) { - std::fs::remove_dir_all(&self.0).unwrap(); - } -} -#[test] -fn public_validator_rejects_chat_metadata_on_plain_layout() { - let mut config = plain(); - config["chat_template"] = json!(true); - let error = - Config::from_value(&config).expect_err("contradictory chat template must be rejected"); - assert!(error.to_string().contains("chat_template")); -} -#[test] -fn file_loader_rejects_chat_metadata_on_plain_layout() { - let dir = Directory::new(); - let mut config = plain(); - config["chat_template"] = json!(true); - dir.calibration(&config); - let error = Config::load(&dir.0).expect_err("file-loading path must reject contradiction"); - assert!(error.to_string().contains("chat_template")); -} -#[test] -fn missing_and_disabled_chat_metadata_preserve_plain_calibration() { - let dir = Directory::new(); - for config in [plain(), { - let mut c = plain(); - c["chat_template"] = json!(false); - c - }] { - assert_eq!( - Config::from_value(&config) - .unwrap() - .temperature(Kind::Choice), - 1.0 - ); - dir.calibration(&config); - assert_eq!(Config::load(&dir.0).unwrap().temperature(Kind::Score), 1.0); - } -} -#[test] -fn unsupported_chat_metadata_types_are_rejected() { - for value in [json!("true"), json!(1), json!(null), json!([]), json!({})] { - let mut config = plain(); - config["chat_template"] = value; - assert!(Config::from_value(&config).is_err()); - } -} diff --git a/tests/decider/contract.rs b/tests/decider/contract.rs deleted file mode 100644 index 344ce5e9..00000000 --- a/tests/decider/contract.rs +++ /dev/null @@ -1,93 +0,0 @@ -use omni_decider_native::{Limits, Processor}; -use serde_json::{Value, json}; - -fn processor() -> Processor { - Processor::load( - std::env::var("DECIDER_MODEL").expect("set DECIDER_MODEL"), - Limits::default(), - ) - .unwrap() -} -#[test] -#[ignore = "requires the pinned tokenizer/config; CPU only, no tensor loading"] -fn pinned_tokens_rows_usage_and_all_labels() { - let processor = processor(); - let golden: Value = serde_json::from_str(include_str!("data/cpu.json")).unwrap(); - let labels: Vec = serde_json::from_value(golden["label_ids"].clone()).unwrap(); - assert_eq!( - processor - .labels() - .iter() - .map(|label| label.id) - .collect::>(), - labels - ); - assert_eq!(processor.labels().last().unwrap().name, "JT"); - for fixture in golden["fixtures"].as_array().unwrap() { - let request = json!({"state":fixture["state"],"questions":fixture["questions"]}); - let prepared = processor.prepare(request.to_string().as_bytes()).unwrap(); - assert_eq!( - prepared.rows.len(), - fixture["items"].as_array().unwrap().len() - ); - assert_eq!( - prepared.context.input_tokens(), - fixture["unique_tokens"].as_u64().unwrap() as usize - ); - assert_eq!( - prepared.processed_tokens(), - fixture["processed_tokens"].as_u64().unwrap() as usize - ); - for (row, item) in prepared - .rows - .iter() - .zip(fixture["items"].as_array().unwrap()) - { - assert_eq!( - row.ids, - serde_json::from_value::>(item["ids"].clone()).unwrap(), - "{}", - fixture["name"] - ); - let count = item["nopts"][0].as_u64().unwrap() as usize; - assert_eq!(row.candidate_ids, labels[..count]); - assert_eq!(row.readout_position, row.ids.len() - 1); - } - } -} -#[test] -#[ignore = "requires the pinned tokenizer/config; CPU only, no tensor loading"] -fn pinned_assemblies_and_state_truncation() { - let processor = processor(); - let golden: Value = serde_json::from_str(include_str!("data/responses.json")).unwrap(); - for fixture in golden.as_array().unwrap() { - let prepared = processor - .prepare(fixture["request"].to_string().as_bytes()) - .unwrap(); - let logits = serde_json::from_value(fixture["logits"].clone()).unwrap(); - assert_eq!( - prepared.context.finish(logits).unwrap()["answers"], - fixture["expected_answers"], - "{}", - fixture["name"] - ); - } - let request = json!({"state":"hello ".repeat(40000),"questions":{"q":{"instructions":"Choose.","criteria":["A","B"]}}}); - let row = processor - .prepare(request.to_string().as_bytes()) - .unwrap() - .rows - .pop() - .unwrap(); - assert!(row.ids.len() > 32768 && row.ids.len() <= 36864); - let first = row.ids[..32768].to_vec(); - let request = json!({"state":"hello ".repeat(45000),"questions":{"q":{"instructions":"Choose.","criteria":["A","B"]}}}); - assert_eq!( - processor - .prepare(request.to_string().as_bytes()) - .unwrap() - .rows[0] - .ids[..32768], - first - ); -} diff --git a/tests/decider/data/README.md b/tests/decider/data/README.md deleted file mode 100644 index 6c4aeae4..00000000 --- a/tests/decider/data/README.md +++ /dev/null @@ -1,23 +0,0 @@ -# Pinned CPU fixtures - -`cpu.json` retains 12 request/token/row/usage fixtures and all 255 label IDs from -Mapika/decider `50d0be0d7cb43d2066965ce5fa7f3fe4e489a60f`, using Decider-2B v11 -`533964dae8be954c5b5e19fa4948e48408094c1e`. The tokenizer SHA-256 is checked by -`Processor::load`; the tests require only tokenizer and configuration files. -The large truncation fixture is replaced with generated assertions in `contract.rs`. - -`responses.json` retains 12 pinned Torch 2.14.0 CPU assemblies, including isolated -Score, 255-label uniform Choice, and zero-fit Score. Inputs are synthetic, with no -private text. Apache-2.0 reference attribution is retained in the native crate. - -Run from the repository root: - -```sh -DECIDER_MODEL=/path/to/decider cargo test --locked -p omni-decider-native \ - --test contract -- --ignored -``` - -`verify_reference.py` independently reconstructs prompts with pinned upstream -`prompt_fast`, `systemone` and `temperature`, then compares actual BF16 CUDA -checkpoint outputs and complete native responses. It saves its protocol before -execution and retains raw rows/logits/responses outside the repository fixtures. diff --git a/tests/decider/data/cpu.json b/tests/decider/data/cpu.json deleted file mode 100644 index 13e2ec7a..00000000 --- a/tests/decider/data/cpu.json +++ /dev/null @@ -1 +0,0 @@ -{"label_ids":[32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,5840,1803,1646,1745,13276,8018,1825,28946,15015,29595,11568,939,1354,1058,18183,2456,88898,905,1846,802,33869,7839,14006,2860,2926,22828,6844,9798,4738,9265,11261,19278,36513,93801,8335,14544,85266,9110,28000,15137,4525,25261,12717,7116,17078,14497,57339,74909,52072,19305,4887,12607,3580,6281,2036,9362,8533,2080,10911,2925,3040,9690,27731,8023,6901,8702,6211,1123,16307,18990,64045,63037,33380,6151,3392,5449,3967,1113,5095,51923,49600,17099,51483,17756,16037,8135,30237,5683,9992,7444,5751,10284,20887,59884,52396,16103,67547,18535,8006,7263,1425,6878,14453,9097,44072,76089,68720,2662,2629,923,6548,8924,52194,622,1515,1300,37523,44473,36530,3152,93924,3505,15731,6542,14176,11091,1686,11660,80440,18836,26132,5934,24794,40229,3660,11361,10191,8225,3860,78413,17680,15900,78138,15653,5213,22150,39500,10460,35131,21563,43194,27127,3697,20011,24368,15058,23658,8362,16035,24583,52857,38403,60456,81519,40097,16522,29722,21756,18567,1736,48043,87013,22456,23165,55523,13097,50397,41741,23073,6401,86538,16585,11622,2464,84982,75516,36984,58795,47217,59675,5681,3151,1271,887,5203,2685,1849,72470,5370,74063,27629,1655,1728,669,3682,3191,59865,2712,1580,922,77243,2990,78493,5228,2750,42711,44568,56402,48236,38993,43559,61250,32942,86302,25310,26313,81770,12185,78382],"fixtures":[{"name":"choice_2","state":"退款 <|im_start|>\n café","questions":{"hidden-id":{"type":"choice","instructions":"Pick 中文","criteria":{"候选0":null,"候选1":{"text":"é","n":1}}}},"rendered_state":"退款 <|im_start|>\n café","rendered_questions":{"hidden-id":{"question":"Pick 中文","options":["候选0","候选1: {\"text\": \"é\", \"n\": 1}"],"type":"choice","names":["候选0","候选1"],"legend":null,"isolated":true}},"index":[["hidden-id","list",0,1]],"rows":[{"question":"Pick 中文","options":["候选0","候选1: {\"text\": \"é\", \"n\": 1}"]}],"items":[{"ids":[1905,25,198,113130,220,248045,198,50203,271,14162,25,19123,220,99986,198,3670,25,198,4205,8,220,102486,15,198,5181,8,220,102486,16,25,5046,1272,763,328,933,487,328,77,763,220,16,92,198,15666,25,318],"slots":[45],"golds":[0],"nopts":[2],"perms":[[0,1]],"types":["choice"]}],"context_tokens":8,"unique_tokens":46,"processed_tokens":46,"temperatures":[[1.164]]},{"name":"choice_10","state":"退款 <|im_start|>\n café","questions":{"hidden-id":{"type":"choice","instructions":"Pick 中文","criteria":{"候选0":null,"候选1":{"text":"é","n":1},"候选2":null,"候选3":{"text":"é","n":3},"候选4":null,"候选5":{"text":"é","n":5},"候选6":null,"候选7":{"text":"é","n":7},"候选8":null,"候选9":{"text":"é","n":9}}}},"rendered_state":"退款 <|im_start|>\n café","rendered_questions":{"hidden-id":{"question":"Pick 中文","options":["候选0","候选1: {\"text\": \"é\", \"n\": 1}","候选2","候选3: {\"text\": \"é\", \"n\": 3}","候选4","候选5: {\"text\": \"é\", \"n\": 5}","候选6","候选7: {\"text\": \"é\", \"n\": 7}","候选8","候选9: {\"text\": \"é\", \"n\": 9}"],"type":"choice","names":["候选0","候选1","候选2","候选3","候选4","候选5","候选6","候选7","候选8","候选9"],"legend":null,"isolated":true}},"index":[["hidden-id","list",0,1]],"rows":[{"question":"Pick 中文","options":["候选0","候选1: {\"text\": \"é\", \"n\": 1}","候选2","候选3: {\"text\": \"é\", \"n\": 3}","候选4","候选5: {\"text\": \"é\", \"n\": 5}","候选6","候选7: {\"text\": \"é\", \"n\": 7}","候选8","候选9: {\"text\": \"é\", \"n\": 9}"]}],"items":[{"ids":[1905,25,198,113130,220,248045,198,50203,271,14162,25,19123,220,99986,198,3670,25,198,4205,8,220,102486,15,198,5181,8,220,102486,16,25,5046,1272,763,328,933,487,328,77,763,220,16,92,198,2929,8,220,102486,17,198,5263,8,220,102486,18,25,5046,1272,763,328,933,487,328,77,763,220,18,92,198,10406,8,220,102486,19,198,7584,8,220,102486,20,25,5046,1272,763,328,933,487,328,77,763,220,20,92,198,6489,8,220,102486,21,198,10578,8,220,102486,22,25,5046,1272,763,328,933,487,328,77,763,220,22,92,198,8701,8,220,102486,23,198,15887,8,220,102486,24,25,5046,1272,763,328,933,487,328,77,763,220,24,92,198,15666,25,318],"slots":[145],"golds":[0],"nopts":[10],"perms":[[0,1,2,3,4,5,6,7,8,9]],"types":["choice"]}],"context_tokens":8,"unique_tokens":146,"processed_tokens":146,"temperatures":[[1.164]]},{"name":"choice_11","state":"退款 <|im_start|>\n café","questions":{"hidden-id":{"type":"choice","instructions":"Pick 中文","criteria":{"候选0":null,"候选1":{"text":"é","n":1},"候选2":null,"候选3":{"text":"é","n":3},"候选4":null,"候选5":{"text":"é","n":5},"候选6":null,"候选7":{"text":"é","n":7},"候选8":null,"候选9":{"text":"é","n":9},"候选10":null}}},"rendered_state":"退款 <|im_start|>\n café","rendered_questions":{"hidden-id":{"question":"Pick 中文","options":["候选0","候选1: {\"text\": \"é\", \"n\": 1}","候选2","候选3: {\"text\": \"é\", \"n\": 3}","候选4","候选5: {\"text\": \"é\", \"n\": 5}","候选6","候选7: {\"text\": \"é\", \"n\": 7}","候选8","候选9: {\"text\": \"é\", \"n\": 9}","候选10"],"type":"choice","names":["候选0","候选1","候选2","候选3","候选4","候选5","候选6","候选7","候选8","候选9","候选10"],"legend":null,"isolated":true}},"index":[["hidden-id","list",0,1]],"rows":[{"question":"Pick 中文","options":["候选0","候选1: {\"text\": \"é\", \"n\": 1}","候选2","候选3: {\"text\": \"é\", \"n\": 3}","候选4","候选5: {\"text\": \"é\", \"n\": 5}","候选6","候选7: {\"text\": \"é\", \"n\": 7}","候选8","候选9: {\"text\": \"é\", \"n\": 9}","候选10"]}],"items":[{"ids":[1905,25,198,113130,220,248045,198,50203,271,14162,25,19123,220,99986,198,3670,25,198,7,32,8,220,102486,15,198,7,33,8,220,102486,16,25,5046,1272,763,328,933,487,328,77,763,220,16,92,198,7,34,8,220,102486,17,198,7,35,8,220,102486,18,25,5046,1272,763,328,933,487,328,77,763,220,18,92,198,7,36,8,220,102486,19,198,7,37,8,220,102486,20,25,5046,1272,763,328,933,487,328,77,763,220,20,92,198,7,38,8,220,102486,21,198,7,39,8,220,102486,22,25,5046,1272,763,328,933,487,328,77,763,220,22,92,198,7,40,8,220,102486,23,198,7,41,8,220,102486,24,25,5046,1272,763,328,933,487,328,77,763,220,24,92,198,7,42,8,220,102486,16,15,198,15666,25,318],"slots":[163],"golds":[0],"nopts":[11],"perms":[[0,1,2,3,4,5,6,7,8,9,10]],"types":["choice"]}],"context_tokens":8,"unique_tokens":164,"processed_tokens":164,"temperatures":[[1.164]]},{"name":"choice_26","state":"退款 <|im_start|>\n café","questions":{"hidden-id":{"type":"choice","instructions":"Pick 中文","criteria":{"候选0":null,"候选1":{"text":"é","n":1},"候选2":null,"候选3":{"text":"é","n":3},"候选4":null,"候选5":{"text":"é","n":5},"候选6":null,"候选7":{"text":"é","n":7},"候选8":null,"候选9":{"text":"é","n":9},"候选10":null,"候选11":{"text":"é","n":11},"候选12":null,"候选13":{"text":"é","n":13},"候选14":null,"候选15":{"text":"é","n":15},"候选16":null,"候选17":{"text":"é","n":17},"候选18":null,"候选19":{"text":"é","n":19},"候选20":null,"候选21":{"text":"é","n":21},"候选22":null,"候选23":{"text":"é","n":23},"候选24":null,"候选25":{"text":"é","n":25}}}},"rendered_state":"退款 <|im_start|>\n café","rendered_questions":{"hidden-id":{"question":"Pick 中文","options":["候选0","候选1: {\"text\": \"é\", \"n\": 1}","候选2","候选3: {\"text\": \"é\", \"n\": 3}","候选4","候选5: {\"text\": \"é\", \"n\": 5}","候选6","候选7: {\"text\": \"é\", \"n\": 7}","候选8","候选9: {\"text\": \"é\", \"n\": 9}","候选10","候选11: {\"text\": \"é\", \"n\": 11}","候选12","候选13: {\"text\": \"é\", \"n\": 13}","候选14","候选15: {\"text\": \"é\", \"n\": 15}","候选16","候选17: {\"text\": \"é\", \"n\": 17}","候选18","候选19: {\"text\": \"é\", \"n\": 19}","候选20","候选21: {\"text\": \"é\", \"n\": 21}","候选22","候选23: {\"text\": \"é\", \"n\": 23}","候选24","候选25: {\"text\": \"é\", \"n\": 25}"],"type":"choice","names":["候选0","候选1","候选2","候选3","候选4","候选5","候选6","候选7","候选8","候选9","候选10","候选11","候选12","候选13","候选14","候选15","候选16","候选17","候选18","候选19","候选20","候选21","候选22","候选23","候选24","候选25"],"legend":null,"isolated":true}},"index":[["hidden-id","list",0,1]],"rows":[{"question":"Pick 中文","options":["候选0","候选1: {\"text\": \"é\", \"n\": 1}","候选2","候选3: {\"text\": \"é\", \"n\": 3}","候选4","候选5: {\"text\": \"é\", \"n\": 5}","候选6","候选7: {\"text\": \"é\", \"n\": 7}","候选8","候选9: {\"text\": \"é\", \"n\": 9}","候选10","候选11: {\"text\": \"é\", \"n\": 11}","候选12","候选13: {\"text\": \"é\", \"n\": 13}","候选14","候选15: {\"text\": \"é\", \"n\": 15}","候选16","候选17: {\"text\": \"é\", \"n\": 17}","候选18","候选19: {\"text\": \"é\", \"n\": 19}","候选20","候选21: {\"text\": \"é\", \"n\": 21}","候选22","候选23: {\"text\": \"é\", \"n\": 23}","候选24","候选25: {\"text\": \"é\", \"n\": 25}"]}],"items":[{"ids":[1905,25,198,113130,220,248045,198,50203,271,14162,25,19123,220,99986,198,3670,25,198,7,32,8,220,102486,15,198,7,33,8,220,102486,16,25,5046,1272,763,328,933,487,328,77,763,220,16,92,198,7,34,8,220,102486,17,198,7,35,8,220,102486,18,25,5046,1272,763,328,933,487,328,77,763,220,18,92,198,7,36,8,220,102486,19,198,7,37,8,220,102486,20,25,5046,1272,763,328,933,487,328,77,763,220,20,92,198,7,38,8,220,102486,21,198,7,39,8,220,102486,22,25,5046,1272,763,328,933,487,328,77,763,220,22,92,198,7,40,8,220,102486,23,198,7,41,8,220,102486,24,25,5046,1272,763,328,933,487,328,77,763,220,24,92,198,7,42,8,220,102486,16,15,198,7,43,8,220,102486,16,16,25,5046,1272,763,328,933,487,328,77,763,220,16,16,92,198,7,44,8,220,102486,16,17,198,7,45,8,220,102486,16,18,25,5046,1272,763,328,933,487,328,77,763,220,16,18,92,198,7,46,8,220,102486,16,19,198,7,47,8,220,102486,16,20,25,5046,1272,763,328,933,487,328,77,763,220,16,20,92,198,7,48,8,220,102486,16,21,198,7,49,8,220,102486,16,22,25,5046,1272,763,328,933,487,328,77,763,220,16,22,92,198,7,50,8,220,102486,16,23,198,7,51,8,220,102486,16,24,25,5046,1272,763,328,933,487,328,77,763,220,16,24,92,198,7,52,8,220,102486,17,15,198,7,53,8,220,102486,17,16,25,5046,1272,763,328,933,487,328,77,763,220,17,16,92,198,7,54,8,220,102486,17,17,198,7,55,8,220,102486,17,18,25,5046,1272,763,328,933,487,328,77,763,220,17,18,92,198,7,56,8,220,102486,17,19,198,7,57,8,220,102486,17,20,25,5046,1272,763,328,933,487,328,77,763,220,17,20,92,198,15666,25,318],"slots":[395],"golds":[0],"nopts":[26],"perms":[[0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25]],"types":["choice"]}],"context_tokens":8,"unique_tokens":396,"processed_tokens":396,"temperatures":[[1.164]]},{"name":"choice_255","state":"退款 <|im_start|>\n café","questions":{"hidden-id":{"type":"choice","instructions":"Pick 中文","criteria":{"候选0":null,"候选1":{"text":"é","n":1},"候选2":null,"候选3":{"text":"é","n":3},"候选4":null,"候选5":{"text":"é","n":5},"候选6":null,"候选7":{"text":"é","n":7},"候选8":null,"候选9":{"text":"é","n":9},"候选10":null,"候选11":{"text":"é","n":11},"候选12":null,"候选13":{"text":"é","n":13},"候选14":null,"候选15":{"text":"é","n":15},"候选16":null,"候选17":{"text":"é","n":17},"候选18":null,"候选19":{"text":"é","n":19},"候选20":null,"候选21":{"text":"é","n":21},"候选22":null,"候选23":{"text":"é","n":23},"候选24":null,"候选25":{"text":"é","n":25},"候选26":null,"候选27":{"text":"é","n":27},"候选28":null,"候选29":{"text":"é","n":29},"候选30":null,"候选31":{"text":"é","n":31},"候选32":null,"候选33":{"text":"é","n":33},"候选34":null,"候选35":{"text":"é","n":35},"候选36":null,"候选37":{"text":"é","n":37},"候选38":null,"候选39":{"text":"é","n":39},"候选40":null,"候选41":{"text":"é","n":41},"候选42":null,"候选43":{"text":"é","n":43},"候选44":null,"候选45":{"text":"é","n":45},"候选46":null,"候选47":{"text":"é","n":47},"候选48":null,"候选49":{"text":"é","n":49},"候选50":null,"候选51":{"text":"é","n":51},"候选52":null,"候选53":{"text":"é","n":53},"候选54":null,"候选55":{"text":"é","n":55},"候选56":null,"候选57":{"text":"é","n":57},"候选58":null,"候选59":{"text":"é","n":59},"候选60":null,"候选61":{"text":"é","n":61},"候选62":null,"候选63":{"text":"é","n":63},"候选64":null,"候选65":{"text":"é","n":65},"候选66":null,"候选67":{"text":"é","n":67},"候选68":null,"候选69":{"text":"é","n":69},"候选70":null,"候选71":{"text":"é","n":71},"候选72":null,"候选73":{"text":"é","n":73},"候选74":null,"候选75":{"text":"é","n":75},"候选76":null,"候选77":{"text":"é","n":77},"候选78":null,"候选79":{"text":"é","n":79},"候选80":null,"候选81":{"text":"é","n":81},"候选82":null,"候选83":{"text":"é","n":83},"候选84":null,"候选85":{"text":"é","n":85},"候选86":null,"候选87":{"text":"é","n":87},"候选88":null,"候选89":{"text":"é","n":89},"候选90":null,"候选91":{"text":"é","n":91},"候选92":null,"候选93":{"text":"é","n":93},"候选94":null,"候选95":{"text":"é","n":95},"候选96":null,"候选97":{"text":"é","n":97},"候选98":null,"候选99":{"text":"é","n":99},"候选100":null,"候选101":{"text":"é","n":101},"候选102":null,"候选103":{"text":"é","n":103},"候选104":null,"候选105":{"text":"é","n":105},"候选106":null,"候选107":{"text":"é","n":107},"候选108":null,"候选109":{"text":"é","n":109},"候选110":null,"候选111":{"text":"é","n":111},"候选112":null,"候选113":{"text":"é","n":113},"候选114":null,"候选115":{"text":"é","n":115},"候选116":null,"候选117":{"text":"é","n":117},"候选118":null,"候选119":{"text":"é","n":119},"候选120":null,"候选121":{"text":"é","n":121},"候选122":null,"候选123":{"text":"é","n":123},"候选124":null,"候选125":{"text":"é","n":125},"候选126":null,"候选127":{"text":"é","n":127},"候选128":null,"候选129":{"text":"é","n":129},"候选130":null,"候选131":{"text":"é","n":131},"候选132":null,"候选133":{"text":"é","n":133},"候选134":null,"候选135":{"text":"é","n":135},"候选136":null,"候选137":{"text":"é","n":137},"候选138":null,"候选139":{"text":"é","n":139},"候选140":null,"候选141":{"text":"é","n":141},"候选142":null,"候选143":{"text":"é","n":143},"候选144":null,"候选145":{"text":"é","n":145},"候选146":null,"候选147":{"text":"é","n":147},"候选148":null,"候选149":{"text":"é","n":149},"候选150":null,"候选151":{"text":"é","n":151},"候选152":null,"候选153":{"text":"é","n":153},"候选154":null,"候选155":{"text":"é","n":155},"候选156":null,"候选157":{"text":"é","n":157},"候选158":null,"候选159":{"text":"é","n":159},"候选160":null,"候选161":{"text":"é","n":161},"候选162":null,"候选163":{"text":"é","n":163},"候选164":null,"候选165":{"text":"é","n":165},"候选166":null,"候选167":{"text":"é","n":167},"候选168":null,"候选169":{"text":"é","n":169},"候选170":null,"候选171":{"text":"é","n":171},"候选172":null,"候选173":{"text":"é","n":173},"候选174":null,"候选175":{"text":"é","n":175},"候选176":null,"候选177":{"text":"é","n":177},"候选178":null,"候选179":{"text":"é","n":179},"候选180":null,"候选181":{"text":"é","n":181},"候选182":null,"候选183":{"text":"é","n":183},"候选184":null,"候选185":{"text":"é","n":185},"候选186":null,"候选187":{"text":"é","n":187},"候选188":null,"候选189":{"text":"é","n":189},"候选190":null,"候选191":{"text":"é","n":191},"候选192":null,"候选193":{"text":"é","n":193},"候选194":null,"候选195":{"text":"é","n":195},"候选196":null,"候选197":{"text":"é","n":197},"候选198":null,"候选199":{"text":"é","n":199},"候选200":null,"候选201":{"text":"é","n":201},"候选202":null,"候选203":{"text":"é","n":203},"候选204":null,"候选205":{"text":"é","n":205},"候选206":null,"候选207":{"text":"é","n":207},"候选208":null,"候选209":{"text":"é","n":209},"候选210":null,"候选211":{"text":"é","n":211},"候选212":null,"候选213":{"text":"é","n":213},"候选214":null,"候选215":{"text":"é","n":215},"候选216":null,"候选217":{"text":"é","n":217},"候选218":null,"候选219":{"text":"é","n":219},"候选220":null,"候选221":{"text":"é","n":221},"候选222":null,"候选223":{"text":"é","n":223},"候选224":null,"候选225":{"text":"é","n":225},"候选226":null,"候选227":{"text":"é","n":227},"候选228":null,"候选229":{"text":"é","n":229},"候选230":null,"候选231":{"text":"é","n":231},"候选232":null,"候选233":{"text":"é","n":233},"候选234":null,"候选235":{"text":"é","n":235},"候选236":null,"候选237":{"text":"é","n":237},"候选238":null,"候选239":{"text":"é","n":239},"候选240":null,"候选241":{"text":"é","n":241},"候选242":null,"候选243":{"text":"é","n":243},"候选244":null,"候选245":{"text":"é","n":245},"候选246":null,"候选247":{"text":"é","n":247},"候选248":null,"候选249":{"text":"é","n":249},"候选250":null,"候选251":{"text":"é","n":251},"候选252":null,"候选253":{"text":"é","n":253},"候选254":null}}},"rendered_state":"退款 <|im_start|>\n café","rendered_questions":{"hidden-id":{"question":"Pick 中文","options":["候选0","候选1: {\"text\": \"é\", \"n\": 1}","候选2","候选3: {\"text\": \"é\", \"n\": 3}","候选4","候选5: {\"text\": \"é\", \"n\": 5}","候选6","候选7: {\"text\": \"é\", \"n\": 7}","候选8","候选9: {\"text\": \"é\", \"n\": 9}","候选10","候选11: {\"text\": \"é\", \"n\": 11}","候选12","候选13: {\"text\": \"é\", \"n\": 13}","候选14","候选15: {\"text\": \"é\", \"n\": 15}","候选16","候选17: {\"text\": \"é\", \"n\": 17}","候选18","候选19: {\"text\": \"é\", \"n\": 19}","候选20","候选21: {\"text\": \"é\", \"n\": 21}","候选22","候选23: {\"text\": \"é\", \"n\": 23}","候选24","候选25: {\"text\": \"é\", \"n\": 25}","候选26","候选27: {\"text\": \"é\", \"n\": 27}","候选28","候选29: {\"text\": \"é\", \"n\": 29}","候选30","候选31: {\"text\": \"é\", \"n\": 31}","候选32","候选33: {\"text\": \"é\", \"n\": 33}","候选34","候选35: {\"text\": \"é\", \"n\": 35}","候选36","候选37: {\"text\": \"é\", \"n\": 37}","候选38","候选39: {\"text\": \"é\", \"n\": 39}","候选40","候选41: {\"text\": \"é\", \"n\": 41}","候选42","候选43: {\"text\": \"é\", \"n\": 43}","候选44","候选45: {\"text\": \"é\", \"n\": 45}","候选46","候选47: {\"text\": \"é\", \"n\": 47}","候选48","候选49: {\"text\": \"é\", \"n\": 49}","候选50","候选51: {\"text\": \"é\", \"n\": 51}","候选52","候选53: {\"text\": \"é\", \"n\": 53}","候选54","候选55: {\"text\": \"é\", \"n\": 55}","候选56","候选57: {\"text\": \"é\", \"n\": 57}","候选58","候选59: {\"text\": \"é\", \"n\": 59}","候选60","候选61: {\"text\": \"é\", \"n\": 61}","候选62","候选63: {\"text\": \"é\", \"n\": 63}","候选64","候选65: {\"text\": \"é\", \"n\": 65}","候选66","候选67: {\"text\": \"é\", \"n\": 67}","候选68","候选69: {\"text\": \"é\", \"n\": 69}","候选70","候选71: {\"text\": \"é\", \"n\": 71}","候选72","候选73: {\"text\": \"é\", \"n\": 73}","候选74","候选75: {\"text\": \"é\", \"n\": 75}","候选76","候选77: {\"text\": \"é\", \"n\": 77}","候选78","候选79: {\"text\": \"é\", \"n\": 79}","候选80","候选81: {\"text\": \"é\", \"n\": 81}","候选82","候选83: {\"text\": \"é\", \"n\": 83}","候选84","候选85: {\"text\": \"é\", \"n\": 85}","候选86","候选87: {\"text\": \"é\", \"n\": 87}","候选88","候选89: {\"text\": \"é\", \"n\": 89}","候选90","候选91: {\"text\": \"é\", \"n\": 91}","候选92","候选93: {\"text\": \"é\", \"n\": 93}","候选94","候选95: {\"text\": \"é\", \"n\": 95}","候选96","候选97: {\"text\": \"é\", \"n\": 97}","候选98","候选99: {\"text\": \"é\", \"n\": 99}","候选100","候选101: {\"text\": \"é\", \"n\": 101}","候选102","候选103: {\"text\": \"é\", \"n\": 103}","候选104","候选105: {\"text\": \"é\", \"n\": 105}","候选106","候选107: {\"text\": \"é\", \"n\": 107}","候选108","候选109: {\"text\": \"é\", \"n\": 109}","候选110","候选111: {\"text\": \"é\", \"n\": 111}","候选112","候选113: {\"text\": \"é\", \"n\": 113}","候选114","候选115: {\"text\": \"é\", \"n\": 115}","候选116","候选117: {\"text\": \"é\", \"n\": 117}","候选118","候选119: {\"text\": \"é\", \"n\": 119}","候选120","候选121: {\"text\": \"é\", \"n\": 121}","候选122","候选123: {\"text\": \"é\", \"n\": 123}","候选124","候选125: {\"text\": \"é\", \"n\": 125}","候选126","候选127: {\"text\": \"é\", \"n\": 127}","候选128","候选129: {\"text\": \"é\", \"n\": 129}","候选130","候选131: {\"text\": \"é\", \"n\": 131}","候选132","候选133: {\"text\": \"é\", \"n\": 133}","候选134","候选135: {\"text\": \"é\", \"n\": 135}","候选136","候选137: {\"text\": \"é\", \"n\": 137}","候选138","候选139: {\"text\": \"é\", \"n\": 139}","候选140","候选141: {\"text\": \"é\", \"n\": 141}","候选142","候选143: {\"text\": \"é\", \"n\": 143}","候选144","候选145: {\"text\": \"é\", \"n\": 145}","候选146","候选147: {\"text\": \"é\", \"n\": 147}","候选148","候选149: {\"text\": \"é\", \"n\": 149}","候选150","候选151: {\"text\": \"é\", \"n\": 151}","候选152","候选153: {\"text\": \"é\", \"n\": 153}","候选154","候选155: {\"text\": \"é\", \"n\": 155}","候选156","候选157: {\"text\": \"é\", \"n\": 157}","候选158","候选159: {\"text\": \"é\", \"n\": 159}","候选160","候选161: {\"text\": \"é\", \"n\": 161}","候选162","候选163: {\"text\": \"é\", \"n\": 163}","候选164","候选165: {\"text\": \"é\", \"n\": 165}","候选166","候选167: {\"text\": \"é\", \"n\": 167}","候选168","候选169: {\"text\": \"é\", \"n\": 169}","候选170","候选171: {\"text\": \"é\", \"n\": 171}","候选172","候选173: {\"text\": \"é\", \"n\": 173}","候选174","候选175: {\"text\": \"é\", \"n\": 175}","候选176","候选177: {\"text\": \"é\", \"n\": 177}","候选178","候选179: {\"text\": \"é\", \"n\": 179}","候选180","候选181: {\"text\": \"é\", \"n\": 181}","候选182","候选183: {\"text\": \"é\", \"n\": 183}","候选184","候选185: {\"text\": \"é\", \"n\": 185}","候选186","候选187: {\"text\": \"é\", \"n\": 187}","候选188","候选189: {\"text\": \"é\", \"n\": 189}","候选190","候选191: {\"text\": \"é\", \"n\": 191}","候选192","候选193: {\"text\": \"é\", \"n\": 193}","候选194","候选195: {\"text\": \"é\", \"n\": 195}","候选196","候选197: {\"text\": \"é\", \"n\": 197}","候选198","候选199: {\"text\": \"é\", \"n\": 199}","候选200","候选201: {\"text\": \"é\", \"n\": 201}","候选202","候选203: {\"text\": \"é\", \"n\": 203}","候选204","候选205: {\"text\": \"é\", \"n\": 205}","候选206","候选207: {\"text\": \"é\", \"n\": 207}","候选208","候选209: {\"text\": \"é\", \"n\": 209}","候选210","候选211: {\"text\": \"é\", \"n\": 211}","候选212","候选213: {\"text\": \"é\", \"n\": 213}","候选214","候选215: {\"text\": \"é\", \"n\": 215}","候选216","候选217: {\"text\": \"é\", \"n\": 217}","候选218","候选219: {\"text\": \"é\", \"n\": 219}","候选220","候选221: {\"text\": \"é\", \"n\": 221}","候选222","候选223: {\"text\": \"é\", \"n\": 223}","候选224","候选225: {\"text\": \"é\", \"n\": 225}","候选226","候选227: {\"text\": \"é\", \"n\": 227}","候选228","候选229: {\"text\": \"é\", \"n\": 229}","候选230","候选231: {\"text\": \"é\", \"n\": 231}","候选232","候选233: {\"text\": \"é\", \"n\": 233}","候选234","候选235: {\"text\": \"é\", \"n\": 235}","候选236","候选237: {\"text\": \"é\", \"n\": 237}","候选238","候选239: {\"text\": \"é\", \"n\": 239}","候选240","候选241: {\"text\": \"é\", \"n\": 241}","候选242","候选243: {\"text\": \"é\", \"n\": 243}","候选244","候选245: {\"text\": \"é\", \"n\": 245}","候选246","候选247: {\"text\": \"é\", \"n\": 247}","候选248","候选249: {\"text\": \"é\", \"n\": 249}","候选250","候选251: {\"text\": \"é\", \"n\": 251}","候选252","候选253: {\"text\": \"é\", \"n\": 253}","候选254"],"type":"choice","names":["候选0","候选1","候选2","候选3","候选4","候选5","候选6","候选7","候选8","候选9","候选10","候选11","候选12","候选13","候选14","候选15","候选16","候选17","候选18","候选19","候选20","候选21","候选22","候选23","候选24","候选25","候选26","候选27","候选28","候选29","候选30","候选31","候选32","候选33","候选34","候选35","候选36","候选37","候选38","候选39","候选40","候选41","候选42","候选43","候选44","候选45","候选46","候选47","候选48","候选49","候选50","候选51","候选52","候选53","候选54","候选55","候选56","候选57","候选58","候选59","候选60","候选61","候选62","候选63","候选64","候选65","候选66","候选67","候选68","候选69","候选70","候选71","候选72","候选73","候选74","候选75","候选76","候选77","候选78","候选79","候选80","候选81","候选82","候选83","候选84","候选85","候选86","候选87","候选88","候选89","候选90","候选91","候选92","候选93","候选94","候选95","候选96","候选97","候选98","候选99","候选100","候选101","候选102","候选103","候选104","候选105","候选106","候选107","候选108","候选109","候选110","候选111","候选112","候选113","候选114","候选115","候选116","候选117","候选118","候选119","候选120","候选121","候选122","候选123","候选124","候选125","候选126","候选127","候选128","候选129","候选130","候选131","候选132","候选133","候选134","候选135","候选136","候选137","候选138","候选139","候选140","候选141","候选142","候选143","候选144","候选145","候选146","候选147","候选148","候选149","候选150","候选151","候选152","候选153","候选154","候选155","候选156","候选157","候选158","候选159","候选160","候选161","候选162","候选163","候选164","候选165","候选166","候选167","候选168","候选169","候选170","候选171","候选172","候选173","候选174","候选175","候选176","候选177","候选178","候选179","候选180","候选181","候选182","候选183","候选184","候选185","候选186","候选187","候选188","候选189","候选190","候选191","候选192","候选193","候选194","候选195","候选196","候选197","候选198","候选199","候选200","候选201","候选202","候选203","候选204","候选205","候选206","候选207","候选208","候选209","候选210","候选211","候选212","候选213","候选214","候选215","候选216","候选217","候选218","候选219","候选220","候选221","候选222","候选223","候选224","候选225","候选226","候选227","候选228","候选229","候选230","候选231","候选232","候选233","候选234","候选235","候选236","候选237","候选238","候选239","候选240","候选241","候选242","候选243","候选244","候选245","候选246","候选247","候选248","候选249","候选250","候选251","候选252","候选253","候选254"],"legend":null,"isolated":true}},"index":[["hidden-id","list",0,1]],"rows":[{"question":"Pick 中文","options":["候选0","候选1: {\"text\": \"é\", \"n\": 1}","候选2","候选3: {\"text\": \"é\", \"n\": 3}","候选4","候选5: {\"text\": \"é\", \"n\": 5}","候选6","候选7: {\"text\": \"é\", \"n\": 7}","候选8","候选9: {\"text\": \"é\", \"n\": 9}","候选10","候选11: {\"text\": \"é\", \"n\": 11}","候选12","候选13: {\"text\": \"é\", \"n\": 13}","候选14","候选15: {\"text\": \"é\", \"n\": 15}","候选16","候选17: {\"text\": \"é\", \"n\": 17}","候选18","候选19: {\"text\": \"é\", \"n\": 19}","候选20","候选21: {\"text\": \"é\", \"n\": 21}","候选22","候选23: {\"text\": \"é\", \"n\": 23}","候选24","候选25: {\"text\": \"é\", \"n\": 25}","候选26","候选27: {\"text\": \"é\", \"n\": 27}","候选28","候选29: {\"text\": \"é\", \"n\": 29}","候选30","候选31: {\"text\": \"é\", \"n\": 31}","候选32","候选33: {\"text\": \"é\", \"n\": 33}","候选34","候选35: {\"text\": \"é\", \"n\": 35}","候选36","候选37: {\"text\": \"é\", \"n\": 37}","候选38","候选39: {\"text\": \"é\", \"n\": 39}","候选40","候选41: {\"text\": \"é\", \"n\": 41}","候选42","候选43: {\"text\": \"é\", \"n\": 43}","候选44","候选45: {\"text\": \"é\", \"n\": 45}","候选46","候选47: {\"text\": \"é\", \"n\": 47}","候选48","候选49: {\"text\": \"é\", \"n\": 49}","候选50","候选51: {\"text\": \"é\", \"n\": 51}","候选52","候选53: {\"text\": \"é\", \"n\": 53}","候选54","候选55: {\"text\": \"é\", \"n\": 55}","候选56","候选57: {\"text\": \"é\", \"n\": 57}","候选58","候选59: {\"text\": \"é\", \"n\": 59}","候选60","候选61: {\"text\": \"é\", \"n\": 61}","候选62","候选63: {\"text\": \"é\", \"n\": 63}","候选64","候选65: {\"text\": \"é\", \"n\": 65}","候选66","候选67: {\"text\": \"é\", \"n\": 67}","候选68","候选69: {\"text\": \"é\", \"n\": 69}","候选70","候选71: {\"text\": \"é\", \"n\": 71}","候选72","候选73: {\"text\": \"é\", \"n\": 73}","候选74","候选75: {\"text\": \"é\", \"n\": 75}","候选76","候选77: {\"text\": \"é\", \"n\": 77}","候选78","候选79: {\"text\": \"é\", \"n\": 79}","候选80","候选81: {\"text\": \"é\", \"n\": 81}","候选82","候选83: {\"text\": \"é\", \"n\": 83}","候选84","候选85: {\"text\": \"é\", \"n\": 85}","候选86","候选87: {\"text\": \"é\", \"n\": 87}","候选88","候选89: {\"text\": \"é\", \"n\": 89}","候选90","候选91: {\"text\": \"é\", \"n\": 91}","候选92","候选93: {\"text\": \"é\", \"n\": 93}","候选94","候选95: {\"text\": \"é\", \"n\": 95}","候选96","候选97: {\"text\": \"é\", \"n\": 97}","候选98","候选99: {\"text\": \"é\", \"n\": 99}","候选100","候选101: {\"text\": \"é\", \"n\": 101}","候选102","候选103: {\"text\": \"é\", \"n\": 103}","候选104","候选105: {\"text\": \"é\", \"n\": 105}","候选106","候选107: {\"text\": \"é\", \"n\": 107}","候选108","候选109: {\"text\": \"é\", \"n\": 109}","候选110","候选111: {\"text\": \"é\", \"n\": 111}","候选112","候选113: {\"text\": \"é\", \"n\": 113}","候选114","候选115: {\"text\": \"é\", \"n\": 115}","候选116","候选117: {\"text\": \"é\", \"n\": 117}","候选118","候选119: {\"text\": \"é\", \"n\": 119}","候选120","候选121: {\"text\": \"é\", \"n\": 121}","候选122","候选123: {\"text\": \"é\", \"n\": 123}","候选124","候选125: {\"text\": \"é\", \"n\": 125}","候选126","候选127: {\"text\": \"é\", \"n\": 127}","候选128","候选129: {\"text\": \"é\", \"n\": 129}","候选130","候选131: {\"text\": \"é\", \"n\": 131}","候选132","候选133: {\"text\": \"é\", \"n\": 133}","候选134","候选135: {\"text\": \"é\", \"n\": 135}","候选136","候选137: {\"text\": \"é\", \"n\": 137}","候选138","候选139: {\"text\": \"é\", \"n\": 139}","候选140","候选141: {\"text\": \"é\", \"n\": 141}","候选142","候选143: {\"text\": \"é\", \"n\": 143}","候选144","候选145: {\"text\": \"é\", \"n\": 145}","候选146","候选147: {\"text\": \"é\", \"n\": 147}","候选148","候选149: {\"text\": \"é\", \"n\": 149}","候选150","候选151: {\"text\": \"é\", \"n\": 151}","候选152","候选153: {\"text\": \"é\", \"n\": 153}","候选154","候选155: {\"text\": \"é\", \"n\": 155}","候选156","候选157: {\"text\": \"é\", \"n\": 157}","候选158","候选159: {\"text\": \"é\", \"n\": 159}","候选160","候选161: {\"text\": \"é\", \"n\": 161}","候选162","候选163: {\"text\": \"é\", \"n\": 163}","候选164","候选165: {\"text\": \"é\", \"n\": 165}","候选166","候选167: {\"text\": \"é\", \"n\": 167}","候选168","候选169: {\"text\": \"é\", \"n\": 169}","候选170","候选171: {\"text\": \"é\", \"n\": 171}","候选172","候选173: {\"text\": \"é\", \"n\": 173}","候选174","候选175: {\"text\": \"é\", \"n\": 175}","候选176","候选177: {\"text\": \"é\", \"n\": 177}","候选178","候选179: {\"text\": \"é\", \"n\": 179}","候选180","候选181: {\"text\": \"é\", \"n\": 181}","候选182","候选183: {\"text\": \"é\", \"n\": 183}","候选184","候选185: {\"text\": \"é\", \"n\": 185}","候选186","候选187: {\"text\": \"é\", \"n\": 187}","候选188","候选189: {\"text\": \"é\", \"n\": 189}","候选190","候选191: {\"text\": \"é\", \"n\": 191}","候选192","候选193: {\"text\": \"é\", \"n\": 193}","候选194","候选195: {\"text\": \"é\", \"n\": 195}","候选196","候选197: {\"text\": \"é\", \"n\": 197}","候选198","候选199: {\"text\": \"é\", \"n\": 199}","候选200","候选201: {\"text\": \"é\", \"n\": 201}","候选202","候选203: {\"text\": \"é\", \"n\": 203}","候选204","候选205: {\"text\": \"é\", \"n\": 205}","候选206","候选207: {\"text\": \"é\", \"n\": 207}","候选208","候选209: {\"text\": \"é\", \"n\": 209}","候选210","候选211: {\"text\": \"é\", \"n\": 211}","候选212","候选213: {\"text\": \"é\", \"n\": 213}","候选214","候选215: {\"text\": \"é\", \"n\": 215}","候选216","候选217: {\"text\": \"é\", \"n\": 217}","候选218","候选219: {\"text\": \"é\", \"n\": 219}","候选220","候选221: {\"text\": \"é\", \"n\": 221}","候选222","候选223: {\"text\": \"é\", \"n\": 223}","候选224","候选225: {\"text\": \"é\", \"n\": 225}","候选226","候选227: {\"text\": \"é\", \"n\": 227}","候选228","候选229: {\"text\": \"é\", \"n\": 229}","候选230","候选231: {\"text\": \"é\", \"n\": 231}","候选232","候选233: {\"text\": \"é\", \"n\": 233}","候选234","候选235: {\"text\": \"é\", \"n\": 235}","候选236","候选237: {\"text\": \"é\", \"n\": 237}","候选238","候选239: {\"text\": \"é\", \"n\": 239}","候选240","候选241: {\"text\": \"é\", \"n\": 241}","候选242","候选243: {\"text\": \"é\", \"n\": 243}","候选244","候选245: {\"text\": \"é\", \"n\": 245}","候选246","候选247: {\"text\": \"é\", \"n\": 247}","候选248","候选249: {\"text\": \"é\", \"n\": 249}","候选250","候选251: {\"text\": \"é\", \"n\": 251}","候选252","候选253: {\"text\": \"é\", \"n\": 253}","候选254"]}],"items":[{"ids":[1905,25,198,113130,220,248045,198,50203,271,14162,25,19123,220,99986,198,3670,25,198,7,32,8,220,102486,15,198,7,33,8,220,102486,16,25,5046,1272,763,328,933,487,328,77,763,220,16,92,198,7,34,8,220,102486,17,198,7,35,8,220,102486,18,25,5046,1272,763,328,933,487,328,77,763,220,18,92,198,7,36,8,220,102486,19,198,7,37,8,220,102486,20,25,5046,1272,763,328,933,487,328,77,763,220,20,92,198,7,38,8,220,102486,21,198,7,39,8,220,102486,22,25,5046,1272,763,328,933,487,328,77,763,220,22,92,198,7,40,8,220,102486,23,198,7,41,8,220,102486,24,25,5046,1272,763,328,933,487,328,77,763,220,24,92,198,7,42,8,220,102486,16,15,198,7,43,8,220,102486,16,16,25,5046,1272,763,328,933,487,328,77,763,220,16,16,92,198,7,44,8,220,102486,16,17,198,7,45,8,220,102486,16,18,25,5046,1272,763,328,933,487,328,77,763,220,16,18,92,198,7,46,8,220,102486,16,19,198,7,47,8,220,102486,16,20,25,5046,1272,763,328,933,487,328,77,763,220,16,20,92,198,7,48,8,220,102486,16,21,198,7,49,8,220,102486,16,22,25,5046,1272,763,328,933,487,328,77,763,220,16,22,92,198,7,50,8,220,102486,16,23,198,7,51,8,220,102486,16,24,25,5046,1272,763,328,933,487,328,77,763,220,16,24,92,198,7,52,8,220,102486,17,15,198,7,53,8,220,102486,17,16,25,5046,1272,763,328,933,487,328,77,763,220,17,16,92,198,7,54,8,220,102486,17,17,198,7,55,8,220,102486,17,18,25,5046,1272,763,328,933,487,328,77,763,220,17,18,92,198,7,56,8,220,102486,17,19,198,7,57,8,220,102486,17,20,25,5046,1272,763,328,933,487,328,77,763,220,17,20,92,198,7,5840,8,220,102486,17,21,198,7,1803,8,220,102486,17,22,25,5046,1272,763,328,933,487,328,77,763,220,17,22,92,198,7,1646,8,220,102486,17,23,198,7,1745,8,220,102486,17,24,25,5046,1272,763,328,933,487,328,77,763,220,17,24,92,198,7,13276,8,220,102486,18,15,198,7,8018,8,220,102486,18,16,25,5046,1272,763,328,933,487,328,77,763,220,18,16,92,198,7,1825,8,220,102486,18,17,198,7,28946,8,220,102486,18,18,25,5046,1272,763,328,933,487,328,77,763,220,18,18,92,198,7,15015,8,220,102486,18,19,198,7,29595,8,220,102486,18,20,25,5046,1272,763,328,933,487,328,77,763,220,18,20,92,198,7,11568,8,220,102486,18,21,198,7,939,8,220,102486,18,22,25,5046,1272,763,328,933,487,328,77,763,220,18,22,92,198,7,1354,8,220,102486,18,23,198,7,1058,8,220,102486,18,24,25,5046,1272,763,328,933,487,328,77,763,220,18,24,92,198,7,18183,8,220,102486,19,15,198,7,2456,8,220,102486,19,16,25,5046,1272,763,328,933,487,328,77,763,220,19,16,92,198,7,88898,8,220,102486,19,17,198,7,905,8,220,102486,19,18,25,5046,1272,763,328,933,487,328,77,763,220,19,18,92,198,7,1846,8,220,102486,19,19,198,7,802,8,220,102486,19,20,25,5046,1272,763,328,933,487,328,77,763,220,19,20,92,198,7,33869,8,220,102486,19,21,198,7,7839,8,220,102486,19,22,25,5046,1272,763,328,933,487,328,77,763,220,19,22,92,198,7,14006,8,220,102486,19,23,198,7,2860,8,220,102486,19,24,25,5046,1272,763,328,933,487,328,77,763,220,19,24,92,198,7,2926,8,220,102486,20,15,198,7,22828,8,220,102486,20,16,25,5046,1272,763,328,933,487,328,77,763,220,20,16,92,198,7,6844,8,220,102486,20,17,198,7,9798,8,220,102486,20,18,25,5046,1272,763,328,933,487,328,77,763,220,20,18,92,198,7,4738,8,220,102486,20,19,198,7,9265,8,220,102486,20,20,25,5046,1272,763,328,933,487,328,77,763,220,20,20,92,198,7,11261,8,220,102486,20,21,198,7,19278,8,220,102486,20,22,25,5046,1272,763,328,933,487,328,77,763,220,20,22,92,198,7,36513,8,220,102486,20,23,198,7,93801,8,220,102486,20,24,25,5046,1272,763,328,933,487,328,77,763,220,20,24,92,198,7,8335,8,220,102486,21,15,198,7,14544,8,220,102486,21,16,25,5046,1272,763,328,933,487,328,77,763,220,21,16,92,198,7,85266,8,220,102486,21,17,198,7,9110,8,220,102486,21,18,25,5046,1272,763,328,933,487,328,77,763,220,21,18,92,198,7,28000,8,220,102486,21,19,198,7,15137,8,220,102486,21,20,25,5046,1272,763,328,933,487,328,77,763,220,21,20,92,198,7,4525,8,220,102486,21,21,198,7,25261,8,220,102486,21,22,25,5046,1272,763,328,933,487,328,77,763,220,21,22,92,198,7,12717,8,220,102486,21,23,198,7,7116,8,220,102486,21,24,25,5046,1272,763,328,933,487,328,77,763,220,21,24,92,198,7,17078,8,220,102486,22,15,198,7,14497,8,220,102486,22,16,25,5046,1272,763,328,933,487,328,77,763,220,22,16,92,198,7,57339,8,220,102486,22,17,198,7,74909,8,220,102486,22,18,25,5046,1272,763,328,933,487,328,77,763,220,22,18,92,198,7,52072,8,220,102486,22,19,198,7,19305,8,220,102486,22,20,25,5046,1272,763,328,933,487,328,77,763,220,22,20,92,198,7,4887,8,220,102486,22,21,198,7,12607,8,220,102486,22,22,25,5046,1272,763,328,933,487,328,77,763,220,22,22,92,198,7,3580,8,220,102486,22,23,198,7,6281,8,220,102486,22,24,25,5046,1272,763,328,933,487,328,77,763,220,22,24,92,198,7,2036,8,220,102486,23,15,198,7,9362,8,220,102486,23,16,25,5046,1272,763,328,933,487,328,77,763,220,23,16,92,198,7,8533,8,220,102486,23,17,198,7,2080,8,220,102486,23,18,25,5046,1272,763,328,933,487,328,77,763,220,23,18,92,198,7,10911,8,220,102486,23,19,198,7,2925,8,220,102486,23,20,25,5046,1272,763,328,933,487,328,77,763,220,23,20,92,198,7,3040,8,220,102486,23,21,198,7,9690,8,220,102486,23,22,25,5046,1272,763,328,933,487,328,77,763,220,23,22,92,198,7,27731,8,220,102486,23,23,198,7,8023,8,220,102486,23,24,25,5046,1272,763,328,933,487,328,77,763,220,23,24,92,198,7,6901,8,220,102486,24,15,198,7,8702,8,220,102486,24,16,25,5046,1272,763,328,933,487,328,77,763,220,24,16,92,198,7,6211,8,220,102486,24,17,198,7,1123,8,220,102486,24,18,25,5046,1272,763,328,933,487,328,77,763,220,24,18,92,198,7,16307,8,220,102486,24,19,198,7,18990,8,220,102486,24,20,25,5046,1272,763,328,933,487,328,77,763,220,24,20,92,198,7,64045,8,220,102486,24,21,198,7,63037,8,220,102486,24,22,25,5046,1272,763,328,933,487,328,77,763,220,24,22,92,198,7,33380,8,220,102486,24,23,198,7,6151,8,220,102486,24,24,25,5046,1272,763,328,933,487,328,77,763,220,24,24,92,198,7,3392,8,220,102486,16,15,15,198,7,5449,8,220,102486,16,15,16,25,5046,1272,763,328,933,487,328,77,763,220,16,15,16,92,198,7,3967,8,220,102486,16,15,17,198,7,1113,8,220,102486,16,15,18,25,5046,1272,763,328,933,487,328,77,763,220,16,15,18,92,198,7,5095,8,220,102486,16,15,19,198,7,51923,8,220,102486,16,15,20,25,5046,1272,763,328,933,487,328,77,763,220,16,15,20,92,198,7,49600,8,220,102486,16,15,21,198,7,17099,8,220,102486,16,15,22,25,5046,1272,763,328,933,487,328,77,763,220,16,15,22,92,198,7,51483,8,220,102486,16,15,23,198,7,17756,8,220,102486,16,15,24,25,5046,1272,763,328,933,487,328,77,763,220,16,15,24,92,198,7,16037,8,220,102486,16,16,15,198,7,8135,8,220,102486,16,16,16,25,5046,1272,763,328,933,487,328,77,763,220,16,16,16,92,198,7,30237,8,220,102486,16,16,17,198,7,5683,8,220,102486,16,16,18,25,5046,1272,763,328,933,487,328,77,763,220,16,16,18,92,198,7,9992,8,220,102486,16,16,19,198,7,7444,8,220,102486,16,16,20,25,5046,1272,763,328,933,487,328,77,763,220,16,16,20,92,198,7,5751,8,220,102486,16,16,21,198,7,10284,8,220,102486,16,16,22,25,5046,1272,763,328,933,487,328,77,763,220,16,16,22,92,198,7,20887,8,220,102486,16,16,23,198,7,59884,8,220,102486,16,16,24,25,5046,1272,763,328,933,487,328,77,763,220,16,16,24,92,198,7,52396,8,220,102486,16,17,15,198,7,16103,8,220,102486,16,17,16,25,5046,1272,763,328,933,487,328,77,763,220,16,17,16,92,198,7,67547,8,220,102486,16,17,17,198,7,18535,8,220,102486,16,17,18,25,5046,1272,763,328,933,487,328,77,763,220,16,17,18,92,198,7,8006,8,220,102486,16,17,19,198,7,7263,8,220,102486,16,17,20,25,5046,1272,763,328,933,487,328,77,763,220,16,17,20,92,198,7,1425,8,220,102486,16,17,21,198,7,6878,8,220,102486,16,17,22,25,5046,1272,763,328,933,487,328,77,763,220,16,17,22,92,198,7,14453,8,220,102486,16,17,23,198,7,9097,8,220,102486,16,17,24,25,5046,1272,763,328,933,487,328,77,763,220,16,17,24,92,198,7,44072,8,220,102486,16,18,15,198,7,76089,8,220,102486,16,18,16,25,5046,1272,763,328,933,487,328,77,763,220,16,18,16,92,198,7,68720,8,220,102486,16,18,17,198,7,2662,8,220,102486,16,18,18,25,5046,1272,763,328,933,487,328,77,763,220,16,18,18,92,198,7,2629,8,220,102486,16,18,19,198,7,923,8,220,102486,16,18,20,25,5046,1272,763,328,933,487,328,77,763,220,16,18,20,92,198,7,6548,8,220,102486,16,18,21,198,7,8924,8,220,102486,16,18,22,25,5046,1272,763,328,933,487,328,77,763,220,16,18,22,92,198,7,52194,8,220,102486,16,18,23,198,7,622,8,220,102486,16,18,24,25,5046,1272,763,328,933,487,328,77,763,220,16,18,24,92,198,7,1515,8,220,102486,16,19,15,198,7,1300,8,220,102486,16,19,16,25,5046,1272,763,328,933,487,328,77,763,220,16,19,16,92,198,7,37523,8,220,102486,16,19,17,198,7,44473,8,220,102486,16,19,18,25,5046,1272,763,328,933,487,328,77,763,220,16,19,18,92,198,7,36530,8,220,102486,16,19,19,198,7,3152,8,220,102486,16,19,20,25,5046,1272,763,328,933,487,328,77,763,220,16,19,20,92,198,7,93924,8,220,102486,16,19,21,198,7,3505,8,220,102486,16,19,22,25,5046,1272,763,328,933,487,328,77,763,220,16,19,22,92,198,7,15731,8,220,102486,16,19,23,198,7,6542,8,220,102486,16,19,24,25,5046,1272,763,328,933,487,328,77,763,220,16,19,24,92,198,7,14176,8,220,102486,16,20,15,198,7,11091,8,220,102486,16,20,16,25,5046,1272,763,328,933,487,328,77,763,220,16,20,16,92,198,7,1686,8,220,102486,16,20,17,198,7,11660,8,220,102486,16,20,18,25,5046,1272,763,328,933,487,328,77,763,220,16,20,18,92,198,7,80440,8,220,102486,16,20,19,198,7,18836,8,220,102486,16,20,20,25,5046,1272,763,328,933,487,328,77,763,220,16,20,20,92,198,7,26132,8,220,102486,16,20,21,198,7,5934,8,220,102486,16,20,22,25,5046,1272,763,328,933,487,328,77,763,220,16,20,22,92,198,7,24794,8,220,102486,16,20,23,198,7,40229,8,220,102486,16,20,24,25,5046,1272,763,328,933,487,328,77,763,220,16,20,24,92,198,7,3660,8,220,102486,16,21,15,198,7,11361,8,220,102486,16,21,16,25,5046,1272,763,328,933,487,328,77,763,220,16,21,16,92,198,7,10191,8,220,102486,16,21,17,198,7,8225,8,220,102486,16,21,18,25,5046,1272,763,328,933,487,328,77,763,220,16,21,18,92,198,7,3860,8,220,102486,16,21,19,198,7,78413,8,220,102486,16,21,20,25,5046,1272,763,328,933,487,328,77,763,220,16,21,20,92,198,7,17680,8,220,102486,16,21,21,198,7,15900,8,220,102486,16,21,22,25,5046,1272,763,328,933,487,328,77,763,220,16,21,22,92,198,7,78138,8,220,102486,16,21,23,198,7,15653,8,220,102486,16,21,24,25,5046,1272,763,328,933,487,328,77,763,220,16,21,24,92,198,7,5213,8,220,102486,16,22,15,198,7,22150,8,220,102486,16,22,16,25,5046,1272,763,328,933,487,328,77,763,220,16,22,16,92,198,7,39500,8,220,102486,16,22,17,198,7,10460,8,220,102486,16,22,18,25,5046,1272,763,328,933,487,328,77,763,220,16,22,18,92,198,7,35131,8,220,102486,16,22,19,198,7,21563,8,220,102486,16,22,20,25,5046,1272,763,328,933,487,328,77,763,220,16,22,20,92,198,7,43194,8,220,102486,16,22,21,198,7,27127,8,220,102486,16,22,22,25,5046,1272,763,328,933,487,328,77,763,220,16,22,22,92,198,7,3697,8,220,102486,16,22,23,198,7,20011,8,220,102486,16,22,24,25,5046,1272,763,328,933,487,328,77,763,220,16,22,24,92,198,7,24368,8,220,102486,16,23,15,198,7,15058,8,220,102486,16,23,16,25,5046,1272,763,328,933,487,328,77,763,220,16,23,16,92,198,7,23658,8,220,102486,16,23,17,198,7,8362,8,220,102486,16,23,18,25,5046,1272,763,328,933,487,328,77,763,220,16,23,18,92,198,7,16035,8,220,102486,16,23,19,198,7,24583,8,220,102486,16,23,20,25,5046,1272,763,328,933,487,328,77,763,220,16,23,20,92,198,7,52857,8,220,102486,16,23,21,198,7,38403,8,220,102486,16,23,22,25,5046,1272,763,328,933,487,328,77,763,220,16,23,22,92,198,7,60456,8,220,102486,16,23,23,198,7,81519,8,220,102486,16,23,24,25,5046,1272,763,328,933,487,328,77,763,220,16,23,24,92,198,7,40097,8,220,102486,16,24,15,198,7,16522,8,220,102486,16,24,16,25,5046,1272,763,328,933,487,328,77,763,220,16,24,16,92,198,7,29722,8,220,102486,16,24,17,198,7,21756,8,220,102486,16,24,18,25,5046,1272,763,328,933,487,328,77,763,220,16,24,18,92,198,7,18567,8,220,102486,16,24,19,198,7,1736,8,220,102486,16,24,20,25,5046,1272,763,328,933,487,328,77,763,220,16,24,20,92,198,7,48043,8,220,102486,16,24,21,198,7,87013,8,220,102486,16,24,22,25,5046,1272,763,328,933,487,328,77,763,220,16,24,22,92,198,7,22456,8,220,102486,16,24,23,198,7,23165,8,220,102486,16,24,24,25,5046,1272,763,328,933,487,328,77,763,220,16,24,24,92,198,7,55523,8,220,102486,17,15,15,198,7,13097,8,220,102486,17,15,16,25,5046,1272,763,328,933,487,328,77,763,220,17,15,16,92,198,7,50397,8,220,102486,17,15,17,198,7,41741,8,220,102486,17,15,18,25,5046,1272,763,328,933,487,328,77,763,220,17,15,18,92,198,7,23073,8,220,102486,17,15,19,198,7,6401,8,220,102486,17,15,20,25,5046,1272,763,328,933,487,328,77,763,220,17,15,20,92,198,7,86538,8,220,102486,17,15,21,198,7,16585,8,220,102486,17,15,22,25,5046,1272,763,328,933,487,328,77,763,220,17,15,22,92,198,7,11622,8,220,102486,17,15,23,198,7,2464,8,220,102486,17,15,24,25,5046,1272,763,328,933,487,328,77,763,220,17,15,24,92,198,7,84982,8,220,102486,17,16,15,198,7,75516,8,220,102486,17,16,16,25,5046,1272,763,328,933,487,328,77,763,220,17,16,16,92,198,7,36984,8,220,102486,17,16,17,198,7,58795,8,220,102486,17,16,18,25,5046,1272,763,328,933,487,328,77,763,220,17,16,18,92,198,7,47217,8,220,102486,17,16,19,198,7,59675,8,220,102486,17,16,20,25,5046,1272,763,328,933,487,328,77,763,220,17,16,20,92,198,7,5681,8,220,102486,17,16,21,198,7,3151,8,220,102486,17,16,22,25,5046,1272,763,328,933,487,328,77,763,220,17,16,22,92,198,7,1271,8,220,102486,17,16,23,198,7,887,8,220,102486,17,16,24,25,5046,1272,763,328,933,487,328,77,763,220,17,16,24,92,198,7,5203,8,220,102486,17,17,15,198,7,2685,8,220,102486,17,17,16,25,5046,1272,763,328,933,487,328,77,763,220,17,17,16,92,198,7,1849,8,220,102486,17,17,17,198,7,72470,8,220,102486,17,17,18,25,5046,1272,763,328,933,487,328,77,763,220,17,17,18,92,198,7,5370,8,220,102486,17,17,19,198,7,74063,8,220,102486,17,17,20,25,5046,1272,763,328,933,487,328,77,763,220,17,17,20,92,198,7,27629,8,220,102486,17,17,21,198,7,1655,8,220,102486,17,17,22,25,5046,1272,763,328,933,487,328,77,763,220,17,17,22,92,198,7,1728,8,220,102486,17,17,23,198,7,669,8,220,102486,17,17,24,25,5046,1272,763,328,933,487,328,77,763,220,17,17,24,92,198,7,3682,8,220,102486,17,18,15,198,7,3191,8,220,102486,17,18,16,25,5046,1272,763,328,933,487,328,77,763,220,17,18,16,92,198,7,59865,8,220,102486,17,18,17,198,7,2712,8,220,102486,17,18,18,25,5046,1272,763,328,933,487,328,77,763,220,17,18,18,92,198,7,1580,8,220,102486,17,18,19,198,7,922,8,220,102486,17,18,20,25,5046,1272,763,328,933,487,328,77,763,220,17,18,20,92,198,7,77243,8,220,102486,17,18,21,198,7,2990,8,220,102486,17,18,22,25,5046,1272,763,328,933,487,328,77,763,220,17,18,22,92,198,7,78493,8,220,102486,17,18,23,198,7,5228,8,220,102486,17,18,24,25,5046,1272,763,328,933,487,328,77,763,220,17,18,24,92,198,7,2750,8,220,102486,17,19,15,198,7,42711,8,220,102486,17,19,16,25,5046,1272,763,328,933,487,328,77,763,220,17,19,16,92,198,7,44568,8,220,102486,17,19,17,198,7,56402,8,220,102486,17,19,18,25,5046,1272,763,328,933,487,328,77,763,220,17,19,18,92,198,7,48236,8,220,102486,17,19,19,198,7,38993,8,220,102486,17,19,20,25,5046,1272,763,328,933,487,328,77,763,220,17,19,20,92,198,7,43559,8,220,102486,17,19,21,198,7,61250,8,220,102486,17,19,22,25,5046,1272,763,328,933,487,328,77,763,220,17,19,22,92,198,7,32942,8,220,102486,17,19,23,198,7,86302,8,220,102486,17,19,24,25,5046,1272,763,328,933,487,328,77,763,220,17,19,24,92,198,7,25310,8,220,102486,17,20,15,198,7,26313,8,220,102486,17,20,16,25,5046,1272,763,328,933,487,328,77,763,220,17,20,16,92,198,7,81770,8,220,102486,17,20,17,198,7,12185,8,220,102486,17,20,18,25,5046,1272,763,328,933,487,328,77,763,220,17,20,18,92,198,7,78382,8,220,102486,17,20,19,198,15666,25,318],"slots":[4055],"golds":[0],"nopts":[255],"perms":[[0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,77,78,79,80,81,82,83,84,85,86,87,88,89,90,91,92,93,94,95,96,97,98,99,100,101,102,103,104,105,106,107,108,109,110,111,112,113,114,115,116,117,118,119,120,121,122,123,124,125,126,127,128,129,130,131,132,133,134,135,136,137,138,139,140,141,142,143,144,145,146,147,148,149,150,151,152,153,154,155,156,157,158,159,160,161,162,163,164,165,166,167,168,169,170,171,172,173,174,175,176,177,178,179,180,181,182,183,184,185,186,187,188,189,190,191,192,193,194,195,196,197,198,199,200,201,202,203,204,205,206,207,208,209,210,211,212,213,214,215,216,217,218,219,220,221,222,223,224,225,226,227,228,229,230,231,232,233,234,235,236,237,238,239,240,241,242,243,244,245,246,247,248,249,250,251,252,253,254]],"types":["choice"]}],"context_tokens":8,"unique_tokens":4056,"processed_tokens":4056,"temperatures":[[1.164]]},{"name":"score_2","state":{"items":[1,{"_index":77,"text":"保留"},1,{"_index":77,"text":"保留"},1,{"_index":77,"text":"保留"},1,{"_index":77,"text":"保留"}]},"questions":{"score-id":{"type":"score","instructions":{"ask":"How severe?"},"criteria":[" 0: level 0"," 1: level 1"]}},"rendered_state":"{\"items\": [{\"_index\": 0, \"value\": 1}, {\"_index\": 77, \"text\": \"保留\"}, {\"_index\": 2, \"value\": 1}, {\"_index\": 77, \"text\": \"保留\"}, {\"_index\": 4, \"value\": 1}, {\"_index\": 77, \"text\": \"保留\"}, {\"_index\": 6, \"value\": 1}, {\"_index\": 77, \"text\": \"保留\"}]}","rendered_questions":{"score-id":{"question":"{\"ask\": \"How severe?\"}","options":["0: 0: level 0","1: 1: level 1"],"type":"score","names":[0,1],"legend":[" 0: level 0"," 1: level 1"],"isolated":true}},"index":[["score-id","iso",0,2]],"rows":[{"question":"{\"ask\": \"How severe?\"}\nProposed answer: level 0\nDoes the proposed answer fit?","options":["no","yes"]},{"question":"{\"ask\": \"How severe?\"}\nProposed answer: level 1\nDoes the proposed answer fit?","options":["no","yes"]}],"items":[{"ids":[1905,25,198,4754,3494,763,59670,62,1209,763,220,15,11,328,927,763,220,16,2069,5046,62,1209,763,220,22,22,11,328,1272,763,328,101780,13933,5046,62,1209,763,220,17,11,328,927,763,220,16,2069,5046,62,1209,763,220,22,22,11,328,1272,763,328,101780,13933,5046,62,1209,763,220,19,11,328,927,763,220,16,2069,5046,62,1209,763,220,22,22,11,328,1272,763,328,101780,13933,5046,62,1209,763,220,21,11,328,927,763,220,16,2069,5046,62,1209,763,220,22,22,11,328,1272,763,328,101780,8934,13587,271,14162,25,5046,1038,763,328,4199,14938,7285,92,198,1941,18988,4087,25,2119,220,15,198,20810,279,10660,4087,4786,30,198,3670,25,198,4205,8,874,198,5181,8,9542,198,15666,25,318],"slots":[155],"golds":[0],"nopts":[2],"perms":[[0,1]],"types":["score"]},{"ids":[1905,25,198,4754,3494,763,59670,62,1209,763,220,15,11,328,927,763,220,16,2069,5046,62,1209,763,220,22,22,11,328,1272,763,328,101780,13933,5046,62,1209,763,220,17,11,328,927,763,220,16,2069,5046,62,1209,763,220,22,22,11,328,1272,763,328,101780,13933,5046,62,1209,763,220,19,11,328,927,763,220,16,2069,5046,62,1209,763,220,22,22,11,328,1272,763,328,101780,13933,5046,62,1209,763,220,21,11,328,927,763,220,16,2069,5046,62,1209,763,220,22,22,11,328,1272,763,328,101780,8934,13587,271,14162,25,5046,1038,763,328,4199,14938,7285,92,198,1941,18988,4087,25,2119,220,16,198,20810,279,10660,4087,4786,30,198,3670,25,198,4205,8,874,198,5181,8,9542,198,15666,25,318],"slots":[155],"golds":[0],"nopts":[2],"perms":[[0,1]],"types":["score"]}],"context_tokens":115,"unique_tokens":179,"processed_tokens":312,"temperatures":[[1.124],[1.124]]},{"name":"score_3","state":{"items":[1,{"_index":77,"text":"保留"},1,{"_index":77,"text":"保留"},1,{"_index":77,"text":"保留"},1,{"_index":77,"text":"保留"}]},"questions":{"score-id":{"type":"score","instructions":{"ask":"How severe?"},"criteria":[" 0: level 0"," 1: level 1"," 2: level 2"]}},"rendered_state":"{\"items\": [{\"_index\": 0, \"value\": 1}, {\"_index\": 77, \"text\": \"保留\"}, {\"_index\": 2, \"value\": 1}, {\"_index\": 77, \"text\": \"保留\"}, {\"_index\": 4, \"value\": 1}, {\"_index\": 77, \"text\": \"保留\"}, {\"_index\": 6, \"value\": 1}, {\"_index\": 77, \"text\": \"保留\"}]}","rendered_questions":{"score-id":{"question":"{\"ask\": \"How severe?\"}","options":["0: 0: level 0","1: 1: level 1","2: 2: level 2"],"type":"score","names":[0,1,2],"legend":[" 0: level 0"," 1: level 1"," 2: level 2"],"isolated":true}},"index":[["score-id","iso",0,3]],"rows":[{"question":"{\"ask\": \"How severe?\"}\nProposed answer: level 0\nDoes the proposed answer fit?","options":["no","yes"]},{"question":"{\"ask\": \"How severe?\"}\nProposed answer: level 1\nDoes the proposed answer fit?","options":["no","yes"]},{"question":"{\"ask\": \"How severe?\"}\nProposed answer: level 2\nDoes the proposed answer fit?","options":["no","yes"]}],"items":[{"ids":[1905,25,198,4754,3494,763,59670,62,1209,763,220,15,11,328,927,763,220,16,2069,5046,62,1209,763,220,22,22,11,328,1272,763,328,101780,13933,5046,62,1209,763,220,17,11,328,927,763,220,16,2069,5046,62,1209,763,220,22,22,11,328,1272,763,328,101780,13933,5046,62,1209,763,220,19,11,328,927,763,220,16,2069,5046,62,1209,763,220,22,22,11,328,1272,763,328,101780,13933,5046,62,1209,763,220,21,11,328,927,763,220,16,2069,5046,62,1209,763,220,22,22,11,328,1272,763,328,101780,8934,13587,271,14162,25,5046,1038,763,328,4199,14938,7285,92,198,1941,18988,4087,25,2119,220,15,198,20810,279,10660,4087,4786,30,198,3670,25,198,4205,8,874,198,5181,8,9542,198,15666,25,318],"slots":[155],"golds":[0],"nopts":[2],"perms":[[0,1]],"types":["score"]},{"ids":[1905,25,198,4754,3494,763,59670,62,1209,763,220,15,11,328,927,763,220,16,2069,5046,62,1209,763,220,22,22,11,328,1272,763,328,101780,13933,5046,62,1209,763,220,17,11,328,927,763,220,16,2069,5046,62,1209,763,220,22,22,11,328,1272,763,328,101780,13933,5046,62,1209,763,220,19,11,328,927,763,220,16,2069,5046,62,1209,763,220,22,22,11,328,1272,763,328,101780,13933,5046,62,1209,763,220,21,11,328,927,763,220,16,2069,5046,62,1209,763,220,22,22,11,328,1272,763,328,101780,8934,13587,271,14162,25,5046,1038,763,328,4199,14938,7285,92,198,1941,18988,4087,25,2119,220,16,198,20810,279,10660,4087,4786,30,198,3670,25,198,4205,8,874,198,5181,8,9542,198,15666,25,318],"slots":[155],"golds":[0],"nopts":[2],"perms":[[0,1]],"types":["score"]},{"ids":[1905,25,198,4754,3494,763,59670,62,1209,763,220,15,11,328,927,763,220,16,2069,5046,62,1209,763,220,22,22,11,328,1272,763,328,101780,13933,5046,62,1209,763,220,17,11,328,927,763,220,16,2069,5046,62,1209,763,220,22,22,11,328,1272,763,328,101780,13933,5046,62,1209,763,220,19,11,328,927,763,220,16,2069,5046,62,1209,763,220,22,22,11,328,1272,763,328,101780,13933,5046,62,1209,763,220,21,11,328,927,763,220,16,2069,5046,62,1209,763,220,22,22,11,328,1272,763,328,101780,8934,13587,271,14162,25,5046,1038,763,328,4199,14938,7285,92,198,1941,18988,4087,25,2119,220,17,198,20810,279,10660,4087,4786,30,198,3670,25,198,4205,8,874,198,5181,8,9542,198,15666,25,318],"slots":[155],"golds":[0],"nopts":[2],"perms":[[0,1]],"types":["score"]}],"context_tokens":115,"unique_tokens":202,"processed_tokens":468,"temperatures":[[1.124],[1.124],[1.124]]},{"name":"score_10","state":{"items":[1,{"_index":77,"text":"保留"},1,{"_index":77,"text":"保留"},1,{"_index":77,"text":"保留"},1,{"_index":77,"text":"保留"}]},"questions":{"score-id":{"type":"score","instructions":{"ask":"How severe?"},"criteria":[" 0: level 0"," 1: level 1"," 2: level 2"," 3: level 3"," 4: level 4"," 5: level 5"," 6: level 6"," 7: level 7"," 8: level 8"," 9: level 9"]}},"rendered_state":"{\"items\": [{\"_index\": 0, \"value\": 1}, {\"_index\": 77, \"text\": \"保留\"}, {\"_index\": 2, \"value\": 1}, {\"_index\": 77, \"text\": \"保留\"}, {\"_index\": 4, \"value\": 1}, {\"_index\": 77, \"text\": \"保留\"}, {\"_index\": 6, \"value\": 1}, {\"_index\": 77, \"text\": \"保留\"}]}","rendered_questions":{"score-id":{"question":"{\"ask\": \"How severe?\"}","options":["0: 0: level 0","1: 1: level 1","2: 2: level 2","3: 3: level 3","4: 4: level 4","5: 5: level 5","6: 6: level 6","7: 7: level 7","8: 8: level 8","9: 9: level 9"],"type":"score","names":[0,1,2,3,4,5,6,7,8,9],"legend":[" 0: level 0"," 1: level 1"," 2: level 2"," 3: level 3"," 4: level 4"," 5: level 5"," 6: level 6"," 7: level 7"," 8: level 8"," 9: level 9"],"isolated":true}},"index":[["score-id","iso",0,10]],"rows":[{"question":"{\"ask\": \"How severe?\"}\nProposed answer: level 0\nDoes the proposed answer fit?","options":["no","yes"]},{"question":"{\"ask\": \"How severe?\"}\nProposed answer: level 1\nDoes the proposed answer fit?","options":["no","yes"]},{"question":"{\"ask\": \"How severe?\"}\nProposed answer: level 2\nDoes the proposed answer fit?","options":["no","yes"]},{"question":"{\"ask\": \"How severe?\"}\nProposed answer: level 3\nDoes the proposed answer fit?","options":["no","yes"]},{"question":"{\"ask\": \"How severe?\"}\nProposed answer: level 4\nDoes the proposed answer fit?","options":["no","yes"]},{"question":"{\"ask\": \"How severe?\"}\nProposed answer: level 5\nDoes the proposed answer fit?","options":["no","yes"]},{"question":"{\"ask\": \"How severe?\"}\nProposed answer: level 6\nDoes the proposed answer fit?","options":["no","yes"]},{"question":"{\"ask\": \"How severe?\"}\nProposed answer: level 7\nDoes the proposed answer fit?","options":["no","yes"]},{"question":"{\"ask\": \"How severe?\"}\nProposed answer: level 8\nDoes the proposed answer fit?","options":["no","yes"]},{"question":"{\"ask\": \"How severe?\"}\nProposed answer: level 9\nDoes the proposed answer fit?","options":["no","yes"]}],"items":[{"ids":[1905,25,198,4754,3494,763,59670,62,1209,763,220,15,11,328,927,763,220,16,2069,5046,62,1209,763,220,22,22,11,328,1272,763,328,101780,13933,5046,62,1209,763,220,17,11,328,927,763,220,16,2069,5046,62,1209,763,220,22,22,11,328,1272,763,328,101780,13933,5046,62,1209,763,220,19,11,328,927,763,220,16,2069,5046,62,1209,763,220,22,22,11,328,1272,763,328,101780,13933,5046,62,1209,763,220,21,11,328,927,763,220,16,2069,5046,62,1209,763,220,22,22,11,328,1272,763,328,101780,8934,13587,271,14162,25,5046,1038,763,328,4199,14938,7285,92,198,1941,18988,4087,25,2119,220,15,198,20810,279,10660,4087,4786,30,198,3670,25,198,4205,8,874,198,5181,8,9542,198,15666,25,318],"slots":[155],"golds":[0],"nopts":[2],"perms":[[0,1]],"types":["score"]},{"ids":[1905,25,198,4754,3494,763,59670,62,1209,763,220,15,11,328,927,763,220,16,2069,5046,62,1209,763,220,22,22,11,328,1272,763,328,101780,13933,5046,62,1209,763,220,17,11,328,927,763,220,16,2069,5046,62,1209,763,220,22,22,11,328,1272,763,328,101780,13933,5046,62,1209,763,220,19,11,328,927,763,220,16,2069,5046,62,1209,763,220,22,22,11,328,1272,763,328,101780,13933,5046,62,1209,763,220,21,11,328,927,763,220,16,2069,5046,62,1209,763,220,22,22,11,328,1272,763,328,101780,8934,13587,271,14162,25,5046,1038,763,328,4199,14938,7285,92,198,1941,18988,4087,25,2119,220,16,198,20810,279,10660,4087,4786,30,198,3670,25,198,4205,8,874,198,5181,8,9542,198,15666,25,318],"slots":[155],"golds":[0],"nopts":[2],"perms":[[0,1]],"types":["score"]},{"ids":[1905,25,198,4754,3494,763,59670,62,1209,763,220,15,11,328,927,763,220,16,2069,5046,62,1209,763,220,22,22,11,328,1272,763,328,101780,13933,5046,62,1209,763,220,17,11,328,927,763,220,16,2069,5046,62,1209,763,220,22,22,11,328,1272,763,328,101780,13933,5046,62,1209,763,220,19,11,328,927,763,220,16,2069,5046,62,1209,763,220,22,22,11,328,1272,763,328,101780,13933,5046,62,1209,763,220,21,11,328,927,763,220,16,2069,5046,62,1209,763,220,22,22,11,328,1272,763,328,101780,8934,13587,271,14162,25,5046,1038,763,328,4199,14938,7285,92,198,1941,18988,4087,25,2119,220,17,198,20810,279,10660,4087,4786,30,198,3670,25,198,4205,8,874,198,5181,8,9542,198,15666,25,318],"slots":[155],"golds":[0],"nopts":[2],"perms":[[0,1]],"types":["score"]},{"ids":[1905,25,198,4754,3494,763,59670,62,1209,763,220,15,11,328,927,763,220,16,2069,5046,62,1209,763,220,22,22,11,328,1272,763,328,101780,13933,5046,62,1209,763,220,17,11,328,927,763,220,16,2069,5046,62,1209,763,220,22,22,11,328,1272,763,328,101780,13933,5046,62,1209,763,220,19,11,328,927,763,220,16,2069,5046,62,1209,763,220,22,22,11,328,1272,763,328,101780,13933,5046,62,1209,763,220,21,11,328,927,763,220,16,2069,5046,62,1209,763,220,22,22,11,328,1272,763,328,101780,8934,13587,271,14162,25,5046,1038,763,328,4199,14938,7285,92,198,1941,18988,4087,25,2119,220,18,198,20810,279,10660,4087,4786,30,198,3670,25,198,4205,8,874,198,5181,8,9542,198,15666,25,318],"slots":[155],"golds":[0],"nopts":[2],"perms":[[0,1]],"types":["score"]},{"ids":[1905,25,198,4754,3494,763,59670,62,1209,763,220,15,11,328,927,763,220,16,2069,5046,62,1209,763,220,22,22,11,328,1272,763,328,101780,13933,5046,62,1209,763,220,17,11,328,927,763,220,16,2069,5046,62,1209,763,220,22,22,11,328,1272,763,328,101780,13933,5046,62,1209,763,220,19,11,328,927,763,220,16,2069,5046,62,1209,763,220,22,22,11,328,1272,763,328,101780,13933,5046,62,1209,763,220,21,11,328,927,763,220,16,2069,5046,62,1209,763,220,22,22,11,328,1272,763,328,101780,8934,13587,271,14162,25,5046,1038,763,328,4199,14938,7285,92,198,1941,18988,4087,25,2119,220,19,198,20810,279,10660,4087,4786,30,198,3670,25,198,4205,8,874,198,5181,8,9542,198,15666,25,318],"slots":[155],"golds":[0],"nopts":[2],"perms":[[0,1]],"types":["score"]},{"ids":[1905,25,198,4754,3494,763,59670,62,1209,763,220,15,11,328,927,763,220,16,2069,5046,62,1209,763,220,22,22,11,328,1272,763,328,101780,13933,5046,62,1209,763,220,17,11,328,927,763,220,16,2069,5046,62,1209,763,220,22,22,11,328,1272,763,328,101780,13933,5046,62,1209,763,220,19,11,328,927,763,220,16,2069,5046,62,1209,763,220,22,22,11,328,1272,763,328,101780,13933,5046,62,1209,763,220,21,11,328,927,763,220,16,2069,5046,62,1209,763,220,22,22,11,328,1272,763,328,101780,8934,13587,271,14162,25,5046,1038,763,328,4199,14938,7285,92,198,1941,18988,4087,25,2119,220,20,198,20810,279,10660,4087,4786,30,198,3670,25,198,4205,8,874,198,5181,8,9542,198,15666,25,318],"slots":[155],"golds":[0],"nopts":[2],"perms":[[0,1]],"types":["score"]},{"ids":[1905,25,198,4754,3494,763,59670,62,1209,763,220,15,11,328,927,763,220,16,2069,5046,62,1209,763,220,22,22,11,328,1272,763,328,101780,13933,5046,62,1209,763,220,17,11,328,927,763,220,16,2069,5046,62,1209,763,220,22,22,11,328,1272,763,328,101780,13933,5046,62,1209,763,220,19,11,328,927,763,220,16,2069,5046,62,1209,763,220,22,22,11,328,1272,763,328,101780,13933,5046,62,1209,763,220,21,11,328,927,763,220,16,2069,5046,62,1209,763,220,22,22,11,328,1272,763,328,101780,8934,13587,271,14162,25,5046,1038,763,328,4199,14938,7285,92,198,1941,18988,4087,25,2119,220,21,198,20810,279,10660,4087,4786,30,198,3670,25,198,4205,8,874,198,5181,8,9542,198,15666,25,318],"slots":[155],"golds":[0],"nopts":[2],"perms":[[0,1]],"types":["score"]},{"ids":[1905,25,198,4754,3494,763,59670,62,1209,763,220,15,11,328,927,763,220,16,2069,5046,62,1209,763,220,22,22,11,328,1272,763,328,101780,13933,5046,62,1209,763,220,17,11,328,927,763,220,16,2069,5046,62,1209,763,220,22,22,11,328,1272,763,328,101780,13933,5046,62,1209,763,220,19,11,328,927,763,220,16,2069,5046,62,1209,763,220,22,22,11,328,1272,763,328,101780,13933,5046,62,1209,763,220,21,11,328,927,763,220,16,2069,5046,62,1209,763,220,22,22,11,328,1272,763,328,101780,8934,13587,271,14162,25,5046,1038,763,328,4199,14938,7285,92,198,1941,18988,4087,25,2119,220,22,198,20810,279,10660,4087,4786,30,198,3670,25,198,4205,8,874,198,5181,8,9542,198,15666,25,318],"slots":[155],"golds":[0],"nopts":[2],"perms":[[0,1]],"types":["score"]},{"ids":[1905,25,198,4754,3494,763,59670,62,1209,763,220,15,11,328,927,763,220,16,2069,5046,62,1209,763,220,22,22,11,328,1272,763,328,101780,13933,5046,62,1209,763,220,17,11,328,927,763,220,16,2069,5046,62,1209,763,220,22,22,11,328,1272,763,328,101780,13933,5046,62,1209,763,220,19,11,328,927,763,220,16,2069,5046,62,1209,763,220,22,22,11,328,1272,763,328,101780,13933,5046,62,1209,763,220,21,11,328,927,763,220,16,2069,5046,62,1209,763,220,22,22,11,328,1272,763,328,101780,8934,13587,271,14162,25,5046,1038,763,328,4199,14938,7285,92,198,1941,18988,4087,25,2119,220,23,198,20810,279,10660,4087,4786,30,198,3670,25,198,4205,8,874,198,5181,8,9542,198,15666,25,318],"slots":[155],"golds":[0],"nopts":[2],"perms":[[0,1]],"types":["score"]},{"ids":[1905,25,198,4754,3494,763,59670,62,1209,763,220,15,11,328,927,763,220,16,2069,5046,62,1209,763,220,22,22,11,328,1272,763,328,101780,13933,5046,62,1209,763,220,17,11,328,927,763,220,16,2069,5046,62,1209,763,220,22,22,11,328,1272,763,328,101780,13933,5046,62,1209,763,220,19,11,328,927,763,220,16,2069,5046,62,1209,763,220,22,22,11,328,1272,763,328,101780,13933,5046,62,1209,763,220,21,11,328,927,763,220,16,2069,5046,62,1209,763,220,22,22,11,328,1272,763,328,101780,8934,13587,271,14162,25,5046,1038,763,328,4199,14938,7285,92,198,1941,18988,4087,25,2119,220,24,198,20810,279,10660,4087,4786,30,198,3670,25,198,4205,8,874,198,5181,8,9542,198,15666,25,318],"slots":[155],"golds":[0],"nopts":[2],"perms":[[0,1]],"types":["score"]}],"context_tokens":115,"unique_tokens":363,"processed_tokens":1560,"temperatures":[[1.124],[1.124],[1.124],[1.124],[1.124],[1.124],[1.124],[1.124],[1.124],[1.124]]},{"name":"noul_desc","state":true,"questions":{"invisible":{"type":"bool","criteria":{"true":{"ask":"refund"}}}},"rendered_state":"true","rendered_questions":{"invisible":{"question":"Which answer fits the context?","options":["no","yes: {\"ask\": \"refund\"}"],"type":"noul","names":[false,true],"legend":null,"isolated":true}},"index":[["invisible","list",0,1]],"rows":[{"question":"Which answer fits the context?","options":["no","yes: {\"ask\": \"refund\"}"]}],"items":[{"ids":[1905,25,198,1802,271,14162,25,15451,4087,17759,279,2193,30,198,3670,25,198,4205,8,874,198,5181,8,9542,25,5046,1038,763,328,66165,8934,198,15666,25,318],"slots":[34],"golds":[0],"nopts":[2],"perms":[[0,1]],"types":["noul"]}],"context_tokens":4,"unique_tokens":35,"processed_tokens":35,"temperatures":[[1.624]]},{"name":"noul_partial","state":"plain","questions":{"id":{"type":"noul","criteria":{"false":false}}},"rendered_state":"plain","rendered_questions":{"id":{"question":"Which answer fits the context?","options":["no: false","yes"],"type":"noul","names":[false,true],"legend":null,"isolated":true}},"index":[["id","list",0,1]],"rows":[{"question":"Which answer fits the context?","options":["no: false","yes"]}],"items":[{"ids":[1905,25,198,20139,271,14162,25,15451,4087,17759,279,2193,30,198,3670,25,198,4205,8,874,25,867,198,5181,8,9542,198,15666,25,318],"slots":[29],"golds":[0],"nopts":[2],"perms":[[0,1]],"types":["noul"]}],"context_tokens":4,"unique_tokens":30,"processed_tokens":30,"temperatures":[[1.624]]},{"name":"structured_mixed","state":{"b":"é","a":[-0.0,1e-05,1e+16,null]},"questions":{"first":{"instructions":["Which?",1],"options":["a","b"]},"second":{"type":"score","instructions":"Level","criteria":{"10":"bad","2":"fine"}},"third":{"type":"noul","instructions":null,"criteria":{"false":"fine"}}},"rendered_state":"{\"b\": \"é\", \"a\": [-0.0, 1e-05, 1e+16, null]}","rendered_questions":{"first":{"question":"[\"Which?\", 1]","options":["a","b"],"type":"choice","names":["a","b"],"legend":null,"isolated":true},"second":{"question":"Level","options":["0: fine","1: bad"],"type":"score","names":[0,1],"legend":["fine","bad"],"isolated":true},"third":{"question":"Which answer fits the context?","options":["no: fine","yes"],"type":"noul","names":[false,true],"legend":null,"isolated":true}},"index":[["first","list",0,1],["second","iso",1,2],["third","list",3,1]],"rows":[{"question":"[\"Which?\", 1]","options":["a","b"]},{"question":"Level\nProposed answer: fine\nDoes the proposed answer fit?","options":["no","yes"]},{"question":"Level\nProposed answer: bad\nDoes the proposed answer fit?","options":["no","yes"]},{"question":"Which answer fits the context?","options":["no: fine","yes"]}],"items":[{"ids":[1905,25,198,4754,65,763,328,933,487,328,64,763,9758,15,13,15,11,220,16,68,12,15,20,11,220,16,68,10,16,21,11,819,13587,271,14162,25,4241,22365,29993,220,16,60,198,3670,25,198,4205,8,264,198,5181,8,292,198,15666,25,318],"slots":[56],"golds":[0],"nopts":[2],"perms":[[0,1]],"types":["choice"]},{"ids":[1905,25,198,4754,65,763,328,933,487,328,64,763,9758,15,13,15,11,220,16,68,12,15,20,11,220,16,68,10,16,21,11,819,13587,271,14162,25,9116,198,1941,18988,4087,25,6699,198,20810,279,10660,4087,4786,30,198,3670,25,198,4205,8,874,198,5181,8,9542,198,15666,25,318],"slots":[64],"golds":[0],"nopts":[2],"perms":[[0,1]],"types":["score"]},{"ids":[1905,25,198,4754,65,763,328,933,487,328,64,763,9758,15,13,15,11,220,16,68,12,15,20,11,220,16,68,10,16,21,11,819,13587,271,14162,25,9116,198,1941,18988,4087,25,3743,198,20810,279,10660,4087,4786,30,198,3670,25,198,4205,8,874,198,5181,8,9542,198,15666,25,318],"slots":[64],"golds":[0],"nopts":[2],"perms":[[0,1]],"types":["score"]},{"ids":[1905,25,198,4754,65,763,328,933,487,328,64,763,9758,15,13,15,11,220,16,68,12,15,20,11,220,16,68,10,16,21,11,819,13587,271,14162,25,15451,4087,17759,279,2193,30,198,3670,25,198,4205,8,874,25,6699,198,5181,8,9542,198,15666,25,318],"slots":[58],"golds":[0],"nopts":[2],"perms":[[0,1]],"types":["noul"]}],"context_tokens":33,"unique_tokens":138,"processed_tokens":246,"temperatures":[[1.164],[1.124],[1.124],[1.624]]},{"name":"empty","state":{"n":1},"questions":{},"rendered_state":"{\"n\": 1}","rendered_questions":{},"index":[],"rows":[],"items":[],"context_tokens":9,"unique_tokens":0,"processed_tokens":0,"temperatures":[]}]} diff --git a/tests/decider/data/responses.json b/tests/decider/data/responses.json deleted file mode 100644 index edcf13a4..00000000 --- a/tests/decider/data/responses.json +++ /dev/null @@ -1 +0,0 @@ -[{"name":"choice2","request":{"state":"x","questions":{"c":{"type":"choice","instructions":"q","criteria":{"0":null,"1":null}}}},"logits":[[0.0,0.25]],"probabilities":[[0.44651126861572266,0.5534886717796326]],"expected_answers":{"c":{"type":"choice","choice":"1","confidence":0.107,"x_p_max":0.5535,"certainty":0.0083,"probabilities":{"0":0.4465,"1":0.5535}}}},{"name":"choice3","request":{"state":"x","questions":{"c":{"type":"choice","instructions":"q","criteria":{"0":null,"1":null,"2":null}}}},"logits":[[0.0,0.25,0.5]],"probabilities":[[0.26481956243515015,0.3282663822174072,0.4069140553474426]],"expected_answers":{"c":{"type":"choice","choice":"2","confidence":0.1104,"x_p_max":0.4069,"certainty":0.0138,"probabilities":{"0":0.2648,"1":0.3283,"2":0.4069}}}},{"name":"choice10","request":{"state":"x","questions":{"c":{"type":"choice","instructions":"q","criteria":{"0":null,"1":null,"2":null,"3":null,"4":null,"5":null,"6":null,"7":null,"8":null,"9":null}}}},"logits":[[0.0,0.25,0.5,0.75,1.0,1.25,1.5,0.0,0.25,0.5]],"probabilities":[[0.05442853271961212,0.0674687922000885,0.08363329619169235,0.10367058217525482,0.12850849330425262,0.15929721295833588,0.19746242463588715,0.05442853271961212,0.0674687922000885,0.08363329619169235]],"expected_answers":{"c":{"type":"choice","choice":"6","confidence":0.1083,"x_p_max":0.1975,"certainty":0.0414,"probabilities":{"0":0.0544,"1":0.0675,"2":0.0836,"3":0.1037,"4":0.1285,"5":0.1593,"6":0.1975,"7":0.0544,"8":0.0675,"9":0.0836}}}},{"name":"choice11","request":{"state":"x","questions":{"c":{"type":"choice","instructions":"q","criteria":{"0":null,"1":null,"2":null,"3":null,"4":null,"5":null,"6":null,"7":null,"8":null,"9":null,"10":null}}}},"logits":[[0.0,0.25,0.5,0.75,1.0,1.25,1.5,0.0,0.25,0.5,0.75]],"probabilities":[[0.0493159219622612,0.06113127991557121,0.07577741146087646,0.09393253922462463,0.11643736809492111,0.14433401823043823,0.17891427874565125,0.0493159219622612,0.06113127991557121,0.07577741146087646,0.09393253922462463]],"expected_answers":{"c":{"type":"choice","choice":"6","confidence":0.0968,"x_p_max":0.1789,"certainty":0.036,"probabilities":{"0":0.0493,"1":0.0611,"2":0.0758,"3":0.0939,"4":0.1164,"5":0.1443,"6":0.1789,"7":0.0493,"8":0.0611,"9":0.0758,"10":0.0939}}}},{"name":"choice26","request":{"state":"x","questions":{"c":{"type":"choice","instructions":"q","criteria":{"0":null,"1":null,"2":null,"3":null,"4":null,"5":null,"6":null,"7":null,"8":null,"9":null,"10":null,"11":null,"12":null,"13":null,"14":null,"15":null,"16":null,"17":null,"18":null,"19":null,"20":null,"21":null,"22":null,"23":null,"24":null,"25":null}}}},"logits":[[0.0,0.25,0.5,0.75,1.0,1.25,1.5,0.0,0.25,0.5,0.75,1.0,1.25,1.5,0.0,0.25,0.5,0.75,1.0,1.25,1.5,0.0,0.25,0.5,0.75,1.0]],"probabilities":[[0.019293250516057014,0.023915624246001244,0.029645446687936783,0.03674805164337158,0.0455523319542408,0.056465987116098404,0.06999439001083374,0.019293250516057014,0.023915624246001244,0.029645446687936783,0.03674805164337158,0.0455523319542408,0.056465987116098404,0.06999439001083374,0.019293250516057014,0.023915624246001244,0.029645446687936783,0.03674805164337158,0.0455523319542408,0.056465987116098404,0.06999439001083374,0.019293250516057014,0.023915624246001244,0.029645446687936783,0.03674805164337158,0.0455523319542408]],"expected_answers":{"c":{"type":"choice","choice":"6","confidence":0.0328,"x_p_max":0.07,"certainty":0.0262,"probabilities":{"0":0.0193,"1":0.0239,"2":0.0296,"3":0.0367,"4":0.0456,"5":0.0565,"6":0.07,"7":0.0193,"8":0.0239,"9":0.0296,"10":0.0367,"11":0.0456,"12":0.0565,"13":0.07,"14":0.0193,"15":0.0239,"16":0.0296,"17":0.0367,"18":0.0456,"19":0.0565,"20":0.07,"21":0.0193,"22":0.0239,"23":0.0296,"24":0.0367,"25":0.0456}}}},{"name":"choice255","request":{"state":"x","questions":{"c":{"type":"choice","instructions":"q","criteria":{"0":null,"1":null,"2":null,"3":null,"4":null,"5":null,"6":null,"7":null,"8":null,"9":null,"10":null,"11":null,"12":null,"13":null,"14":null,"15":null,"16":null,"17":null,"18":null,"19":null,"20":null,"21":null,"22":null,"23":null,"24":null,"25":null,"26":null,"27":null,"28":null,"29":null,"30":null,"31":null,"32":null,"33":null,"34":null,"35":null,"36":null,"37":null,"38":null,"39":null,"40":null,"41":null,"42":null,"43":null,"44":null,"45":null,"46":null,"47":null,"48":null,"49":null,"50":null,"51":null,"52":null,"53":null,"54":null,"55":null,"56":null,"57":null,"58":null,"59":null,"60":null,"61":null,"62":null,"63":null,"64":null,"65":null,"66":null,"67":null,"68":null,"69":null,"70":null,"71":null,"72":null,"73":null,"74":null,"75":null,"76":null,"77":null,"78":null,"79":null,"80":null,"81":null,"82":null,"83":null,"84":null,"85":null,"86":null,"87":null,"88":null,"89":null,"90":null,"91":null,"92":null,"93":null,"94":null,"95":null,"96":null,"97":null,"98":null,"99":null,"100":null,"101":null,"102":null,"103":null,"104":null,"105":null,"106":null,"107":null,"108":null,"109":null,"110":null,"111":null,"112":null,"113":null,"114":null,"115":null,"116":null,"117":null,"118":null,"119":null,"120":null,"121":null,"122":null,"123":null,"124":null,"125":null,"126":null,"127":null,"128":null,"129":null,"130":null,"131":null,"132":null,"133":null,"134":null,"135":null,"136":null,"137":null,"138":null,"139":null,"140":null,"141":null,"142":null,"143":null,"144":null,"145":null,"146":null,"147":null,"148":null,"149":null,"150":null,"151":null,"152":null,"153":null,"154":null,"155":null,"156":null,"157":null,"158":null,"159":null,"160":null,"161":null,"162":null,"163":null,"164":null,"165":null,"166":null,"167":null,"168":null,"169":null,"170":null,"171":null,"172":null,"173":null,"174":null,"175":null,"176":null,"177":null,"178":null,"179":null,"180":null,"181":null,"182":null,"183":null,"184":null,"185":null,"186":null,"187":null,"188":null,"189":null,"190":null,"191":null,"192":null,"193":null,"194":null,"195":null,"196":null,"197":null,"198":null,"199":null,"200":null,"201":null,"202":null,"203":null,"204":null,"205":null,"206":null,"207":null,"208":null,"209":null,"210":null,"211":null,"212":null,"213":null,"214":null,"215":null,"216":null,"217":null,"218":null,"219":null,"220":null,"221":null,"222":null,"223":null,"224":null,"225":null,"226":null,"227":null,"228":null,"229":null,"230":null,"231":null,"232":null,"233":null,"234":null,"235":null,"236":null,"237":null,"238":null,"239":null,"240":null,"241":null,"242":null,"243":null,"244":null,"245":null,"246":null,"247":null,"248":null,"249":null,"250":null,"251":null,"252":null,"253":null,"254":null}}}},"logits":[[0.0,0.25,0.5,0.75,1.0,1.25,1.5,0.0,0.25,0.5,0.75,1.0,1.25,1.5,0.0,0.25,0.5,0.75,1.0,1.25,1.5,0.0,0.25,0.5,0.75,1.0,1.25,1.5,0.0,0.25,0.5,0.75,1.0,1.25,1.5,0.0,0.25,0.5,0.75,1.0,1.25,1.5,0.0,0.25,0.5,0.75,1.0,1.25,1.5,0.0,0.25,0.5,0.75,1.0,1.25,1.5,0.0,0.25,0.5,0.75,1.0,1.25,1.5,0.0,0.25,0.5,0.75,1.0,1.25,1.5,0.0,0.25,0.5,0.75,1.0,1.25,1.5,0.0,0.25,0.5,0.75,1.0,1.25,1.5,0.0,0.25,0.5,0.75,1.0,1.25,1.5,0.0,0.25,0.5,0.75,1.0,1.25,1.5,0.0,0.25,0.5,0.75,1.0,1.25,1.5,0.0,0.25,0.5,0.75,1.0,1.25,1.5,0.0,0.25,0.5,0.75,1.0,1.25,1.5,0.0,0.25,0.5,0.75,1.0,1.25,1.5,0.0,0.25,0.5,0.75,1.0,1.25,1.5,0.0,0.25,0.5,0.75,1.0,1.25,1.5,0.0,0.25,0.5,0.75,1.0,1.25,1.5,0.0,0.25,0.5,0.75,1.0,1.25,1.5,0.0,0.25,0.5,0.75,1.0,1.25,1.5,0.0,0.25,0.5,0.75,1.0,1.25,1.5,0.0,0.25,0.5,0.75,1.0,1.25,1.5,0.0,0.25,0.5,0.75,1.0,1.25,1.5,0.0,0.25,0.5,0.75,1.0,1.25,1.5,0.0,0.25,0.5,0.75,1.0,1.25,1.5,0.0,0.25,0.5,0.75,1.0,1.25,1.5,0.0,0.25,0.5,0.75,1.0,1.25,1.5,0.0,0.25,0.5,0.75,1.0,1.25,1.5,0.0,0.25,0.5,0.75,1.0,1.25,1.5,0.0,0.25,0.5,0.75,1.0,1.25,1.5,0.0,0.25,0.5,0.75,1.0,1.25,1.5,0.0,0.25,0.5,0.75,1.0,1.25,1.5,0.0,0.25,0.5,0.75,1.0,1.25,1.5,0.0,0.25,0.5]],"probabilities":[[0.0018894582754001021,0.0023421442601829767,0.002903286600485444,0.003598870476707816,0.004461105912923813,0.005529920104891062,0.006854805629700422,0.0018894582754001021,0.0023421442601829767,0.002903286600485444,0.003598870476707816,0.004461105912923813,0.005529920104891062,0.006854805629700422,0.0018894582754001021,0.0023421442601829767,0.002903286600485444,0.003598870476707816,0.004461105912923813,0.005529920104891062,0.006854805629700422,0.0018894582754001021,0.0023421442601829767,0.002903286600485444,0.003598870476707816,0.004461105912923813,0.005529920104891062,0.006854805629700422,0.0018894582754001021,0.0023421442601829767,0.002903286600485444,0.003598870476707816,0.004461105912923813,0.005529920104891062,0.006854805629700422,0.0018894582754001021,0.0023421442601829767,0.002903286600485444,0.003598870476707816,0.004461105912923813,0.005529920104891062,0.006854805629700422,0.0018894582754001021,0.0023421442601829767,0.002903286600485444,0.003598870476707816,0.004461105912923813,0.005529920104891062,0.006854805629700422,0.0018894582754001021,0.0023421442601829767,0.002903286600485444,0.003598870476707816,0.004461105912923813,0.005529920104891062,0.006854805629700422,0.0018894582754001021,0.0023421442601829767,0.002903286600485444,0.003598870476707816,0.004461105912923813,0.005529920104891062,0.006854805629700422,0.0018894582754001021,0.0023421442601829767,0.002903286600485444,0.003598870476707816,0.004461105912923813,0.005529920104891062,0.006854805629700422,0.0018894582754001021,0.0023421442601829767,0.002903286600485444,0.003598870476707816,0.004461105912923813,0.005529920104891062,0.006854805629700422,0.0018894582754001021,0.0023421442601829767,0.002903286600485444,0.003598870476707816,0.004461105912923813,0.005529920104891062,0.006854805629700422,0.0018894582754001021,0.0023421442601829767,0.002903286600485444,0.003598870476707816,0.004461105912923813,0.005529920104891062,0.006854805629700422,0.0018894582754001021,0.0023421442601829767,0.002903286600485444,0.003598870476707816,0.004461105912923813,0.005529920104891062,0.006854805629700422,0.0018894582754001021,0.0023421442601829767,0.002903286600485444,0.003598870476707816,0.004461105912923813,0.005529920104891062,0.006854805629700422,0.0018894582754001021,0.0023421442601829767,0.002903286600485444,0.003598870476707816,0.004461105912923813,0.005529920104891062,0.006854805629700422,0.0018894582754001021,0.0023421442601829767,0.002903286600485444,0.003598870476707816,0.004461105912923813,0.005529920104891062,0.006854805629700422,0.0018894582754001021,0.0023421442601829767,0.002903286600485444,0.003598870476707816,0.004461105912923813,0.005529920104891062,0.006854805629700422,0.0018894582754001021,0.0023421442601829767,0.002903286600485444,0.003598870476707816,0.004461105912923813,0.005529920104891062,0.006854805629700422,0.0018894582754001021,0.0023421442601829767,0.002903286600485444,0.003598870476707816,0.004461105912923813,0.005529920104891062,0.006854805629700422,0.0018894582754001021,0.0023421442601829767,0.002903286600485444,0.003598870476707816,0.004461105912923813,0.005529920104891062,0.006854805629700422,0.0018894582754001021,0.0023421442601829767,0.002903286600485444,0.003598870476707816,0.004461105912923813,0.005529920104891062,0.006854805629700422,0.0018894582754001021,0.0023421442601829767,0.002903286600485444,0.003598870476707816,0.004461105912923813,0.005529920104891062,0.006854805629700422,0.0018894582754001021,0.0023421442601829767,0.002903286600485444,0.003598870476707816,0.004461105912923813,0.005529920104891062,0.006854805629700422,0.0018894582754001021,0.0023421442601829767,0.002903286600485444,0.003598870476707816,0.004461105912923813,0.005529920104891062,0.006854805629700422,0.0018894582754001021,0.0023421442601829767,0.002903286600485444,0.003598870476707816,0.004461105912923813,0.005529920104891062,0.006854805629700422,0.0018894582754001021,0.0023421442601829767,0.002903286600485444,0.003598870476707816,0.004461105912923813,0.005529920104891062,0.006854805629700422,0.0018894582754001021,0.0023421442601829767,0.002903286600485444,0.003598870476707816,0.004461105912923813,0.005529920104891062,0.006854805629700422,0.0018894582754001021,0.0023421442601829767,0.002903286600485444,0.003598870476707816,0.004461105912923813,0.005529920104891062,0.006854805629700422,0.0018894582754001021,0.0023421442601829767,0.002903286600485444,0.003598870476707816,0.004461105912923813,0.005529920104891062,0.006854805629700422,0.0018894582754001021,0.0023421442601829767,0.002903286600485444,0.003598870476707816,0.004461105912923813,0.005529920104891062,0.006854805629700422,0.0018894582754001021,0.0023421442601829767,0.002903286600485444,0.003598870476707816,0.004461105912923813,0.005529920104891062,0.006854805629700422,0.0018894582754001021,0.0023421442601829767,0.002903286600485444,0.003598870476707816,0.004461105912923813,0.005529920104891062,0.006854805629700422,0.0018894582754001021,0.0023421442601829767,0.002903286600485444,0.003598870476707816,0.004461105912923813,0.005529920104891062,0.006854805629700422,0.0018894582754001021,0.0023421442601829767,0.002903286600485444,0.003598870476707816,0.004461105912923813,0.005529920104891062,0.006854805629700422,0.0018894582754001021,0.0023421442601829767,0.002903286600485444,0.003598870476707816,0.004461105912923813,0.005529920104891062,0.006854805629700422,0.0018894582754001021,0.0023421442601829767,0.002903286600485444]],"expected_answers":{"c":{"type":"choice","choice":"6","confidence":0.0029,"x_p_max":0.0069,"certainty":0.0158,"probabilities":{"0":0.0019,"1":0.0023,"2":0.0029,"3":0.0036,"4":0.0045,"5":0.0055,"6":0.0069,"7":0.0019,"8":0.0023,"9":0.0029,"10":0.0036,"11":0.0045,"12":0.0055,"13":0.0069,"14":0.0019,"15":0.0023,"16":0.0029,"17":0.0036,"18":0.0045,"19":0.0055,"20":0.0069,"21":0.0019,"22":0.0023,"23":0.0029,"24":0.0036,"25":0.0045,"26":0.0055,"27":0.0069,"28":0.0019,"29":0.0023,"30":0.0029,"31":0.0036,"32":0.0045,"33":0.0055,"34":0.0069,"35":0.0019,"36":0.0023,"37":0.0029,"38":0.0036,"39":0.0045,"40":0.0055,"41":0.0069,"42":0.0019,"43":0.0023,"44":0.0029,"45":0.0036,"46":0.0045,"47":0.0055,"48":0.0069,"49":0.0019,"50":0.0023,"51":0.0029,"52":0.0036,"53":0.0045,"54":0.0055,"55":0.0069,"56":0.0019,"57":0.0023,"58":0.0029,"59":0.0036,"60":0.0045,"61":0.0055,"62":0.0069,"63":0.0019,"64":0.0023,"65":0.0029,"66":0.0036,"67":0.0045,"68":0.0055,"69":0.0069,"70":0.0019,"71":0.0023,"72":0.0029,"73":0.0036,"74":0.0045,"75":0.0055,"76":0.0069,"77":0.0019,"78":0.0023,"79":0.0029,"80":0.0036,"81":0.0045,"82":0.0055,"83":0.0069,"84":0.0019,"85":0.0023,"86":0.0029,"87":0.0036,"88":0.0045,"89":0.0055,"90":0.0069,"91":0.0019,"92":0.0023,"93":0.0029,"94":0.0036,"95":0.0045,"96":0.0055,"97":0.0069,"98":0.0019,"99":0.0023,"100":0.0029,"101":0.0036,"102":0.0045,"103":0.0055,"104":0.0069,"105":0.0019,"106":0.0023,"107":0.0029,"108":0.0036,"109":0.0045,"110":0.0055,"111":0.0069,"112":0.0019,"113":0.0023,"114":0.0029,"115":0.0036,"116":0.0045,"117":0.0055,"118":0.0069,"119":0.0019,"120":0.0023,"121":0.0029,"122":0.0036,"123":0.0045,"124":0.0055,"125":0.0069,"126":0.0019,"127":0.0023,"128":0.0029,"129":0.0036,"130":0.0045,"131":0.0055,"132":0.0069,"133":0.0019,"134":0.0023,"135":0.0029,"136":0.0036,"137":0.0045,"138":0.0055,"139":0.0069,"140":0.0019,"141":0.0023,"142":0.0029,"143":0.0036,"144":0.0045,"145":0.0055,"146":0.0069,"147":0.0019,"148":0.0023,"149":0.0029,"150":0.0036,"151":0.0045,"152":0.0055,"153":0.0069,"154":0.0019,"155":0.0023,"156":0.0029,"157":0.0036,"158":0.0045,"159":0.0055,"160":0.0069,"161":0.0019,"162":0.0023,"163":0.0029,"164":0.0036,"165":0.0045,"166":0.0055,"167":0.0069,"168":0.0019,"169":0.0023,"170":0.0029,"171":0.0036,"172":0.0045,"173":0.0055,"174":0.0069,"175":0.0019,"176":0.0023,"177":0.0029,"178":0.0036,"179":0.0045,"180":0.0055,"181":0.0069,"182":0.0019,"183":0.0023,"184":0.0029,"185":0.0036,"186":0.0045,"187":0.0055,"188":0.0069,"189":0.0019,"190":0.0023,"191":0.0029,"192":0.0036,"193":0.0045,"194":0.0055,"195":0.0069,"196":0.0019,"197":0.0023,"198":0.0029,"199":0.0036,"200":0.0045,"201":0.0055,"202":0.0069,"203":0.0019,"204":0.0023,"205":0.0029,"206":0.0036,"207":0.0045,"208":0.0055,"209":0.0069,"210":0.0019,"211":0.0023,"212":0.0029,"213":0.0036,"214":0.0045,"215":0.0055,"216":0.0069,"217":0.0019,"218":0.0023,"219":0.0029,"220":0.0036,"221":0.0045,"222":0.0055,"223":0.0069,"224":0.0019,"225":0.0023,"226":0.0029,"227":0.0036,"228":0.0045,"229":0.0055,"230":0.0069,"231":0.0019,"232":0.0023,"233":0.0029,"234":0.0036,"235":0.0045,"236":0.0055,"237":0.0069,"238":0.0019,"239":0.0023,"240":0.0029,"241":0.0036,"242":0.0045,"243":0.0055,"244":0.0069,"245":0.0019,"246":0.0023,"247":0.0029,"248":0.0036,"249":0.0045,"250":0.0055,"251":0.0069,"252":0.0019,"253":0.0023,"254":0.0029}}}},{"name":"score2","request":{"state":"x","questions":{"s":{"type":"score","instructions":"q","criteria":["level0","level1"]}}},"logits":[[0.0,0.0],[0.0,0.25]],"probabilities":[[0.5,0.5],[0.44462311267852783,0.5553768873214722]],"expected_answers":{"s":{"type":"score","score":0.53,"confidence":0.0525,"x_p_max":0.5262,"certainty":0.002,"legend":{"0":"level0","1":"level1"},"probabilities":{"0":0.4738,"1":0.5262},"level_fit":{"0":0.5,"1":0.5554},"fit_mass":1.0554}}},{"name":"score3","request":{"state":"x","questions":{"s":{"type":"score","instructions":"q","criteria":["level0","level1","level2"]}}},"logits":[[0.0,0.0],[0.0,0.25],[0.0,0.5]],"probabilities":[[0.5,0.5],[0.44462311267852783,0.5553768873214722],[0.3905883729457855,0.6094117164611816]],"expected_answers":{"s":{"type":"score","score":1.07,"confidence":0.0,"x_p_max":0.3661,"certainty":0.003,"legend":{"0":"level0","1":"level1","2":"level2"},"probabilities":{"0":0.3003,"1":0.3336,"2":0.3661},"level_fit":{"0":0.5,"1":0.5554,"2":0.6094},"fit_mass":1.6648}}},{"name":"score10","request":{"state":"x","questions":{"s":{"type":"score","instructions":"q","criteria":["level0","level1","level2","level3","level4","level5","level6","level7","level8","level9"]}}},"logits":[[0.0,0.0],[0.0,0.25],[0.0,0.5],[0.0,0.75],[0.0,1.0],[0.0,1.25],[0.0,1.5],[0.0,1.75],[0.0,2.0],[0.0,2.25]],"probabilities":[[0.5,0.5],[0.44462311267852783,0.5553768873214722],[0.3905883729457855,0.6094117164611816],[0.33911067247390747,0.6608893275260925],[0.291175901889801,0.7088240385055542],[0.24747967720031738,0.7525203824043274],[0.2084127515554428,0.791587233543396],[0.17408624291419983,0.8259137868881226],[0.14438223838806152,0.8556177020072937],[0.11901620775461197,0.880983829498291]],"expected_answers":{"s":{"type":"score","score":4.99,"confidence":0.0,"x_p_max":0.1234,"certainty":0.0067,"legend":{"0":"level0","1":"level1","2":"level2","3":"level3","4":"level4","5":"level5","6":"level6","7":"level7","8":"level8","9":"level9"},"probabilities":{"0":0.07,"1":0.0778,"2":0.0853,"3":0.0925,"4":0.0993,"5":0.1054,"6":0.1108,"7":0.1157,"8":0.1198,"9":0.1234},"level_fit":{"0":0.5,"1":0.5554,"2":0.6094,"3":0.6609,"4":0.7088,"5":0.7525,"6":0.7916,"7":0.8259,"8":0.8556,"9":0.881},"fit_mass":7.1411}}},{"name":"score_zero","request":{"state":"x","questions":{"s":{"type":"score","instructions":"q","criteria":["a","b","c"]}}},"logits":[[1000.0,-1000.0],[1000.0,-1000.0],[1000.0,-1000.0]],"probabilities":[[1.0,0.0],[1.0,0.0],[1.0,0.0]],"expected_answers":{"s":{"type":"score","score":0.0,"confidence":0.0,"x_p_max":0.0,"certainty":1.0,"legend":{"0":"a","1":"b","2":"c"},"probabilities":{"0":0.0,"1":0.0,"2":0.0},"level_fit":{"0":0.0,"1":0.0,"2":0.0},"fit_mass":0.0}}},{"name":"noul","request":{"state":"x","questions":{"n":{"type":"noul","instructions":"q"}}},"logits":[[1.0,0.0]],"probabilities":[[0.6492544412612915,0.3507455885410309]],"expected_answers":{"n":{"type":"noul","noul":0.3507}}},{"name":"choice_uniform255","request":{"state":"x","questions":{"c":{"type":"choice","instructions":"q","criteria":{"0":null,"1":null,"2":null,"3":null,"4":null,"5":null,"6":null,"7":null,"8":null,"9":null,"10":null,"11":null,"12":null,"13":null,"14":null,"15":null,"16":null,"17":null,"18":null,"19":null,"20":null,"21":null,"22":null,"23":null,"24":null,"25":null,"26":null,"27":null,"28":null,"29":null,"30":null,"31":null,"32":null,"33":null,"34":null,"35":null,"36":null,"37":null,"38":null,"39":null,"40":null,"41":null,"42":null,"43":null,"44":null,"45":null,"46":null,"47":null,"48":null,"49":null,"50":null,"51":null,"52":null,"53":null,"54":null,"55":null,"56":null,"57":null,"58":null,"59":null,"60":null,"61":null,"62":null,"63":null,"64":null,"65":null,"66":null,"67":null,"68":null,"69":null,"70":null,"71":null,"72":null,"73":null,"74":null,"75":null,"76":null,"77":null,"78":null,"79":null,"80":null,"81":null,"82":null,"83":null,"84":null,"85":null,"86":null,"87":null,"88":null,"89":null,"90":null,"91":null,"92":null,"93":null,"94":null,"95":null,"96":null,"97":null,"98":null,"99":null,"100":null,"101":null,"102":null,"103":null,"104":null,"105":null,"106":null,"107":null,"108":null,"109":null,"110":null,"111":null,"112":null,"113":null,"114":null,"115":null,"116":null,"117":null,"118":null,"119":null,"120":null,"121":null,"122":null,"123":null,"124":null,"125":null,"126":null,"127":null,"128":null,"129":null,"130":null,"131":null,"132":null,"133":null,"134":null,"135":null,"136":null,"137":null,"138":null,"139":null,"140":null,"141":null,"142":null,"143":null,"144":null,"145":null,"146":null,"147":null,"148":null,"149":null,"150":null,"151":null,"152":null,"153":null,"154":null,"155":null,"156":null,"157":null,"158":null,"159":null,"160":null,"161":null,"162":null,"163":null,"164":null,"165":null,"166":null,"167":null,"168":null,"169":null,"170":null,"171":null,"172":null,"173":null,"174":null,"175":null,"176":null,"177":null,"178":null,"179":null,"180":null,"181":null,"182":null,"183":null,"184":null,"185":null,"186":null,"187":null,"188":null,"189":null,"190":null,"191":null,"192":null,"193":null,"194":null,"195":null,"196":null,"197":null,"198":null,"199":null,"200":null,"201":null,"202":null,"203":null,"204":null,"205":null,"206":null,"207":null,"208":null,"209":null,"210":null,"211":null,"212":null,"213":null,"214":null,"215":null,"216":null,"217":null,"218":null,"219":null,"220":null,"221":null,"222":null,"223":null,"224":null,"225":null,"226":null,"227":null,"228":null,"229":null,"230":null,"231":null,"232":null,"233":null,"234":null,"235":null,"236":null,"237":null,"238":null,"239":null,"240":null,"241":null,"242":null,"243":null,"244":null,"245":null,"246":null,"247":null,"248":null,"249":null,"250":null,"251":null,"252":null,"253":null,"254":null}}}},"logits":[[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0]],"probabilities":[[0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563]],"expected_answers":{"c":{"type":"choice","choice":"0","confidence":0.0,"x_p_max":0.0039,"certainty":0.0,"probabilities":{"0":0.0039,"1":0.0039,"2":0.0039,"3":0.0039,"4":0.0039,"5":0.0039,"6":0.0039,"7":0.0039,"8":0.0039,"9":0.0039,"10":0.0039,"11":0.0039,"12":0.0039,"13":0.0039,"14":0.0039,"15":0.0039,"16":0.0039,"17":0.0039,"18":0.0039,"19":0.0039,"20":0.0039,"21":0.0039,"22":0.0039,"23":0.0039,"24":0.0039,"25":0.0039,"26":0.0039,"27":0.0039,"28":0.0039,"29":0.0039,"30":0.0039,"31":0.0039,"32":0.0039,"33":0.0039,"34":0.0039,"35":0.0039,"36":0.0039,"37":0.0039,"38":0.0039,"39":0.0039,"40":0.0039,"41":0.0039,"42":0.0039,"43":0.0039,"44":0.0039,"45":0.0039,"46":0.0039,"47":0.0039,"48":0.0039,"49":0.0039,"50":0.0039,"51":0.0039,"52":0.0039,"53":0.0039,"54":0.0039,"55":0.0039,"56":0.0039,"57":0.0039,"58":0.0039,"59":0.0039,"60":0.0039,"61":0.0039,"62":0.0039,"63":0.0039,"64":0.0039,"65":0.0039,"66":0.0039,"67":0.0039,"68":0.0039,"69":0.0039,"70":0.0039,"71":0.0039,"72":0.0039,"73":0.0039,"74":0.0039,"75":0.0039,"76":0.0039,"77":0.0039,"78":0.0039,"79":0.0039,"80":0.0039,"81":0.0039,"82":0.0039,"83":0.0039,"84":0.0039,"85":0.0039,"86":0.0039,"87":0.0039,"88":0.0039,"89":0.0039,"90":0.0039,"91":0.0039,"92":0.0039,"93":0.0039,"94":0.0039,"95":0.0039,"96":0.0039,"97":0.0039,"98":0.0039,"99":0.0039,"100":0.0039,"101":0.0039,"102":0.0039,"103":0.0039,"104":0.0039,"105":0.0039,"106":0.0039,"107":0.0039,"108":0.0039,"109":0.0039,"110":0.0039,"111":0.0039,"112":0.0039,"113":0.0039,"114":0.0039,"115":0.0039,"116":0.0039,"117":0.0039,"118":0.0039,"119":0.0039,"120":0.0039,"121":0.0039,"122":0.0039,"123":0.0039,"124":0.0039,"125":0.0039,"126":0.0039,"127":0.0039,"128":0.0039,"129":0.0039,"130":0.0039,"131":0.0039,"132":0.0039,"133":0.0039,"134":0.0039,"135":0.0039,"136":0.0039,"137":0.0039,"138":0.0039,"139":0.0039,"140":0.0039,"141":0.0039,"142":0.0039,"143":0.0039,"144":0.0039,"145":0.0039,"146":0.0039,"147":0.0039,"148":0.0039,"149":0.0039,"150":0.0039,"151":0.0039,"152":0.0039,"153":0.0039,"154":0.0039,"155":0.0039,"156":0.0039,"157":0.0039,"158":0.0039,"159":0.0039,"160":0.0039,"161":0.0039,"162":0.0039,"163":0.0039,"164":0.0039,"165":0.0039,"166":0.0039,"167":0.0039,"168":0.0039,"169":0.0039,"170":0.0039,"171":0.0039,"172":0.0039,"173":0.0039,"174":0.0039,"175":0.0039,"176":0.0039,"177":0.0039,"178":0.0039,"179":0.0039,"180":0.0039,"181":0.0039,"182":0.0039,"183":0.0039,"184":0.0039,"185":0.0039,"186":0.0039,"187":0.0039,"188":0.0039,"189":0.0039,"190":0.0039,"191":0.0039,"192":0.0039,"193":0.0039,"194":0.0039,"195":0.0039,"196":0.0039,"197":0.0039,"198":0.0039,"199":0.0039,"200":0.0039,"201":0.0039,"202":0.0039,"203":0.0039,"204":0.0039,"205":0.0039,"206":0.0039,"207":0.0039,"208":0.0039,"209":0.0039,"210":0.0039,"211":0.0039,"212":0.0039,"213":0.0039,"214":0.0039,"215":0.0039,"216":0.0039,"217":0.0039,"218":0.0039,"219":0.0039,"220":0.0039,"221":0.0039,"222":0.0039,"223":0.0039,"224":0.0039,"225":0.0039,"226":0.0039,"227":0.0039,"228":0.0039,"229":0.0039,"230":0.0039,"231":0.0039,"232":0.0039,"233":0.0039,"234":0.0039,"235":0.0039,"236":0.0039,"237":0.0039,"238":0.0039,"239":0.0039,"240":0.0039,"241":0.0039,"242":0.0039,"243":0.0039,"244":0.0039,"245":0.0039,"246":0.0039,"247":0.0039,"248":0.0039,"249":0.0039,"250":0.0039,"251":0.0039,"252":0.0039,"253":0.0039,"254":0.0039}}}}] diff --git a/tests/decider/executor.rs b/tests/decider/executor.rs deleted file mode 100644 index c5a73532..00000000 --- a/tests/decider/executor.rs +++ /dev/null @@ -1,87 +0,0 @@ -use super::*; -use crate::Kind; -fn row(ids: Vec, candidates: Vec, position: usize) -> RowInput { - RowInput { - ids, - candidate_ids: candidates, - readout_position: position, - question_id: "q".into(), - level_index: None, - kind: Kind::Choice, - temperature: 1.164, - } -} -#[test] -fn validates_every_row_before_dispatch() { - let labels = [32, 33, 34]; - assert!(validate_rows(&[row(vec![1, 2], vec![32, 33], 1)], &labels).is_ok()); - for input in [ - row(vec![], vec![32, 33], 0), - row(vec![1, 2], vec![32, 33], 0), - row(vec![VOCAB as u32], vec![32, 33], 0), - row(vec![1], vec![33, 32], 0), - row(vec![1], vec![], 0), - row(vec![1], vec![32], 0), - row(vec![1; 36865], vec![32, 33], 36864), - ] { - assert!(validate_rows(&[row(vec![1], vec![32, 33], 0), input], &labels).is_err()); - } - assert!(validate_rows(&[], &labels).is_ok()); -} - -#[test] -#[ignore = "requires the built Qwen CUDA library and an available NVIDIA GPU"] -fn bf16_head_rounds_before_fp32_calibration() { - let library = std::env::var("DECIDER_CUDA_LIB").unwrap(); - cuda::load(Path::new(&library)).unwrap(); - cuda::set_device(0).unwrap(); - let mut weights = vec![0; PADDED_LABELS * HIDDEN * 2]; - for (offset, value) in [ - (0, 1.0), - (2, 0.00390625), - (HIDDEN * 2, 1.0), - (HIDDEN * 2 + 2, 0.015625), - ] { - weights[offset..offset + 2].copy_from_slice(&bf16::from_f32(value).to_le_bytes()); - } - let head = Head::new(&weights, 1).unwrap(); - let mut hidden = vec![0.0; HIDDEN]; - hidden[..2].copy_from_slice(&[1.0, 1.0]); - assert_eq!(head.project(&hidden, 2).unwrap(), vec![1.0, 1.015625]); - assert_eq!(head.project(&hidden, 255).unwrap()[254], 0.0); - hidden[0] = f32::NAN; - assert!(head.project(&hidden, 2).is_err()); -} - -#[test] -#[ignore = "requires the built Qwen CUDA library and an available NVIDIA GPU"] -fn batched_head_preserves_rows_counts_and_bf16_rounding() { - cuda::load(Path::new(&std::env::var("DECIDER_CUDA_LIB").unwrap())).unwrap(); - cuda::set_device(0).unwrap(); - let mut weights = vec![0; PADDED_LABELS * HIDDEN * 2]; - for (offset, value) in [ - (0, 1.0), - (2, 0.00390625), - (HIDDEN * 2, 1.0), - (HIDDEN * 2 + 2, 0.015625), - ] { - weights[offset..offset + 2].copy_from_slice(&bf16::from_f32(value).to_le_bytes()); - } - let head = Head::new(&weights, 2).unwrap(); - let mut a = vec![0.0; HIDDEN]; - a[..2].copy_from_slice(&[1.0, 1.0]); - let mut b = vec![0.0; HIDDEN]; - b[..2].copy_from_slice(&[2.0, 1.0]); - let first = head.project_batch(&[&a, &b], &[2, 255]).unwrap(); - assert_eq!(first[0], vec![1.0, 1.015625]); - assert_eq!(&first[1][..2], &[2.0, 2.015625]); - assert_eq!(first[1].len(), 255); - assert!(first[1][2..].iter().all(|&x| x == 0.0)); - let reversed = head.project_batch(&[&b, &a], &[255, 2]).unwrap(); - assert_eq!(reversed[0], first[1]); - assert_eq!(reversed[1], first[0]); - assert!(head.project_batch(&[&a, &b, &a], &[2, 2, 2]).is_err()); - assert!(head.project_batch(&[&a, &b], &[2]).is_err()); - b[0] = f32::NAN; - assert!(head.project_batch(&[&a, &b], &[2, 2]).is_err()); -} diff --git a/tests/decider/fail_head.c b/tests/decider/fail_head.c deleted file mode 100644 index 07418ff3..00000000 --- a/tests/decider/fail_head.c +++ /dev/null @@ -1,18 +0,0 @@ -// Test-only wrapper: real CUDA math, fail the second selected-head projection. -#define _GNU_SOURCE -#include -#include -#include "ops.h" - -typedef int (*gemm_fn)(void*, const void*, const void*, void*, int, int, int, int, void*); -int cs1_gemm(void* handle, const void* x, const void* w, void* y, - int m, int n, int k, int ldy, void* stream) { - static unsigned projection_calls; - gemm_fn original = (gemm_fn)dlsym(RTLD_NEXT, "cs1_gemm"); - if (!original) return 999; - if (m >= 1 && m <= 16 && n == 256 && k == 2048) { - fprintf(stderr, "Decider test projection %u rows %d\n", ++projection_calls, m); - if (projection_calls == 2) return 2; - } - return original(handle, x, w, y, m, n, k, ldy, stream); -} diff --git a/tests/decider/gpu.rs b/tests/decider/gpu.rs deleted file mode 100644 index 8bec730e..00000000 --- a/tests/decider/gpu.rs +++ /dev/null @@ -1,137 +0,0 @@ -use omni_decider_native::engine::Engine; -use std::path::Path; - -#[tokio::test] -async fn missing_artifacts_fail_before_cuda_initialization() { - let dir = std::env::temp_dir().join(format!("missing-decider-{}", std::process::id())); - let result = Engine::load(&dir, Path::new("unavailable-cuda-library.so")).await; - assert!(result.is_err()); - assert!(format!("{:#}", result.err().unwrap()).contains("No such file")); -} - -#[tokio::test] -#[ignore = "requires pinned Decider-2B checkpoint and NVIDIA CUDA"] -async fn complete_eager_requests_preserve_order_and_recover_from_validation_errors() { - let dir = std::env::var("DECIDER_MODEL").unwrap(); - let library = std::env::var("DECIDER_CUDA_LIB").unwrap(); - let engine = Engine::load(Path::new(&dir), Path::new(&library)) - .await - .unwrap(); - let raw = br#"{"state":"The ticket requests a refund.","questions":{"category":{"type":"choice","instructions":"Choose the category.","criteria":{"refund":"Refund","shipping":"Shipping"}},"refund":{"type":"noul","instructions":"The customer requests a refund."},"urgency":{"type":"score","instructions":"How urgent is the ticket?","criteria":["low urgency","high urgency"]}}}"#; - let first = engine.predict(raw).await.unwrap(); - assert_eq!( - first["answers"] - .as_object() - .unwrap() - .keys() - .map(String::as_str) - .collect::>(), - vec!["category", "refund", "urgency"] - ); - assert_eq!(first["usage"]["output_tokens"], 0); - assert!( - engine - .predict(br#"{"state":0,"questions":{"bad":{"criteria":["one"]}}}"#) - .await - .is_err() - ); - assert!(engine.executor.is_ready()); - assert_eq!(engine.predict(raw).await.unwrap(), first); - let empty = engine - .predict(br#"{"state":{},"questions":{}}"#) - .await - .unwrap(); - assert!(empty["answers"].as_object().unwrap().is_empty()); - assert_eq!(empty["usage"]["input_tokens"], 0); - assert_eq!(engine.health()["status"], "ready"); -} - -#[tokio::test] -#[ignore = "requires pinned Decider-2B checkpoint and NVIDIA CUDA; near-limit allocation"] -async fn maximum_complete_row_grows_scratch_without_leaking_state() { - use omni_decider_native::Limits; - use serde_json::json; - let dir = std::env::var("DECIDER_MODEL").unwrap(); - let library = std::env::var("DECIDER_CUDA_LIB").unwrap(); - let engine = Engine::load(Path::new(&dir), Path::new(&library)) - .await - .unwrap(); - let short = br#"{"state":"A customer requests a refund.","questions":{"q":{"instructions":"Choose a team.","criteria":["billing","shipping"]}}}"#; - let baseline = engine.predict(short).await.unwrap(); - let request = |n: usize| { - json!({"state":"hello ".repeat(45000),"questions":{"q":{"instructions":format!("Choose.{}"," x".repeat(n)),"criteria":["billing","shipping"]}}}).to_string() - }; - let (mut low, mut high) = (0usize, 8000usize); - while low < high { - let mid = (low + high).div_ceil(2); - if engine.processor.prepare(request(mid).as_bytes()).is_ok() { - low = mid; - } else { - high = mid - 1; - } - } - let raw = request(low); - let prepared = engine.processor.prepare(raw.as_bytes()).unwrap(); - assert_eq!(prepared.rows[0].ids.len(), Limits::default().max_row_tokens); - assert!( - engine - .processor - .prepare(request(low + 1).as_bytes()) - .is_err() - ); - let first = engine.predict(raw.as_bytes()).await.unwrap(); - assert!( - first["answers"]["q"]["probabilities"] - .as_object() - .unwrap() - .values() - .all(|value| value.as_f64().unwrap().is_finite()) - ); - assert_eq!(engine.predict(raw.as_bytes()).await.unwrap(), first); - assert_eq!(engine.predict(short).await.unwrap(), baseline); - assert!(engine.executor.is_ready()); -} - -#[tokio::test] -#[ignore = "requires pinned Decider-2B checkpoint and NVIDIA CUDA; request packing"] -async fn packed_mixed_rows_preserve_order_usage_and_recovery() { - use omni_decider_native::batching::BatchLimits; - use serde_json::json; - let dir = std::env::var("DECIDER_MODEL").unwrap(); - let library = std::env::var("DECIDER_CUDA_LIB").unwrap(); - let engine = Engine::load_with_batch( - Path::new(&dir), - Path::new(&library), - BatchLimits::new(4, 4096).unwrap(), - ) - .await - .unwrap(); - let raw=json!({"state":"A customer requests a refund for a damaged item.","questions":{"first":{"instructions":"Choose the team.","criteria":["refund","shipping"]},"score":{"type":"score","instructions":"Severity.","criteria":["low","medium","high"]},"last":{"type":"noul","instructions":"The item is damaged."}}}).to_string(); - let first = engine.predict(raw.as_bytes()).await.unwrap(); - assert_eq!( - first["answers"] - .as_object() - .unwrap() - .keys() - .map(String::as_str) - .collect::>(), - vec!["first", "score", "last"] - ); - assert_eq!(engine.health()["batch_limits"]["rows"], 4); - let long=json!({"state":"The item is damaged. ".repeat(1000),"questions":{"q":{"instructions":"Choose the team.","criteria":["refund","shipping"]}}}).to_string(); - assert!( - engine.processor.prepare(long.as_bytes()).unwrap().rows[0] - .ids - .len() - > 4096 - ); - assert!(engine.predict(long.as_bytes()).await.is_ok()); - assert_eq!(engine.predict(raw.as_bytes()).await.unwrap(), first); - assert!( - engine - .predict(br#"{"state":0,"questions":{"q":{"criteria":["only"]}}}"#) - .await - .is_err() - ); - assert_eq!(engine.predict(raw.as_bytes()).await.unwrap(), first); -} diff --git a/tests/decider/test_download.py b/tests/decider/test_download.py deleted file mode 100644 index e193a8c5..00000000 --- a/tests/decider/test_download.py +++ /dev/null @@ -1,28 +0,0 @@ -import importlib.util -from pathlib import Path -import tempfile -import unittest - -MODULE_PATH = Path(__file__).parents[2] / "recipe/decider/download_weights.py" -spec = importlib.util.spec_from_file_location("decider_download", MODULE_PATH) -module = importlib.util.module_from_spec(spec) -spec.loader.exec_module(module) - - -class Verification(unittest.TestCase): - def test_rejects_content_and_size_mismatches(self): - expected = (3, "ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad") - with tempfile.TemporaryDirectory() as root: - path = Path(root) / "model" - path.write_bytes(b"abc") - module.verify(path, expected) - path.write_bytes(b"abd") - with self.assertRaisesRegex(ValueError, "checksum"): - module.verify(path, expected) - path.write_bytes(b"abcd") - with self.assertRaisesRegex(ValueError, "size"): - module.verify(path, expected) - - -if __name__ == "__main__": - unittest.main() diff --git a/tests/decider/test_reference.py b/tests/decider/test_reference.py deleted file mode 100644 index 2130f8c9..00000000 --- a/tests/decider/test_reference.py +++ /dev/null @@ -1,29 +0,0 @@ -import hashlib -import importlib.util -from pathlib import Path -import tempfile -import unittest - -spec = importlib.util.spec_from_file_location("decider_reference", Path(__file__).with_name("verify_reference.py")) -module = importlib.util.module_from_spec(spec) -spec.loader.exec_module(module) - - -class ReferenceProvenance(unittest.TestCase): - def test_source_hashes_reject_modified_and_missing_reference(self): - with tempfile.TemporaryDirectory() as temporary: - root = Path(temporary) - source = root / "model.py" - source.write_bytes(b"abc") - expected = {"model.py": hashlib.sha256(b"abc").hexdigest()} - self.assertEqual(module.verify_sources(root, expected), expected) - source.write_bytes(b"abd") - with self.assertRaisesRegex(ValueError, "reference checksum"): - module.verify_sources(root, expected) - source.unlink() - with self.assertRaises(FileNotFoundError): - module.verify_sources(root, expected) - - -if __name__ == "__main__": - unittest.main() diff --git a/tests/decider/verify_http.py b/tests/decider/verify_http.py deleted file mode 100644 index 595f6c64..00000000 --- a/tests/decider/verify_http.py +++ /dev/null @@ -1,124 +0,0 @@ -#!/usr/bin/env python3 -"""Exercise real native and frontend sockets, readiness, rejection and recovery.""" -import argparse -import concurrent.futures -import json -import os -from pathlib import Path -import subprocess -import time -import urllib.error -import urllib.request - - -def request(base, path, body=None, content_type="application/json"): - headers = {"Content-Type": content_type} if body is not None else {} - req = urllib.request.Request(base + path, data=body, headers=headers) - try: - with urllib.request.urlopen(req, timeout=120) as response: - return response.status, response.read() - except urllib.error.HTTPError as response: - return response.code, response.read() - - -def wait_ready(base, child, log): - deadline = time.monotonic() + 180 - while time.monotonic() < deadline: - if child.poll() is not None: - raise RuntimeError(f"worker exited: {log.read_text()[-3000:]}") - try: - status, raw = request(base, "/health") - if status == 200: - return json.loads(raw) - except (OSError, urllib.error.URLError): - pass - time.sleep(0.1) - raise TimeoutError("readiness timeout") - - -def main(): - parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument("--worker", type=Path, required=True) - parser.add_argument("--frontend", type=Path, required=True) - parser.add_argument("--model", type=Path, required=True) - parser.add_argument("--library", type=Path, required=True) - parser.add_argument("--parity-output", type=Path, required=True) - parser.add_argument("--output", type=Path, required=True) - parser.add_argument("--execution", default="eager") - parser.add_argument("--native-records", type=Path, help="verify_modes records.jsonl for the selected mode") - args = parser.parse_args() - args.output.mkdir(parents=True, exist_ok=True) - worker_base, frontend_base = "http://127.0.0.1:18110", "http://127.0.0.1:18111" - environment = os.environ.copy() - environment.update(DECIDER_MODEL=str(args.model), DECIDER_CUDA_LIB=str(args.library), DECIDER_PORT="18110", CUA_S1_GRAPH="0") - processes, logs, checks = [], [], [] - try: - path = args.output / "worker.log" - logs.append(path.open("w")) - worker = subprocess.Popen([str(args.worker)], env=environment, stdout=logs[-1], stderr=subprocess.STDOUT) - processes.append(worker) - metadata = wait_ready(worker_base, worker, path) - assert metadata["checkpoint_revision"] == "533964dae8be954c5b5e19fa4948e48408094c1e" - assert metadata["execution"] == args.execution and metadata["dtype"] == "bfloat16" - environment.update(OMNI_JEV_BIND="127.0.0.1:18111", OMNI_JEV_BACKEND_URL=worker_base) - path = args.output / "frontend.log" - logs.append(path.open("w")) - frontend = subprocess.Popen([str(args.frontend)], env=environment, stdout=logs[-1], stderr=subprocess.STDOUT) - processes.append(frontend) - wait_ready(frontend_base, frontend, path) - protocol = json.loads((args.parity_output / "protocol.json").read_text()) - native_records = [json.loads(line) for line in (args.parity_output / "native.jsonl").read_text().splitlines()] - if args.native_records: - native_records = [item["record"] for line in args.native_records.read_text().splitlines() - for item in [json.loads(line)] if item["phase"] == "parity"] - assert len(native_records) == len(protocol["cases"]) - for case, native in zip(protocol["cases"], native_records): - body = json.dumps(case["request"], ensure_ascii=False).encode() - direct, proxy = request(worker_base, "/v1/systemone", body), request(frontend_base, "/v1/systemone", body) - assert direct[0] == proxy[0] == 200 - assert direct[1] == proxy[1] - assert json.loads(direct[1]) == native["response"] - checks.append({"name": case["name"], "status": direct[0], "byte_parity": True, "response": json.loads(direct[1])}) - negative = [ - (b"{}", "text/plain", 415), - (b"{invalid", "application/json", 422), - (b'{"state":0,"state":1,"questions":{}}', "application/json", 422), - (b'{"state":0,"questions":{"q":{"instructions":"q","criteria":["one"]}}}', "application/json", 422), - (b'{"state":0,"questions":{},"independent":false}', "application/json", 422), - (b'{"state":0,"questions":{"q":{"type":"image","instructions":"q","criteria":["a","b"]}}}', "application/json", 422), - ] - for i, (body, content_type, expected) in enumerate(negative): - direct = request(worker_base, "/v1/systemone", body, content_type) - proxy = request(frontend_base, "/v1/systemone", body, content_type) - assert direct[0] == expected and proxy == direct, (i, direct[0], proxy[0], expected) - assert isinstance(json.loads(direct[1])["detail"], str) - checks.append({"name": f"invalid_{i}", "status": expected, "byte_parity": True}) - oversized = request(worker_base, "/v1/systemone", b" " * (8 * 1024 * 1024 + 1)) - assert oversized[0] == 413 - checks.append({"name": "body_limit", "status": 413}) - raw = json.dumps(protocol["cases"][13]["request"]).encode() - baseline = request(worker_base, "/v1/systemone", raw) - with concurrent.futures.ThreadPoolExecutor(max_workers=4) as pool: - concurrent_results = list(pool.map(lambda i: request(frontend_base if i % 2 else worker_base, "/v1/systemone", raw), range(12))) - assert all(result == baseline for result in concurrent_results) - checks.append({"name": "concurrent_direct_and_proxy", "requests": len(concurrent_results), "byte_parity": True}) - status, health = request(worker_base, "/health") - assert status == 200 and json.loads(health)["status"] == "ready" - checks.append({"name": "recovery_health", "status": status}) - (args.output / "summary.json").write_text(json.dumps({"passed": len(checks), "metadata": metadata, "checks": checks}, indent=2, ensure_ascii=False) + "\n") - print(json.dumps({"passed": len(checks), "concurrent_requests": len(concurrent_results)}), flush=True) - finally: - for child in reversed(processes): - if child.poll() is None: - child.terminate() - try: - child.wait(timeout=30) - except subprocess.TimeoutExpired: - child.kill() - child.wait() - for handle in logs: - handle.close() - - -if __name__ == "__main__": - main() diff --git a/tests/decider/verify_modes.py b/tests/decider/verify_modes.py deleted file mode 100644 index bb5d439b..00000000 --- a/tests/decider/verify_modes.py +++ /dev/null @@ -1,104 +0,0 @@ -#!/usr/bin/env python3 -"""Sequential matched diagnostic-runner comparisons; raw output and frozen plan retained.""" -import argparse -import hashlib -import json -import os -from pathlib import Path -import statistics -import subprocess -import time - -from verify_reference import GATES, compare - - -def digest(path): - return hashlib.sha256(Path(path).read_bytes()).hexdigest() - - -def main(): - parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument("--plan", type=Path, required=True) - parser.add_argument("--output", type=Path, required=True) - args = parser.parse_args() - plan = json.loads(args.plan.read_text()) - assert plan["repetitions"] == 2 - parity = Path(plan["parity_output"]) - protocol = json.loads((parity / "protocol.json").read_text()) - reference = [json.loads(line) for line in (parity / "reference.jsonl").read_text().splitlines()] - baseline = [json.loads(line) for line in (parity / "native.jsonl").read_text().splitlines()] - cases = protocol["cases"] - assert len(cases) == len(reference) == len(baseline) - selected = [i for i, case in enumerate(cases) if case["name"] in plan["timed_cases"]] - assert len(selected) == len(plan["timed_cases"]) - args.output.mkdir(parents=True, exist_ok=False) - frozen = {**plan, "gates": GATES, "baseline_protocol": protocol, - "input_hashes": {name: digest(parity / name) for name in ("protocol.json", "reference.jsonl", "native.jsonl")}, - "runner_sha256": digest(__file__), "comparison_sha256": digest(Path(__file__).with_name("verify_reference.py")), "library_sha256": digest(plan["library"]), - "binary_hashes": {mode["name"]: digest(mode["binary"]) for mode in plan["modes"]}, - "timing": "wall roundtrip: request JSON encoding/write through diagnostic stdout JSON decode; includes CPU preparation, GPU synchronization and serialization; excludes startup and one workload warmup; concurrency=1"} - (args.output / "protocol.json").write_text(json.dumps(frozen, indent=2) + "\n") - summaries = [] - for mode in plan["modes"]: - directory = args.output / mode["name"] - directory.mkdir() - env = dict(os.environ) - for key in ("CUA_S1_GRAPH", "DECIDER_GRAPH", "DECIDER_PREFIX", "DECIDER_FIXED", "DECIDER_BATCH_MAX_ROWS", "DECIDER_BATCH_MAX_TOKENS"): - env.pop(key, None) - env.update(mode["env"]) - with (directory / "stderr.log").open("w") as stderr, (directory / "records.jsonl").open("w") as records: - start = time.perf_counter() - child = subprocess.Popen([mode["binary"], plan["model"], plan["library"]], env=env, - stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=stderr, text=True) - def run(request): - begun = time.perf_counter() - child.stdin.write(json.dumps(request, ensure_ascii=False) + "\n") - child.stdin.flush() - line = child.stdout.readline() - if not line: - raise RuntimeError(f"{mode['name']} terminated: {child.poll()}; see stderr.log") - value = json.loads(line) - return value, (time.perf_counter() - begun) * 1000 - try: - run({"state": "", "questions": {}}) - startup_ms = (time.perf_counter() - start) * 1000 - checks, samples = [], [] - for i, case in enumerate(cases): - got, ms = run(case["request"]) - records.write(json.dumps({"phase": "parity", "name": case["name"], "elapsed_ms": ms, "record": got}, ensure_ascii=False) + "\n") - records.flush() - assert got["rows"] == baseline[i]["rows"], (mode["name"], case["name"], "prepared rows") - checks.append({"name": case["name"], **compare(reference[i]["response"], got["response"])}) - for i in selected: - got, ms = run(cases[i]["request"]) - records.write(json.dumps({"phase": "warmup", "name": cases[i]["name"], "elapsed_ms": ms, "record": got}, ensure_ascii=False) + "\n") - for repetition in range(plan["repetitions"]): - for i in selected: - got, ms = run(cases[i]["request"]) - sample = {"name": cases[i]["name"], "repetition": repetition, "elapsed_ms": ms} - samples.append(sample) - records.write(json.dumps({"phase": "measured", **sample, "record": got}, ensure_ascii=False) + "\n") - records.flush() - assert got["rows"] == baseline[i]["rows"], (mode["name"], cases[i]["name"], "measured rows") - compare(reference[i]["response"], got["response"]) - memory = subprocess.check_output(["nvidia-smi", "--query-gpu=memory.used", "--format=csv,noheader,nounits"], text=True).strip() - summary = {"name": mode["name"], "startup_to_empty_diagnostic_ms": startup_ms, - "checks": checks, "samples": samples, "device_used_mib_at_end": int(memory), - "warm_ms": {name: {"median": statistics.median(values), "min": min(values), "max": max(values)} - for name in plan["timed_cases"] - for values in [[s["elapsed_ms"] for s in samples if s["name"] == name]]}} - summaries.append(summary) - (directory / "summary.json").write_text(json.dumps(summary, indent=2) + "\n") - print(json.dumps(summary), flush=True) - finally: - child.stdin.close() - try: - child.wait(timeout=30) - except subprocess.TimeoutExpired: - child.kill() - child.wait() - (args.output / "summary.json").write_text(json.dumps({"modes": summaries}, indent=2) + "\n") - - -if __name__ == "__main__": - main() diff --git a/tests/decider/verify_reference.py b/tests/decider/verify_reference.py deleted file mode 100644 index 309504f0..00000000 --- a/tests/decider/verify_reference.py +++ /dev/null @@ -1,176 +0,0 @@ -#!/usr/bin/env python3 -"""Pinned full-checkpoint eager parity; no latency or task-quality claim.""" -import argparse -import gc -import hashlib -import json -import os -from pathlib import Path -import subprocess -import sys -import types - -GATES = {"max_probability_drift": 0.02, "max_score_drift": 0.1, "discrete_margin": 0.05} -REFERENCE_REVISION = "50d0be0d7cb43d2066965ce5fa7f3fe4e489a60f" -REFERENCE_FILES = { - "model.py": "74e0ca564bb52f7c2bd74c23cb34b16e0004ceadf7a13816f223c68829b5e1c5", - "prompt.py": "f21ae1016a11ca3500b5a8e2e8c3ef524434d196bbd4891199b96a33b3ad6314", - "prompt_fast.py": "fbfcc3a381a89785174cb9d29083ed14d404c7273d123ba8fe2b69584649ef25", - "systemone.py": "884e4b0bae36f5e09dbd689b4390e94320cbbd77f7b8d675e50490b1d7c61d34", - "temperature.py": "b940c253e5626145e806195cf5b2bc0de61461de633c07b174a924d18ceeab88", -} - - -def verify_sources(directory, expected): - verified = {} - for name, checksum in expected.items(): - actual = hashlib.sha256((directory / name).read_bytes()).hexdigest() - if actual != checksum: - raise ValueError(f"reference checksum mismatch: {name}") - verified[name] = actual - return verified - - -def workloads(): - cases = [] - for n in (2, 3, 10, 11, 26, 255): - cases.append((f"choice_{n}", {"state": "The customer asks for a refund for a damaged order.", "questions": { - "category": {"type": "choice", "instructions": "Choose the matching category.", "criteria": { - f"option_{i}": "Refund request" if i == n - 1 else f"Unrelated category {i}" for i in range(n)}}}})) - for n in (2, 3, 10): - cases.append((f"score_{n}", {"state": "The customer received a damaged item and requests help immediately.", "questions": { - "urgency": {"type": "score", "instructions": "How urgent is the customer request?", "criteria": [f"{i}. Urgency level {i}" for i in range(n)]}}})) - cases.extend([ - ("noul_true", {"state": "The customer requests a refund.", "questions": {"refund": {"type": "noul", "instructions": "The customer requests a refund."}}}), - ("noul_false", {"state": "The customer asks when the package will arrive.", "questions": {"refund": {"type": "noul", "instructions": "The customer requests a refund."}}}), - ("noul_descriptions", {"state": "The parcel is delivered.", "questions": {"delivered": {"type": "noul", "criteria": {"true": "The parcel is delivered.", "false": "The parcel is still in transit."}}}}), - ("structured_unicode", {"state": {"客户": "订单损坏，请退款", "orders": [{"id": i, "status": "damaged" if i == 7 else "delivered"} for i in range(9)]}, "questions": { - "z_last": {"type": "choice", "instructions": {"task": "Choose the status of orders[7]."}, "criteria": {"damaged": "Damaged", "delivered": "Delivered"}}, - "a_first": {"type": "score", "instructions": "Severity of the damage", "criteria": {"0": "Low", "1": "Medium", "2": "High"}}, - "m_middle": {"type": "noul", "instructions": "The customer requests a refund."}}}), - ("mixed", {"state": "A customer requests a refund because a delivery was damaged.", "questions": { - "route": {"type": "choice", "instructions": "Choose the team.", "criteria": {"billing": "Refund and billing", "delivery": "Shipping tracking", "sales": "New purchases"}}, - "refund": {"type": "noul", "instructions": "The customer requests a refund."}, - "severity": {"type": "score", "instructions": "Severity of the damage", "criteria": ["Minor", "Moderate", "Severe"]}}}), - ("long_context", {"state": "The item is damaged. " * 700 + "The customer requests a refund.", "questions": {"route": {"type": "choice", "instructions": "Choose the team.", "criteria": {"billing": "Refund and billing", "shipping": "Delivery status"}}}}), - ("empty", {"state": {}, "questions": {}}), - ]) - # Repetition and reverse question order test request/state isolation. - cases.append(("mixed_reordered", {"state": cases[13][1]["state"], "questions": dict(reversed(list(cases[13][1]["questions"].items())))})) - cases.append(("choice_repeat", cases[0][1])) - return cases - - -def compare(reference, native): - assert native["model"] == "decider-2b-v11" - assert native["usage"] == reference["usage"] - assert list(native["answers"]) == list(reference["answers"]) - worst_p, worst_score = 0.0, 0.0 - mismatches, low_margin = [], [] - for key, ref in reference["answers"].items(): - got = native["answers"][key] - assert set(got) == set(ref), (key, set(got), set(ref)) - assert got["type"] == ref["type"] - if ref["type"] == "noul": - drift = abs(got["noul"] - ref["noul"]) - else: - assert list(got["probabilities"]) == list(ref["probabilities"]) - drift = max(abs(got["probabilities"][k] - v) for k, v in ref["probabilities"].items()) - if ref["type"] == "choice": - probabilities = sorted(ref["probabilities"].values(), reverse=True) - if ref["choice"] != got["choice"]: - (mismatches if probabilities[0] - probabilities[1] >= GATES["discrete_margin"] else low_margin).append(key) - else: - assert got["legend"] == ref["legend"] - worst_score = max(worst_score, abs(got["score"] - ref["score"])) - drift = max(drift, max(abs(got["level_fit"][k] - v) for k, v in ref["level_fit"].items())) - worst_p = max(worst_p, drift) - assert worst_p <= GATES["max_probability_drift"], ("probability", worst_p) - assert worst_score <= GATES["max_score_drift"], ("score", worst_score) - assert not mismatches, ("discrete", mismatches) - return {"max_probability_drift": worst_p, "max_score_drift": worst_score, "low_margin_disagreements": low_margin} - - -def main(): - parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument("--model", type=Path, required=True) - parser.add_argument("--reference", type=Path, required=True) - parser.add_argument("--binary", type=Path, required=True) - parser.add_argument("--library", type=Path, required=True) - parser.add_argument("--output", type=Path, required=True) - args = parser.parse_args() - args.output.mkdir(parents=True, exist_ok=True) - sources = verify_sources(args.reference / "decider", REFERENCE_FILES) - cases = workloads() - manifest = {"reference_revision": REFERENCE_REVISION, "reference_source_sha256": sources, "gates": GATES, "runs": 1, "cases": [{"name": name, "request": request} for name, request in cases]} - # Freeze the manifest before executing either implementation. - (args.output / "protocol.json").write_text(json.dumps(manifest, ensure_ascii=False, indent=2) + "\n") - inputs = "".join(json.dumps(request, ensure_ascii=False) + "\n" for _, request in cases) - (args.output / "requests.jsonl").write_text(inputs) - with (args.output / "native.stderr.log").open("w") as stderr: - native = subprocess.run([str(args.binary), str(args.model), str(args.library)], input=inputs, capture_output=False, - stdout=subprocess.PIPE, stderr=stderr, text=True, check=True) - (args.output / "native.jsonl").write_text(native.stdout) - native = [json.loads(line) for line in native.stdout.splitlines()] - assert len(native) == len(cases) - - # Import only the pinned reference package, with no remote-code execution. - package = types.ModuleType("decider") - package.__path__ = [str(args.reference / "decider")] - sys.modules["decider"] = package - import torch - from transformers import AutoTokenizer - from decider import systemone as s1, prompt_fast, temperature as tt, prompt - from decider.model import DecisionModel - - torch.set_num_threads(4) - torch.backends.cuda.matmul.allow_tf32 = False - tokenizer = AutoTokenizer.from_pretrained(args.model, local_files_only=True) - scalar, temperatures = tt.from_config(json.loads((args.model / "decider_config.json").read_text()))[0] - del scalar - labels = prompt.label_table(tokenizer)[1] - model = DecisionModel(str(args.model), dtype=torch.bfloat16, grad_ckpt=False).to("cuda").eval() - model.lm.model.config.use_cache = False - results = [] - with (args.output / "reference.jsonl").open("w") as records, torch.inference_mode(): - for (name, request), got in zip(cases, native): - rendered = {key: s1.render_question(value) for key, value in request["questions"].items()} - plans, index = s1.plan_rows(rendered, True) - items, _ = prompt_fast.build_rows(tokenizer, s1.render_state(request["state"]), - [[(plan["question"], plan["options"])] for plan in plans], max_ctx_tokens=32768) - kinds = s1.row_types(rendered, index) - assert len(got["rows"]) == len(items), name - logits, probabilities = [], [] - max_logit_drift = 0.0 - for i, (item, kind) in enumerate(zip(items, kinds)): - count = item["nopts"][0] - row = got["rows"][i] - assert row["ids"] == item["ids"], (name, i, "tokens") - assert row["candidate_ids"] == labels[:count], (name, i, "candidate IDs") - assert row["readout_position"] == item["slots"][0] == len(item["ids"]) - 1 - ids = torch.tensor([item["ids"]], device="cuda", dtype=torch.long) - lg = model.slot_logits(ids, torch.ones_like(ids), torch.tensor([item["slots"][0]], device="cuda"), - torch.tensor([0], device="cuda"), torch.tensor([count], device="cuda"))[0, :count] - logits.append(lg.cpu().tolist()) - probabilities.append(tt.scaled_softmax(lg[None], [temperatures[kind]])[0].cpu().tolist()) - max_logit_drift = max(max_logit_drift, max(abs(a - b) for a, b in zip(logits[-1], got["logits"][i]))) - response = {"model": "decider-2b-v11", "answers": s1.assemble(rendered, index, probabilities), - "usage": {"input_tokens": s1.unique_tokens(items), "output_tokens": 0}} - records.write(json.dumps({"name": name, "logits": logits, "probabilities": probabilities, "response": response}, ensure_ascii=False) + "\n") - records.flush() - result = {"name": name, "rows": len(items), "processed_tokens": sum(len(item["ids"]) for item in items), - "max_logit_drift": max_logit_drift, **compare(response, got["response"])} - results.append(result) - print(json.dumps(result), flush=True) - del model - gc.collect() - torch.cuda.empty_cache() - summary = {"passed": len(results), "cases": results, "gates": GATES, - "torch": torch.__version__, "transformers": __import__("transformers").__version__, - "gpu": torch.cuda.get_device_name(), "native_sha256": hashlib.sha256(args.binary.read_bytes()).hexdigest(), - "library_sha256": hashlib.sha256(args.library.read_bytes()).hexdigest()} - (args.output / "summary.json").write_text(json.dumps(summary, indent=2) + "\n") - - -if __name__ == "__main__": - main() diff --git a/tests/decider/verify_retirement.py b/tests/decider/verify_retirement.py deleted file mode 100644 index 8bb5e125..00000000 --- a/tests/decider/verify_retirement.py +++ /dev/null @@ -1,68 +0,0 @@ -#!/usr/bin/env python3 -"""Fault-injection check using an isolated real worker after its genuine warmup.""" -import argparse -import json -import os -from pathlib import Path -import subprocess -import time -from verify_http import request, wait_ready - - -def memory(pid): - result = subprocess.run(["nvidia-smi", "--query-compute-apps=pid,used_gpu_memory", "--format=csv,noheader,nounits"], capture_output=True, text=True, check=True) - for line in result.stdout.splitlines(): - fields = line.split(",") - if fields[0].strip() == str(pid): - return int(fields[1].strip()) - return 0 - - -def main(): - parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument("--worker", type=Path, required=True) - parser.add_argument("--model", type=Path, required=True) - parser.add_argument("--wrapper", type=Path, required=True) - parser.add_argument("--output", type=Path, required=True) - parser.add_argument("--batched", action="store_true") - args = parser.parse_args() - args.output.mkdir(parents=True, exist_ok=True) - log = args.output / "worker.log" - environment = os.environ.copy() - environment.update(DECIDER_MODEL=str(args.model), DECIDER_CUDA_LIB=str(args.wrapper), DECIDER_PORT="18112", CUA_S1_GRAPH="0") - with log.open("w") as output: - child = subprocess.Popen([str(args.worker)], env=environment, stdout=output, stderr=subprocess.STDOUT) - try: - base = "http://127.0.0.1:18112" - health = wait_ready(base, child, log) - assert health["status"] == "ready" - before = memory(child.pid) - raw = b'{"state":"The item is damaged.","questions":{"q":{"instructions":"Choose a category.","criteria":["refund","shipping"]}}}' - if args.batched: - raw = b'{"state":"The item is damaged.","questions":{"q":{"type":"score","instructions":"Severity.","criteria":["low","high"]}}}' - status, body = request(base, "/v1/systemone", raw) - assert status == 503 and json.loads(body)["detail"] == "model inference failed" - status, body = request(base, "/health") - assert status == 503 and json.loads(body)["status"] == "unavailable" - for _ in range(2): - status, body = request(base, "/v1/systemone", raw) - assert status == 503 and json.loads(body)["detail"] == "model unavailable" - after = memory(child.pid) - # The checksum-pinned 3.76GB model allocation must actually be retired. - assert before - after > 3000, (before, after) - text = log.read_text() - assert text.count("Decider test projection") == 2 - assert "Decider test projection 2" in text - if args.batched: - assert "Decider test projection 2 rows 2" in text - result = {"passed": True, "projection_calls": 2, "health_after": "unavailable", "response_status": 503, - "sampled_before_mib": before, "sampled_after_mib": after, "later_requests_refused": 2} - (args.output / "summary.json").write_text(json.dumps(result, indent=2) + "\n") - print(json.dumps(result), flush=True) - finally: - child.terminate() - child.wait(timeout=30) - - -if __name__ == "__main__": - main() From 7eea74a9776fdce97917c6c9eab14da67c19c596 Mon Sep 17 00:00:00 2001 From: levius <2114377220@qq.com> Date: Thu, 8 Oct 2026 14:24:01 +0800 Subject: [PATCH 3/4] test: restore focused regression coverage --- src/models/decider/native/Cargo.toml | 20 +++ src/models/decider/native/src/checkpoint.rs | 4 + src/models/decider/native/src/executor.rs | 4 + tests/decider/batching.rs | 37 ++++ tests/decider/checkpoint.rs | 69 ++++++++ tests/decider/config.rs | 75 +++++++++ tests/decider/contract.rs | 93 +++++++++++ tests/decider/data/cpu.json | 1 + tests/decider/data/responses.json | 1 + tests/decider/executor.rs | 87 ++++++++++ tests/decider/fail_head.c | 18 ++ tests/decider/gpu.rs | 137 +++++++++++++++ tests/decider/runner.rs | 35 ++++ tests/decider/test_download.py | 28 ++++ tests/decider/test_reference.py | 29 ++++ tests/decider/verify_http.py | 124 ++++++++++++++ tests/decider/verify_modes.py | 104 ++++++++++++ tests/decider/verify_reference.py | 176 ++++++++++++++++++++ tests/decider/verify_retirement.py | 68 ++++++++ 19 files changed, 1110 insertions(+) create mode 100644 tests/decider/batching.rs create mode 100644 tests/decider/checkpoint.rs create mode 100644 tests/decider/config.rs create mode 100644 tests/decider/contract.rs create mode 100644 tests/decider/data/cpu.json create mode 100644 tests/decider/data/responses.json create mode 100644 tests/decider/executor.rs create mode 100644 tests/decider/fail_head.c create mode 100644 tests/decider/gpu.rs create mode 100644 tests/decider/runner.rs create mode 100644 tests/decider/test_download.py create mode 100644 tests/decider/test_reference.py create mode 100644 tests/decider/verify_http.py create mode 100644 tests/decider/verify_modes.py create mode 100644 tests/decider/verify_reference.py create mode 100644 tests/decider/verify_retirement.py diff --git a/src/models/decider/native/Cargo.toml b/src/models/decider/native/Cargo.toml index 5df4bfa6..c11f694d 100644 --- a/src/models/decider/native/Cargo.toml +++ b/src/models/decider/native/Cargo.toml @@ -21,3 +21,23 @@ tokio = { version = "1.49.0", features = ["macros", "net", "rt-multi-thread", "s [[bin]] name = "omni-decider" path = "src/main.rs" + +[[test]] +name = "gpu" +path = "../../../../tests/decider/gpu.rs" + +[[test]] +name = "contract" +path = "../../../../tests/decider/contract.rs" + +[[test]] +name = "config" +path = "../../../../tests/decider/config.rs" + +[[test]] +name = "batching" +path = "../../../../tests/decider/batching.rs" + +[[example]] +name = "decider-run" +path = "../../../../tests/decider/runner.rs" diff --git a/src/models/decider/native/src/checkpoint.rs b/src/models/decider/native/src/checkpoint.rs index 14bd30da..05051eac 100644 --- a/src/models/decider/native/src/checkpoint.rs +++ b/src/models/decider/native/src/checkpoint.rs @@ -143,3 +143,7 @@ fn selected_rows( } Ok(rows) } + +#[cfg(test)] +#[path = "../../../../../tests/decider/checkpoint.rs"] +mod tests; diff --git a/src/models/decider/native/src/executor.rs b/src/models/decider/native/src/executor.rs index 10d2f8ac..bd832fce 100644 --- a/src/models/decider/native/src/executor.rs +++ b/src/models/decider/native/src/executor.rs @@ -288,3 +288,7 @@ fn validate_rows(rows: &[RowInput], labels: &[u32]) -> Result<()> { } Ok(()) } + +#[cfg(test)] +#[path = "../../../../../tests/decider/executor.rs"] +mod tests; diff --git a/tests/decider/batching.rs b/tests/decider/batching.rs new file mode 100644 index 00000000..becc198b --- /dev/null +++ b/tests/decider/batching.rs @@ -0,0 +1,37 @@ +use omni_decider_native::batching::BatchLimits; +#[test] +fn bounded_contiguous_ranges_preserve_order_and_long_rows() { + let limits = BatchLimits::new(3, 8).unwrap(); + assert_eq!( + limits.ranges(&[2, 3, 3, 1, 9, 2, 2, 2, 2]), + vec![0..3, 3..4, 4..5, 5..8, 8..9] + ); + assert!(limits.ranges(&[]).is_empty()); + assert_eq!(limits.ranges(&[9, 9]), vec![0..1, 1..2]); +} +#[test] +fn row_limit_and_exact_token_boundary_are_independent() { + assert_eq!( + BatchLimits::new(2, 4096).unwrap().ranges(&[1, 1, 1]), + vec![0..2, 2..3] + ); + assert_eq!( + BatchLimits::new(4, 4096).unwrap().ranges(&[2048, 2048, 1]), + vec![0..2, 2..3] + ); + assert_eq!( + BatchLimits::default().ranges(&[1, 2, 3]), + vec![0..1, 1..2, 2..3] + ); +} +#[test] +fn startup_limits_reject_unbounded_or_malformed_settings() { + for (rows, tokens) in [(0, 1), (5, 1), (8, 1), (16, 1), (1, 0), (1, 4097)] { + assert!(BatchLimits::new(rows, tokens).is_err()); + } + assert!(BatchLimits::from_values(Some("wrong"), None).is_err()); + assert!(BatchLimits::from_values(None, Some("-1")).is_err()); + let limits = BatchLimits::from_values(Some("4"), Some("4096")).unwrap(); + assert_eq!(limits.max_rows(), 4); + assert_eq!(limits.max_tokens(), 4096); +} diff --git a/tests/decider/checkpoint.rs b/tests/decider/checkpoint.rs new file mode 100644 index 00000000..a32746f6 --- /dev/null +++ b/tests/decider/checkpoint.rs @@ -0,0 +1,69 @@ +use super::*; +use safetensors::{Dtype, tensor::TensorView}; +use std::collections::HashMap; + +fn tensor(dtype: Dtype, shape: Vec, bytes: &[u8]) -> Vec { + let view = TensorView::new(dtype, shape, bytes).unwrap(); + safetensors::serialize( + HashMap::from([("model.language_model.embed_tokens.weight", view)]), + None, + ) + .unwrap() +} + +#[test] +fn selects_requested_tied_rows_and_zero_pads_the_head() { + let bytes: Vec = (0..12) + .flat_map(|i| half::bf16::from_f32(i as f32).to_le_bytes()) + .collect(); + let encoded = tensor(Dtype::BF16, vec![3, 4], &bytes); + let st = SafeTensors::deserialize(&encoded).unwrap(); + let selected = selected_rows(&st, &[2, 0], 4, 3).unwrap(); + assert_eq!(&selected[..8], &bytes[16..24]); + assert_eq!(&selected[8..16], &bytes[..8]); + assert_eq!(selected.len(), 8 * 4 * 2); + assert!(selected[16..].iter().all(|&b| b == 0)); +} + +#[test] +fn rejects_invalid_head_layouts_ids_and_nonfinite_weights() { + let bytes = vec![0; 24]; + for (dtype, shape, data) in [ + (Dtype::BF16, vec![3, 4], bytes.clone()), + (Dtype::F32, vec![3, 2], bytes.clone()), + (Dtype::BF16, vec![2, 6], bytes.clone()), + ] { + let encoded = tensor(dtype, shape, &data); + let st = SafeTensors::deserialize(&encoded).unwrap(); + assert!(selected_rows(&st, &[3], 4, 3).is_err()); + } + let mut bytes = bytes; + bytes[..2].copy_from_slice(&half::bf16::NAN.to_le_bytes()); + let encoded = tensor(Dtype::BF16, vec![3, 4], &bytes); + let st = SafeTensors::deserialize(&encoded).unwrap(); + assert!(selected_rows(&st, &[0], 4, 3).is_err()); + assert!(selected_rows(&st, &[], 4, 3).is_err()); + assert!(selected_rows(&st, &[1, 1], 4, 3).is_err()); +} + +#[test] +fn checks_actual_file_content_and_length() { + let path = std::env::temp_dir().join(format!("decider-hash-{}", std::process::id())); + std::fs::write(&path, b"abc").unwrap(); + let digest = "ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad"; + verify_file(&path, 3, digest).unwrap(); + assert!(verify_file(&path, 4, digest).is_err()); + assert!(verify_file(&path, 3, &"0".repeat(64)).is_err()); + std::fs::remove_file(path).unwrap(); +} + +#[test] +fn refuses_an_index_that_would_override_the_verified_single_file() { + let dir = std::env::temp_dir().join(format!("decider-index-{}", std::process::id())); + std::fs::create_dir_all(&dir).unwrap(); + std::fs::write(dir.join("model.safetensors.index.json"), b"{}").unwrap(); + let labels: Vec = (0..255).collect(); + let error = Checkpoint::load(&dir, &labels).err().unwrap(); + std::fs::remove_dir_all(dir).unwrap(); + assert!(error.to_string().contains("sharded"), "{error}"); +} diff --git a/tests/decider/config.rs b/tests/decider/config.rs new file mode 100644 index 00000000..29c670a2 --- /dev/null +++ b/tests/decider/config.rs @@ -0,0 +1,75 @@ +use omni_decider_native::{Config, Kind}; +use serde_json::{Value, json}; +use std::{ + path::PathBuf, + sync::atomic::{AtomicU64, Ordering}, +}; + +fn plain() -> Value { + json!({"version":"2b-v11","layout":"plain","max_options":255,"max_state_tokens":32768,"neutralize_none":false,"isolated_levels":true,"temperature":1.0}) +} +struct Directory(PathBuf); +impl Directory { + fn new() -> Self { + static NEXT: AtomicU64 = AtomicU64::new(0); + let path = std::env::temp_dir().join(format!( + "decider-config-{}-{}", + std::process::id(), + NEXT.fetch_add(1, Ordering::Relaxed) + )); + std::fs::create_dir(&path).unwrap(); + std::fs::write(path.join("config.json"),json!({"model_type":"qwen3_5_text","hidden_size":2048,"num_hidden_layers":24,"tie_word_embeddings":true,"vocab_size":248320,"dtype":"bfloat16"}).to_string()).unwrap(); + Self(path) + } + fn calibration(&self, config: &Value) { + std::fs::write(self.0.join("decider_config.json"), config.to_string()).unwrap(); + } +} +impl Drop for Directory { + fn drop(&mut self) { + std::fs::remove_dir_all(&self.0).unwrap(); + } +} +#[test] +fn public_validator_rejects_chat_metadata_on_plain_layout() { + let mut config = plain(); + config["chat_template"] = json!(true); + let error = + Config::from_value(&config).expect_err("contradictory chat template must be rejected"); + assert!(error.to_string().contains("chat_template")); +} +#[test] +fn file_loader_rejects_chat_metadata_on_plain_layout() { + let dir = Directory::new(); + let mut config = plain(); + config["chat_template"] = json!(true); + dir.calibration(&config); + let error = Config::load(&dir.0).expect_err("file-loading path must reject contradiction"); + assert!(error.to_string().contains("chat_template")); +} +#[test] +fn missing_and_disabled_chat_metadata_preserve_plain_calibration() { + let dir = Directory::new(); + for config in [plain(), { + let mut c = plain(); + c["chat_template"] = json!(false); + c + }] { + assert_eq!( + Config::from_value(&config) + .unwrap() + .temperature(Kind::Choice), + 1.0 + ); + dir.calibration(&config); + assert_eq!(Config::load(&dir.0).unwrap().temperature(Kind::Score), 1.0); + } +} +#[test] +fn unsupported_chat_metadata_types_are_rejected() { + for value in [json!("true"), json!(1), json!(null), json!([]), json!({})] { + let mut config = plain(); + config["chat_template"] = value; + assert!(Config::from_value(&config).is_err()); + } +} diff --git a/tests/decider/contract.rs b/tests/decider/contract.rs new file mode 100644 index 00000000..344ce5e9 --- /dev/null +++ b/tests/decider/contract.rs @@ -0,0 +1,93 @@ +use omni_decider_native::{Limits, Processor}; +use serde_json::{Value, json}; + +fn processor() -> Processor { + Processor::load( + std::env::var("DECIDER_MODEL").expect("set DECIDER_MODEL"), + Limits::default(), + ) + .unwrap() +} +#[test] +#[ignore = "requires the pinned tokenizer/config; CPU only, no tensor loading"] +fn pinned_tokens_rows_usage_and_all_labels() { + let processor = processor(); + let golden: Value = serde_json::from_str(include_str!("data/cpu.json")).unwrap(); + let labels: Vec = serde_json::from_value(golden["label_ids"].clone()).unwrap(); + assert_eq!( + processor + .labels() + .iter() + .map(|label| label.id) + .collect::>(), + labels + ); + assert_eq!(processor.labels().last().unwrap().name, "JT"); + for fixture in golden["fixtures"].as_array().unwrap() { + let request = json!({"state":fixture["state"],"questions":fixture["questions"]}); + let prepared = processor.prepare(request.to_string().as_bytes()).unwrap(); + assert_eq!( + prepared.rows.len(), + fixture["items"].as_array().unwrap().len() + ); + assert_eq!( + prepared.context.input_tokens(), + fixture["unique_tokens"].as_u64().unwrap() as usize + ); + assert_eq!( + prepared.processed_tokens(), + fixture["processed_tokens"].as_u64().unwrap() as usize + ); + for (row, item) in prepared + .rows + .iter() + .zip(fixture["items"].as_array().unwrap()) + { + assert_eq!( + row.ids, + serde_json::from_value::>(item["ids"].clone()).unwrap(), + "{}", + fixture["name"] + ); + let count = item["nopts"][0].as_u64().unwrap() as usize; + assert_eq!(row.candidate_ids, labels[..count]); + assert_eq!(row.readout_position, row.ids.len() - 1); + } + } +} +#[test] +#[ignore = "requires the pinned tokenizer/config; CPU only, no tensor loading"] +fn pinned_assemblies_and_state_truncation() { + let processor = processor(); + let golden: Value = serde_json::from_str(include_str!("data/responses.json")).unwrap(); + for fixture in golden.as_array().unwrap() { + let prepared = processor + .prepare(fixture["request"].to_string().as_bytes()) + .unwrap(); + let logits = serde_json::from_value(fixture["logits"].clone()).unwrap(); + assert_eq!( + prepared.context.finish(logits).unwrap()["answers"], + fixture["expected_answers"], + "{}", + fixture["name"] + ); + } + let request = json!({"state":"hello ".repeat(40000),"questions":{"q":{"instructions":"Choose.","criteria":["A","B"]}}}); + let row = processor + .prepare(request.to_string().as_bytes()) + .unwrap() + .rows + .pop() + .unwrap(); + assert!(row.ids.len() > 32768 && row.ids.len() <= 36864); + let first = row.ids[..32768].to_vec(); + let request = json!({"state":"hello ".repeat(45000),"questions":{"q":{"instructions":"Choose.","criteria":["A","B"]}}}); + assert_eq!( + processor + .prepare(request.to_string().as_bytes()) + .unwrap() + .rows[0] + .ids[..32768], + first + ); +} diff --git a/tests/decider/data/cpu.json b/tests/decider/data/cpu.json new file mode 100644 index 00000000..13e2ec7a --- /dev/null +++ b/tests/decider/data/cpu.json @@ -0,0 +1 @@ +{"label_ids":[32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,5840,1803,1646,1745,13276,8018,1825,28946,15015,29595,11568,939,1354,1058,18183,2456,88898,905,1846,802,33869,7839,14006,2860,2926,22828,6844,9798,4738,9265,11261,19278,36513,93801,8335,14544,85266,9110,28000,15137,4525,25261,12717,7116,17078,14497,57339,74909,52072,19305,4887,12607,3580,6281,2036,9362,8533,2080,10911,2925,3040,9690,27731,8023,6901,8702,6211,1123,16307,18990,64045,63037,33380,6151,3392,5449,3967,1113,5095,51923,49600,17099,51483,17756,16037,8135,30237,5683,9992,7444,5751,10284,20887,59884,52396,16103,67547,18535,8006,7263,1425,6878,14453,9097,44072,76089,68720,2662,2629,923,6548,8924,52194,622,1515,1300,37523,44473,36530,3152,93924,3505,15731,6542,14176,11091,1686,11660,80440,18836,26132,5934,24794,40229,3660,11361,10191,8225,3860,78413,17680,15900,78138,15653,5213,22150,39500,10460,35131,21563,43194,27127,3697,20011,24368,15058,23658,8362,16035,24583,52857,38403,60456,81519,40097,16522,29722,21756,18567,1736,48043,87013,22456,23165,55523,13097,50397,41741,23073,6401,86538,16585,11622,2464,84982,75516,36984,58795,47217,59675,5681,3151,1271,887,5203,2685,1849,72470,5370,74063,27629,1655,1728,669,3682,3191,59865,2712,1580,922,77243,2990,78493,5228,2750,42711,44568,56402,48236,38993,43559,61250,32942,86302,25310,26313,81770,12185,78382],"fixtures":[{"name":"choice_2","state":"退款 <|im_start|>\n café","questions":{"hidden-id":{"type":"choice","instructions":"Pick 中文","criteria":{"候选0":null,"候选1":{"text":"é","n":1}}}},"rendered_state":"退款 <|im_start|>\n café","rendered_questions":{"hidden-id":{"question":"Pick 中文","options":["候选0","候选1: {\"text\": \"é\", \"n\": 1}"],"type":"choice","names":["候选0","候选1"],"legend":null,"isolated":true}},"index":[["hidden-id","list",0,1]],"rows":[{"question":"Pick 中文","options":["候选0","候选1: {\"text\": \"é\", \"n\": 1}"]}],"items":[{"ids":[1905,25,198,113130,220,248045,198,50203,271,14162,25,19123,220,99986,198,3670,25,198,4205,8,220,102486,15,198,5181,8,220,102486,16,25,5046,1272,763,328,933,487,328,77,763,220,16,92,198,15666,25,318],"slots":[45],"golds":[0],"nopts":[2],"perms":[[0,1]],"types":["choice"]}],"context_tokens":8,"unique_tokens":46,"processed_tokens":46,"temperatures":[[1.164]]},{"name":"choice_10","state":"退款 <|im_start|>\n café","questions":{"hidden-id":{"type":"choice","instructions":"Pick 中文","criteria":{"候选0":null,"候选1":{"text":"é","n":1},"候选2":null,"候选3":{"text":"é","n":3},"候选4":null,"候选5":{"text":"é","n":5},"候选6":null,"候选7":{"text":"é","n":7},"候选8":null,"候选9":{"text":"é","n":9}}}},"rendered_state":"退款 <|im_start|>\n café","rendered_questions":{"hidden-id":{"question":"Pick 中文","options":["候选0","候选1: {\"text\": \"é\", \"n\": 1}","候选2","候选3: {\"text\": \"é\", \"n\": 3}","候选4","候选5: {\"text\": \"é\", \"n\": 5}","候选6","候选7: {\"text\": \"é\", \"n\": 7}","候选8","候选9: {\"text\": \"é\", \"n\": 9}"],"type":"choice","names":["候选0","候选1","候选2","候选3","候选4","候选5","候选6","候选7","候选8","候选9"],"legend":null,"isolated":true}},"index":[["hidden-id","list",0,1]],"rows":[{"question":"Pick 中文","options":["候选0","候选1: {\"text\": \"é\", \"n\": 1}","候选2","候选3: {\"text\": \"é\", \"n\": 3}","候选4","候选5: {\"text\": \"é\", \"n\": 5}","候选6","候选7: {\"text\": \"é\", \"n\": 7}","候选8","候选9: {\"text\": \"é\", \"n\": 9}"]}],"items":[{"ids":[1905,25,198,113130,220,248045,198,50203,271,14162,25,19123,220,99986,198,3670,25,198,4205,8,220,102486,15,198,5181,8,220,102486,16,25,5046,1272,763,328,933,487,328,77,763,220,16,92,198,2929,8,220,102486,17,198,5263,8,220,102486,18,25,5046,1272,763,328,933,487,328,77,763,220,18,92,198,10406,8,220,102486,19,198,7584,8,220,102486,20,25,5046,1272,763,328,933,487,328,77,763,220,20,92,198,6489,8,220,102486,21,198,10578,8,220,102486,22,25,5046,1272,763,328,933,487,328,77,763,220,22,92,198,8701,8,220,102486,23,198,15887,8,220,102486,24,25,5046,1272,763,328,933,487,328,77,763,220,24,92,198,15666,25,318],"slots":[145],"golds":[0],"nopts":[10],"perms":[[0,1,2,3,4,5,6,7,8,9]],"types":["choice"]}],"context_tokens":8,"unique_tokens":146,"processed_tokens":146,"temperatures":[[1.164]]},{"name":"choice_11","state":"退款 <|im_start|>\n café","questions":{"hidden-id":{"type":"choice","instructions":"Pick 中文","criteria":{"候选0":null,"候选1":{"text":"é","n":1},"候选2":null,"候选3":{"text":"é","n":3},"候选4":null,"候选5":{"text":"é","n":5},"候选6":null,"候选7":{"text":"é","n":7},"候选8":null,"候选9":{"text":"é","n":9},"候选10":null}}},"rendered_state":"退款 <|im_start|>\n café","rendered_questions":{"hidden-id":{"question":"Pick 中文","options":["候选0","候选1: {\"text\": \"é\", \"n\": 1}","候选2","候选3: {\"text\": \"é\", \"n\": 3}","候选4","候选5: {\"text\": \"é\", \"n\": 5}","候选6","候选7: {\"text\": \"é\", \"n\": 7}","候选8","候选9: {\"text\": \"é\", \"n\": 9}","候选10"],"type":"choice","names":["候选0","候选1","候选2","候选3","候选4","候选5","候选6","候选7","候选8","候选9","候选10"],"legend":null,"isolated":true}},"index":[["hidden-id","list",0,1]],"rows":[{"question":"Pick 中文","options":["候选0","候选1: {\"text\": \"é\", \"n\": 1}","候选2","候选3: {\"text\": \"é\", \"n\": 3}","候选4","候选5: {\"text\": \"é\", \"n\": 5}","候选6","候选7: {\"text\": \"é\", \"n\": 7}","候选8","候选9: {\"text\": \"é\", \"n\": 9}","候选10"]}],"items":[{"ids":[1905,25,198,113130,220,248045,198,50203,271,14162,25,19123,220,99986,198,3670,25,198,7,32,8,220,102486,15,198,7,33,8,220,102486,16,25,5046,1272,763,328,933,487,328,77,763,220,16,92,198,7,34,8,220,102486,17,198,7,35,8,220,102486,18,25,5046,1272,763,328,933,487,328,77,763,220,18,92,198,7,36,8,220,102486,19,198,7,37,8,220,102486,20,25,5046,1272,763,328,933,487,328,77,763,220,20,92,198,7,38,8,220,102486,21,198,7,39,8,220,102486,22,25,5046,1272,763,328,933,487,328,77,763,220,22,92,198,7,40,8,220,102486,23,198,7,41,8,220,102486,24,25,5046,1272,763,328,933,487,328,77,763,220,24,92,198,7,42,8,220,102486,16,15,198,15666,25,318],"slots":[163],"golds":[0],"nopts":[11],"perms":[[0,1,2,3,4,5,6,7,8,9,10]],"types":["choice"]}],"context_tokens":8,"unique_tokens":164,"processed_tokens":164,"temperatures":[[1.164]]},{"name":"choice_26","state":"退款 <|im_start|>\n café","questions":{"hidden-id":{"type":"choice","instructions":"Pick 中文","criteria":{"候选0":null,"候选1":{"text":"é","n":1},"候选2":null,"候选3":{"text":"é","n":3},"候选4":null,"候选5":{"text":"é","n":5},"候选6":null,"候选7":{"text":"é","n":7},"候选8":null,"候选9":{"text":"é","n":9},"候选10":null,"候选11":{"text":"é","n":11},"候选12":null,"候选13":{"text":"é","n":13},"候选14":null,"候选15":{"text":"é","n":15},"候选16":null,"候选17":{"text":"é","n":17},"候选18":null,"候选19":{"text":"é","n":19},"候选20":null,"候选21":{"text":"é","n":21},"候选22":null,"候选23":{"text":"é","n":23},"候选24":null,"候选25":{"text":"é","n":25}}}},"rendered_state":"退款 <|im_start|>\n café","rendered_questions":{"hidden-id":{"question":"Pick 中文","options":["候选0","候选1: {\"text\": \"é\", \"n\": 1}","候选2","候选3: {\"text\": \"é\", \"n\": 3}","候选4","候选5: {\"text\": \"é\", \"n\": 5}","候选6","候选7: {\"text\": \"é\", \"n\": 7}","候选8","候选9: {\"text\": \"é\", \"n\": 9}","候选10","候选11: {\"text\": \"é\", \"n\": 11}","候选12","候选13: {\"text\": \"é\", \"n\": 13}","候选14","候选15: {\"text\": \"é\", \"n\": 15}","候选16","候选17: {\"text\": \"é\", \"n\": 17}","候选18","候选19: {\"text\": \"é\", \"n\": 19}","候选20","候选21: {\"text\": \"é\", \"n\": 21}","候选22","候选23: {\"text\": \"é\", \"n\": 23}","候选24","候选25: {\"text\": \"é\", \"n\": 25}"],"type":"choice","names":["候选0","候选1","候选2","候选3","候选4","候选5","候选6","候选7","候选8","候选9","候选10","候选11","候选12","候选13","候选14","候选15","候选16","候选17","候选18","候选19","候选20","候选21","候选22","候选23","候选24","候选25"],"legend":null,"isolated":true}},"index":[["hidden-id","list",0,1]],"rows":[{"question":"Pick 中文","options":["候选0","候选1: {\"text\": \"é\", \"n\": 1}","候选2","候选3: {\"text\": \"é\", \"n\": 3}","候选4","候选5: {\"text\": \"é\", \"n\": 5}","候选6","候选7: {\"text\": \"é\", \"n\": 7}","候选8","候选9: {\"text\": \"é\", \"n\": 9}","候选10","候选11: {\"text\": \"é\", \"n\": 11}","候选12","候选13: {\"text\": \"é\", \"n\": 13}","候选14","候选15: {\"text\": \"é\", \"n\": 15}","候选16","候选17: {\"text\": \"é\", \"n\": 17}","候选18","候选19: {\"text\": \"é\", \"n\": 19}","候选20","候选21: {\"text\": \"é\", \"n\": 21}","候选22","候选23: {\"text\": \"é\", \"n\": 23}","候选24","候选25: {\"text\": \"é\", \"n\": 25}"]}],"items":[{"ids":[1905,25,198,113130,220,248045,198,50203,271,14162,25,19123,220,99986,198,3670,25,198,7,32,8,220,102486,15,198,7,33,8,220,102486,16,25,5046,1272,763,328,933,487,328,77,763,220,16,92,198,7,34,8,220,102486,17,198,7,35,8,220,102486,18,25,5046,1272,763,328,933,487,328,77,763,220,18,92,198,7,36,8,220,102486,19,198,7,37,8,220,102486,20,25,5046,1272,763,328,933,487,328,77,763,220,20,92,198,7,38,8,220,102486,21,198,7,39,8,220,102486,22,25,5046,1272,763,328,933,487,328,77,763,220,22,92,198,7,40,8,220,102486,23,198,7,41,8,220,102486,24,25,5046,1272,763,328,933,487,328,77,763,220,24,92,198,7,42,8,220,102486,16,15,198,7,43,8,220,102486,16,16,25,5046,1272,763,328,933,487,328,77,763,220,16,16,92,198,7,44,8,220,102486,16,17,198,7,45,8,220,102486,16,18,25,5046,1272,763,328,933,487,328,77,763,220,16,18,92,198,7,46,8,220,102486,16,19,198,7,47,8,220,102486,16,20,25,5046,1272,763,328,933,487,328,77,763,220,16,20,92,198,7,48,8,220,102486,16,21,198,7,49,8,220,102486,16,22,25,5046,1272,763,328,933,487,328,77,763,220,16,22,92,198,7,50,8,220,102486,16,23,198,7,51,8,220,102486,16,24,25,5046,1272,763,328,933,487,328,77,763,220,16,24,92,198,7,52,8,220,102486,17,15,198,7,53,8,220,102486,17,16,25,5046,1272,763,328,933,487,328,77,763,220,17,16,92,198,7,54,8,220,102486,17,17,198,7,55,8,220,102486,17,18,25,5046,1272,763,328,933,487,328,77,763,220,17,18,92,198,7,56,8,220,102486,17,19,198,7,57,8,220,102486,17,20,25,5046,1272,763,328,933,487,328,77,763,220,17,20,92,198,15666,25,318],"slots":[395],"golds":[0],"nopts":[26],"perms":[[0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25]],"types":["choice"]}],"context_tokens":8,"unique_tokens":396,"processed_tokens":396,"temperatures":[[1.164]]},{"name":"choice_255","state":"退款 <|im_start|>\n café","questions":{"hidden-id":{"type":"choice","instructions":"Pick 中文","criteria":{"候选0":null,"候选1":{"text":"é","n":1},"候选2":null,"候选3":{"text":"é","n":3},"候选4":null,"候选5":{"text":"é","n":5},"候选6":null,"候选7":{"text":"é","n":7},"候选8":null,"候选9":{"text":"é","n":9},"候选10":null,"候选11":{"text":"é","n":11},"候选12":null,"候选13":{"text":"é","n":13},"候选14":null,"候选15":{"text":"é","n":15},"候选16":null,"候选17":{"text":"é","n":17},"候选18":null,"候选19":{"text":"é","n":19},"候选20":null,"候选21":{"text":"é","n":21},"候选22":null,"候选23":{"text":"é","n":23},"候选24":null,"候选25":{"text":"é","n":25},"候选26":null,"候选27":{"text":"é","n":27},"候选28":null,"候选29":{"text":"é","n":29},"候选30":null,"候选31":{"text":"é","n":31},"候选32":null,"候选33":{"text":"é","n":33},"候选34":null,"候选35":{"text":"é","n":35},"候选36":null,"候选37":{"text":"é","n":37},"候选38":null,"候选39":{"text":"é","n":39},"候选40":null,"候选41":{"text":"é","n":41},"候选42":null,"候选43":{"text":"é","n":43},"候选44":null,"候选45":{"text":"é","n":45},"候选46":null,"候选47":{"text":"é","n":47},"候选48":null,"候选49":{"text":"é","n":49},"候选50":null,"候选51":{"text":"é","n":51},"候选52":null,"候选53":{"text":"é","n":53},"候选54":null,"候选55":{"text":"é","n":55},"候选56":null,"候选57":{"text":"é","n":57},"候选58":null,"候选59":{"text":"é","n":59},"候选60":null,"候选61":{"text":"é","n":61},"候选62":null,"候选63":{"text":"é","n":63},"候选64":null,"候选65":{"text":"é","n":65},"候选66":null,"候选67":{"text":"é","n":67},"候选68":null,"候选69":{"text":"é","n":69},"候选70":null,"候选71":{"text":"é","n":71},"候选72":null,"候选73":{"text":"é","n":73},"候选74":null,"候选75":{"text":"é","n":75},"候选76":null,"候选77":{"text":"é","n":77},"候选78":null,"候选79":{"text":"é","n":79},"候选80":null,"候选81":{"text":"é","n":81},"候选82":null,"候选83":{"text":"é","n":83},"候选84":null,"候选85":{"text":"é","n":85},"候选86":null,"候选87":{"text":"é","n":87},"候选88":null,"候选89":{"text":"é","n":89},"候选90":null,"候选91":{"text":"é","n":91},"候选92":null,"候选93":{"text":"é","n":93},"候选94":null,"候选95":{"text":"é","n":95},"候选96":null,"候选97":{"text":"é","n":97},"候选98":null,"候选99":{"text":"é","n":99},"候选100":null,"候选101":{"text":"é","n":101},"候选102":null,"候选103":{"text":"é","n":103},"候选104":null,"候选105":{"text":"é","n":105},"候选106":null,"候选107":{"text":"é","n":107},"候选108":null,"候选109":{"text":"é","n":109},"候选110":null,"候选111":{"text":"é","n":111},"候选112":null,"候选113":{"text":"é","n":113},"候选114":null,"候选115":{"text":"é","n":115},"候选116":null,"候选117":{"text":"é","n":117},"候选118":null,"候选119":{"text":"é","n":119},"候选120":null,"候选121":{"text":"é","n":121},"候选122":null,"候选123":{"text":"é","n":123},"候选124":null,"候选125":{"text":"é","n":125},"候选126":null,"候选127":{"text":"é","n":127},"候选128":null,"候选129":{"text":"é","n":129},"候选130":null,"候选131":{"text":"é","n":131},"候选132":null,"候选133":{"text":"é","n":133},"候选134":null,"候选135":{"text":"é","n":135},"候选136":null,"候选137":{"text":"é","n":137},"候选138":null,"候选139":{"text":"é","n":139},"候选140":null,"候选141":{"text":"é","n":141},"候选142":null,"候选143":{"text":"é","n":143},"候选144":null,"候选145":{"text":"é","n":145},"候选146":null,"候选147":{"text":"é","n":147},"候选148":null,"候选149":{"text":"é","n":149},"候选150":null,"候选151":{"text":"é","n":151},"候选152":null,"候选153":{"text":"é","n":153},"候选154":null,"候选155":{"text":"é","n":155},"候选156":null,"候选157":{"text":"é","n":157},"候选158":null,"候选159":{"text":"é","n":159},"候选160":null,"候选161":{"text":"é","n":161},"候选162":null,"候选163":{"text":"é","n":163},"候选164":null,"候选165":{"text":"é","n":165},"候选166":null,"候选167":{"text":"é","n":167},"候选168":null,"候选169":{"text":"é","n":169},"候选170":null,"候选171":{"text":"é","n":171},"候选172":null,"候选173":{"text":"é","n":173},"候选174":null,"候选175":{"text":"é","n":175},"候选176":null,"候选177":{"text":"é","n":177},"候选178":null,"候选179":{"text":"é","n":179},"候选180":null,"候选181":{"text":"é","n":181},"候选182":null,"候选183":{"text":"é","n":183},"候选184":null,"候选185":{"text":"é","n":185},"候选186":null,"候选187":{"text":"é","n":187},"候选188":null,"候选189":{"text":"é","n":189},"候选190":null,"候选191":{"text":"é","n":191},"候选192":null,"候选193":{"text":"é","n":193},"候选194":null,"候选195":{"text":"é","n":195},"候选196":null,"候选197":{"text":"é","n":197},"候选198":null,"候选199":{"text":"é","n":199},"候选200":null,"候选201":{"text":"é","n":201},"候选202":null,"候选203":{"text":"é","n":203},"候选204":null,"候选205":{"text":"é","n":205},"候选206":null,"候选207":{"text":"é","n":207},"候选208":null,"候选209":{"text":"é","n":209},"候选210":null,"候选211":{"text":"é","n":211},"候选212":null,"候选213":{"text":"é","n":213},"候选214":null,"候选215":{"text":"é","n":215},"候选216":null,"候选217":{"text":"é","n":217},"候选218":null,"候选219":{"text":"é","n":219},"候选220":null,"候选221":{"text":"é","n":221},"候选222":null,"候选223":{"text":"é","n":223},"候选224":null,"候选225":{"text":"é","n":225},"候选226":null,"候选227":{"text":"é","n":227},"候选228":null,"候选229":{"text":"é","n":229},"候选230":null,"候选231":{"text":"é","n":231},"候选232":null,"候选233":{"text":"é","n":233},"候选234":null,"候选235":{"text":"é","n":235},"候选236":null,"候选237":{"text":"é","n":237},"候选238":null,"候选239":{"text":"é","n":239},"候选240":null,"候选241":{"text":"é","n":241},"候选242":null,"候选243":{"text":"é","n":243},"候选244":null,"候选245":{"text":"é","n":245},"候选246":null,"候选247":{"text":"é","n":247},"候选248":null,"候选249":{"text":"é","n":249},"候选250":null,"候选251":{"text":"é","n":251},"候选252":null,"候选253":{"text":"é","n":253},"候选254":null}}},"rendered_state":"退款 <|im_start|>\n café","rendered_questions":{"hidden-id":{"question":"Pick 中文","options":["候选0","候选1: {\"text\": \"é\", \"n\": 1}","候选2","候选3: {\"text\": \"é\", \"n\": 3}","候选4","候选5: {\"text\": \"é\", \"n\": 5}","候选6","候选7: {\"text\": \"é\", \"n\": 7}","候选8","候选9: {\"text\": \"é\", \"n\": 9}","候选10","候选11: {\"text\": \"é\", \"n\": 11}","候选12","候选13: {\"text\": \"é\", \"n\": 13}","候选14","候选15: {\"text\": \"é\", \"n\": 15}","候选16","候选17: {\"text\": \"é\", \"n\": 17}","候选18","候选19: {\"text\": \"é\", \"n\": 19}","候选20","候选21: {\"text\": \"é\", \"n\": 21}","候选22","候选23: {\"text\": \"é\", \"n\": 23}","候选24","候选25: {\"text\": \"é\", \"n\": 25}","候选26","候选27: {\"text\": \"é\", \"n\": 27}","候选28","候选29: {\"text\": \"é\", \"n\": 29}","候选30","候选31: {\"text\": \"é\", \"n\": 31}","候选32","候选33: {\"text\": \"é\", \"n\": 33}","候选34","候选35: {\"text\": \"é\", \"n\": 35}","候选36","候选37: {\"text\": \"é\", \"n\": 37}","候选38","候选39: {\"text\": \"é\", \"n\": 39}","候选40","候选41: {\"text\": \"é\", \"n\": 41}","候选42","候选43: {\"text\": \"é\", \"n\": 43}","候选44","候选45: {\"text\": \"é\", \"n\": 45}","候选46","候选47: {\"text\": \"é\", \"n\": 47}","候选48","候选49: {\"text\": \"é\", \"n\": 49}","候选50","候选51: {\"text\": \"é\", \"n\": 51}","候选52","候选53: {\"text\": \"é\", \"n\": 53}","候选54","候选55: {\"text\": \"é\", \"n\": 55}","候选56","候选57: {\"text\": \"é\", \"n\": 57}","候选58","候选59: {\"text\": \"é\", \"n\": 59}","候选60","候选61: {\"text\": \"é\", \"n\": 61}","候选62","候选63: {\"text\": \"é\", \"n\": 63}","候选64","候选65: {\"text\": \"é\", \"n\": 65}","候选66","候选67: {\"text\": \"é\", \"n\": 67}","候选68","候选69: {\"text\": \"é\", \"n\": 69}","候选70","候选71: {\"text\": \"é\", \"n\": 71}","候选72","候选73: {\"text\": \"é\", \"n\": 73}","候选74","候选75: {\"text\": \"é\", \"n\": 75}","候选76","候选77: {\"text\": \"é\", \"n\": 77}","候选78","候选79: {\"text\": \"é\", \"n\": 79}","候选80","候选81: {\"text\": \"é\", \"n\": 81}","候选82","候选83: {\"text\": \"é\", \"n\": 83}","候选84","候选85: {\"text\": \"é\", \"n\": 85}","候选86","候选87: {\"text\": \"é\", \"n\": 87}","候选88","候选89: {\"text\": \"é\", \"n\": 89}","候选90","候选91: {\"text\": \"é\", \"n\": 91}","候选92","候选93: {\"text\": \"é\", \"n\": 93}","候选94","候选95: {\"text\": \"é\", \"n\": 95}","候选96","候选97: {\"text\": \"é\", \"n\": 97}","候选98","候选99: {\"text\": \"é\", \"n\": 99}","候选100","候选101: {\"text\": \"é\", \"n\": 101}","候选102","候选103: {\"text\": \"é\", \"n\": 103}","候选104","候选105: {\"text\": \"é\", \"n\": 105}","候选106","候选107: {\"text\": \"é\", \"n\": 107}","候选108","候选109: {\"text\": \"é\", \"n\": 109}","候选110","候选111: {\"text\": \"é\", \"n\": 111}","候选112","候选113: {\"text\": \"é\", \"n\": 113}","候选114","候选115: {\"text\": \"é\", \"n\": 115}","候选116","候选117: {\"text\": \"é\", \"n\": 117}","候选118","候选119: {\"text\": \"é\", \"n\": 119}","候选120","候选121: {\"text\": \"é\", \"n\": 121}","候选122","候选123: {\"text\": \"é\", \"n\": 123}","候选124","候选125: {\"text\": \"é\", \"n\": 125}","候选126","候选127: {\"text\": \"é\", \"n\": 127}","候选128","候选129: {\"text\": \"é\", \"n\": 129}","候选130","候选131: {\"text\": \"é\", \"n\": 131}","候选132","候选133: {\"text\": \"é\", \"n\": 133}","候选134","候选135: {\"text\": \"é\", \"n\": 135}","候选136","候选137: {\"text\": \"é\", \"n\": 137}","候选138","候选139: {\"text\": \"é\", \"n\": 139}","候选140","候选141: {\"text\": \"é\", \"n\": 141}","候选142","候选143: {\"text\": \"é\", \"n\": 143}","候选144","候选145: {\"text\": \"é\", \"n\": 145}","候选146","候选147: {\"text\": \"é\", \"n\": 147}","候选148","候选149: {\"text\": \"é\", \"n\": 149}","候选150","候选151: {\"text\": \"é\", \"n\": 151}","候选152","候选153: {\"text\": \"é\", \"n\": 153}","候选154","候选155: {\"text\": \"é\", \"n\": 155}","候选156","候选157: {\"text\": \"é\", \"n\": 157}","候选158","候选159: {\"text\": \"é\", \"n\": 159}","候选160","候选161: {\"text\": \"é\", \"n\": 161}","候选162","候选163: {\"text\": \"é\", \"n\": 163}","候选164","候选165: {\"text\": \"é\", \"n\": 165}","候选166","候选167: {\"text\": \"é\", \"n\": 167}","候选168","候选169: {\"text\": \"é\", \"n\": 169}","候选170","候选171: {\"text\": \"é\", \"n\": 171}","候选172","候选173: {\"text\": \"é\", \"n\": 173}","候选174","候选175: {\"text\": \"é\", \"n\": 175}","候选176","候选177: {\"text\": \"é\", \"n\": 177}","候选178","候选179: {\"text\": \"é\", \"n\": 179}","候选180","候选181: {\"text\": \"é\", \"n\": 181}","候选182","候选183: {\"text\": \"é\", \"n\": 183}","候选184","候选185: {\"text\": \"é\", \"n\": 185}","候选186","候选187: {\"text\": \"é\", \"n\": 187}","候选188","候选189: {\"text\": \"é\", \"n\": 189}","候选190","候选191: {\"text\": \"é\", \"n\": 191}","候选192","候选193: {\"text\": \"é\", \"n\": 193}","候选194","候选195: {\"text\": \"é\", \"n\": 195}","候选196","候选197: {\"text\": \"é\", \"n\": 197}","候选198","候选199: {\"text\": \"é\", \"n\": 199}","候选200","候选201: {\"text\": \"é\", \"n\": 201}","候选202","候选203: {\"text\": \"é\", \"n\": 203}","候选204","候选205: {\"text\": \"é\", \"n\": 205}","候选206","候选207: {\"text\": \"é\", \"n\": 207}","候选208","候选209: {\"text\": \"é\", \"n\": 209}","候选210","候选211: {\"text\": \"é\", \"n\": 211}","候选212","候选213: {\"text\": \"é\", \"n\": 213}","候选214","候选215: {\"text\": \"é\", \"n\": 215}","候选216","候选217: {\"text\": \"é\", \"n\": 217}","候选218","候选219: {\"text\": \"é\", \"n\": 219}","候选220","候选221: {\"text\": \"é\", \"n\": 221}","候选222","候选223: {\"text\": \"é\", \"n\": 223}","候选224","候选225: {\"text\": \"é\", \"n\": 225}","候选226","候选227: {\"text\": \"é\", \"n\": 227}","候选228","候选229: {\"text\": \"é\", \"n\": 229}","候选230","候选231: {\"text\": \"é\", \"n\": 231}","候选232","候选233: {\"text\": \"é\", \"n\": 233}","候选234","候选235: {\"text\": \"é\", \"n\": 235}","候选236","候选237: {\"text\": \"é\", \"n\": 237}","候选238","候选239: {\"text\": \"é\", \"n\": 239}","候选240","候选241: {\"text\": \"é\", \"n\": 241}","候选242","候选243: {\"text\": \"é\", \"n\": 243}","候选244","候选245: {\"text\": \"é\", \"n\": 245}","候选246","候选247: {\"text\": \"é\", \"n\": 247}","候选248","候选249: {\"text\": \"é\", \"n\": 249}","候选250","候选251: {\"text\": \"é\", \"n\": 251}","候选252","候选253: {\"text\": \"é\", \"n\": 253}","候选254"],"type":"choice","names":["候选0","候选1","候选2","候选3","候选4","候选5","候选6","候选7","候选8","候选9","候选10","候选11","候选12","候选13","候选14","候选15","候选16","候选17","候选18","候选19","候选20","候选21","候选22","候选23","候选24","候选25","候选26","候选27","候选28","候选29","候选30","候选31","候选32","候选33","候选34","候选35","候选36","候选37","候选38","候选39","候选40","候选41","候选42","候选43","候选44","候选45","候选46","候选47","候选48","候选49","候选50","候选51","候选52","候选53","候选54","候选55","候选56","候选57","候选58","候选59","候选60","候选61","候选62","候选63","候选64","候选65","候选66","候选67","候选68","候选69","候选70","候选71","候选72","候选73","候选74","候选75","候选76","候选77","候选78","候选79","候选80","候选81","候选82","候选83","候选84","候选85","候选86","候选87","候选88","候选89","候选90","候选91","候选92","候选93","候选94","候选95","候选96","候选97","候选98","候选99","候选100","候选101","候选102","候选103","候选104","候选105","候选106","候选107","候选108","候选109","候选110","候选111","候选112","候选113","候选114","候选115","候选116","候选117","候选118","候选119","候选120","候选121","候选122","候选123","候选124","候选125","候选126","候选127","候选128","候选129","候选130","候选131","候选132","候选133","候选134","候选135","候选136","候选137","候选138","候选139","候选140","候选141","候选142","候选143","候选144","候选145","候选146","候选147","候选148","候选149","候选150","候选151","候选152","候选153","候选154","候选155","候选156","候选157","候选158","候选159","候选160","候选161","候选162","候选163","候选164","候选165","候选166","候选167","候选168","候选169","候选170","候选171","候选172","候选173","候选174","候选175","候选176","候选177","候选178","候选179","候选180","候选181","候选182","候选183","候选184","候选185","候选186","候选187","候选188","候选189","候选190","候选191","候选192","候选193","候选194","候选195","候选196","候选197","候选198","候选199","候选200","候选201","候选202","候选203","候选204","候选205","候选206","候选207","候选208","候选209","候选210","候选211","候选212","候选213","候选214","候选215","候选216","候选217","候选218","候选219","候选220","候选221","候选222","候选223","候选224","候选225","候选226","候选227","候选228","候选229","候选230","候选231","候选232","候选233","候选234","候选235","候选236","候选237","候选238","候选239","候选240","候选241","候选242","候选243","候选244","候选245","候选246","候选247","候选248","候选249","候选250","候选251","候选252","候选253","候选254"],"legend":null,"isolated":true}},"index":[["hidden-id","list",0,1]],"rows":[{"question":"Pick 中文","options":["候选0","候选1: {\"text\": \"é\", \"n\": 1}","候选2","候选3: {\"text\": \"é\", \"n\": 3}","候选4","候选5: {\"text\": \"é\", \"n\": 5}","候选6","候选7: {\"text\": \"é\", \"n\": 7}","候选8","候选9: {\"text\": \"é\", \"n\": 9}","候选10","候选11: {\"text\": \"é\", \"n\": 11}","候选12","候选13: {\"text\": \"é\", \"n\": 13}","候选14","候选15: {\"text\": \"é\", \"n\": 15}","候选16","候选17: {\"text\": \"é\", \"n\": 17}","候选18","候选19: {\"text\": \"é\", \"n\": 19}","候选20","候选21: {\"text\": \"é\", \"n\": 21}","候选22","候选23: {\"text\": \"é\", \"n\": 23}","候选24","候选25: {\"text\": \"é\", \"n\": 25}","候选26","候选27: {\"text\": \"é\", \"n\": 27}","候选28","候选29: {\"text\": \"é\", \"n\": 29}","候选30","候选31: {\"text\": \"é\", \"n\": 31}","候选32","候选33: {\"text\": \"é\", \"n\": 33}","候选34","候选35: {\"text\": \"é\", \"n\": 35}","候选36","候选37: {\"text\": \"é\", \"n\": 37}","候选38","候选39: {\"text\": \"é\", \"n\": 39}","候选40","候选41: {\"text\": \"é\", \"n\": 41}","候选42","候选43: {\"text\": \"é\", \"n\": 43}","候选44","候选45: {\"text\": \"é\", \"n\": 45}","候选46","候选47: {\"text\": \"é\", \"n\": 47}","候选48","候选49: {\"text\": \"é\", \"n\": 49}","候选50","候选51: {\"text\": \"é\", \"n\": 51}","候选52","候选53: {\"text\": \"é\", \"n\": 53}","候选54","候选55: {\"text\": \"é\", \"n\": 55}","候选56","候选57: {\"text\": \"é\", \"n\": 57}","候选58","候选59: {\"text\": \"é\", \"n\": 59}","候选60","候选61: {\"text\": \"é\", \"n\": 61}","候选62","候选63: {\"text\": \"é\", \"n\": 63}","候选64","候选65: {\"text\": \"é\", \"n\": 65}","候选66","候选67: {\"text\": \"é\", \"n\": 67}","候选68","候选69: {\"text\": \"é\", \"n\": 69}","候选70","候选71: {\"text\": \"é\", \"n\": 71}","候选72","候选73: {\"text\": \"é\", \"n\": 73}","候选74","候选75: {\"text\": \"é\", \"n\": 75}","候选76","候选77: {\"text\": \"é\", \"n\": 77}","候选78","候选79: {\"text\": \"é\", \"n\": 79}","候选80","候选81: {\"text\": \"é\", \"n\": 81}","候选82","候选83: {\"text\": \"é\", \"n\": 83}","候选84","候选85: {\"text\": \"é\", \"n\": 85}","候选86","候选87: {\"text\": \"é\", \"n\": 87}","候选88","候选89: {\"text\": \"é\", \"n\": 89}","候选90","候选91: {\"text\": \"é\", \"n\": 91}","候选92","候选93: {\"text\": \"é\", \"n\": 93}","候选94","候选95: {\"text\": \"é\", \"n\": 95}","候选96","候选97: {\"text\": \"é\", \"n\": 97}","候选98","候选99: {\"text\": \"é\", \"n\": 99}","候选100","候选101: {\"text\": \"é\", \"n\": 101}","候选102","候选103: {\"text\": \"é\", \"n\": 103}","候选104","候选105: {\"text\": \"é\", \"n\": 105}","候选106","候选107: {\"text\": \"é\", \"n\": 107}","候选108","候选109: {\"text\": \"é\", \"n\": 109}","候选110","候选111: {\"text\": \"é\", \"n\": 111}","候选112","候选113: {\"text\": \"é\", \"n\": 113}","候选114","候选115: {\"text\": \"é\", \"n\": 115}","候选116","候选117: {\"text\": \"é\", \"n\": 117}","候选118","候选119: {\"text\": \"é\", \"n\": 119}","候选120","候选121: {\"text\": \"é\", \"n\": 121}","候选122","候选123: {\"text\": \"é\", \"n\": 123}","候选124","候选125: {\"text\": \"é\", \"n\": 125}","候选126","候选127: {\"text\": \"é\", \"n\": 127}","候选128","候选129: {\"text\": \"é\", \"n\": 129}","候选130","候选131: {\"text\": \"é\", \"n\": 131}","候选132","候选133: {\"text\": \"é\", \"n\": 133}","候选134","候选135: {\"text\": \"é\", \"n\": 135}","候选136","候选137: {\"text\": \"é\", \"n\": 137}","候选138","候选139: {\"text\": \"é\", \"n\": 139}","候选140","候选141: {\"text\": \"é\", \"n\": 141}","候选142","候选143: {\"text\": \"é\", \"n\": 143}","候选144","候选145: {\"text\": \"é\", \"n\": 145}","候选146","候选147: {\"text\": \"é\", \"n\": 147}","候选148","候选149: {\"text\": \"é\", \"n\": 149}","候选150","候选151: {\"text\": \"é\", \"n\": 151}","候选152","候选153: {\"text\": \"é\", \"n\": 153}","候选154","候选155: {\"text\": \"é\", \"n\": 155}","候选156","候选157: {\"text\": \"é\", \"n\": 157}","候选158","候选159: {\"text\": \"é\", \"n\": 159}","候选160","候选161: {\"text\": \"é\", \"n\": 161}","候选162","候选163: {\"text\": \"é\", \"n\": 163}","候选164","候选165: {\"text\": \"é\", \"n\": 165}","候选166","候选167: {\"text\": \"é\", \"n\": 167}","候选168","候选169: {\"text\": \"é\", \"n\": 169}","候选170","候选171: {\"text\": \"é\", \"n\": 171}","候选172","候选173: {\"text\": \"é\", \"n\": 173}","候选174","候选175: {\"text\": \"é\", \"n\": 175}","候选176","候选177: {\"text\": \"é\", \"n\": 177}","候选178","候选179: {\"text\": \"é\", \"n\": 179}","候选180","候选181: {\"text\": \"é\", \"n\": 181}","候选182","候选183: {\"text\": \"é\", \"n\": 183}","候选184","候选185: {\"text\": \"é\", \"n\": 185}","候选186","候选187: {\"text\": \"é\", \"n\": 187}","候选188","候选189: {\"text\": \"é\", \"n\": 189}","候选190","候选191: {\"text\": \"é\", \"n\": 191}","候选192","候选193: {\"text\": \"é\", \"n\": 193}","候选194","候选195: {\"text\": \"é\", \"n\": 195}","候选196","候选197: {\"text\": \"é\", \"n\": 197}","候选198","候选199: {\"text\": \"é\", \"n\": 199}","候选200","候选201: {\"text\": \"é\", \"n\": 201}","候选202","候选203: {\"text\": \"é\", \"n\": 203}","候选204","候选205: {\"text\": \"é\", \"n\": 205}","候选206","候选207: {\"text\": \"é\", \"n\": 207}","候选208","候选209: {\"text\": \"é\", \"n\": 209}","候选210","候选211: {\"text\": \"é\", \"n\": 211}","候选212","候选213: {\"text\": \"é\", \"n\": 213}","候选214","候选215: {\"text\": \"é\", \"n\": 215}","候选216","候选217: {\"text\": \"é\", \"n\": 217}","候选218","候选219: {\"text\": \"é\", \"n\": 219}","候选220","候选221: {\"text\": \"é\", \"n\": 221}","候选222","候选223: {\"text\": \"é\", \"n\": 223}","候选224","候选225: {\"text\": \"é\", \"n\": 225}","候选226","候选227: {\"text\": \"é\", \"n\": 227}","候选228","候选229: {\"text\": \"é\", \"n\": 229}","候选230","候选231: {\"text\": \"é\", \"n\": 231}","候选232","候选233: {\"text\": \"é\", \"n\": 233}","候选234","候选235: {\"text\": \"é\", \"n\": 235}","候选236","候选237: {\"text\": \"é\", \"n\": 237}","候选238","候选239: {\"text\": \"é\", \"n\": 239}","候选240","候选241: {\"text\": \"é\", \"n\": 241}","候选242","候选243: {\"text\": \"é\", \"n\": 243}","候选244","候选245: {\"text\": \"é\", \"n\": 245}","候选246","候选247: {\"text\": \"é\", \"n\": 247}","候选248","候选249: {\"text\": \"é\", \"n\": 249}","候选250","候选251: {\"text\": \"é\", \"n\": 251}","候选252","候选253: {\"text\": \"é\", \"n\": 253}","候选254"]}],"items":[{"ids":[1905,25,198,113130,220,248045,198,50203,271,14162,25,19123,220,99986,198,3670,25,198,7,32,8,220,102486,15,198,7,33,8,220,102486,16,25,5046,1272,763,328,933,487,328,77,763,220,16,92,198,7,34,8,220,102486,17,198,7,35,8,220,102486,18,25,5046,1272,763,328,933,487,328,77,763,220,18,92,198,7,36,8,220,102486,19,198,7,37,8,220,102486,20,25,5046,1272,763,328,933,487,328,77,763,220,20,92,198,7,38,8,220,102486,21,198,7,39,8,220,102486,22,25,5046,1272,763,328,933,487,328,77,763,220,22,92,198,7,40,8,220,102486,23,198,7,41,8,220,102486,24,25,5046,1272,763,328,933,487,328,77,763,220,24,92,198,7,42,8,220,102486,16,15,198,7,43,8,220,102486,16,16,25,5046,1272,763,328,933,487,328,77,763,220,16,16,92,198,7,44,8,220,102486,16,17,198,7,45,8,220,102486,16,18,25,5046,1272,763,328,933,487,328,77,763,220,16,18,92,198,7,46,8,220,102486,16,19,198,7,47,8,220,102486,16,20,25,5046,1272,763,328,933,487,328,77,763,220,16,20,92,198,7,48,8,220,102486,16,21,198,7,49,8,220,102486,16,22,25,5046,1272,763,328,933,487,328,77,763,220,16,22,92,198,7,50,8,220,102486,16,23,198,7,51,8,220,102486,16,24,25,5046,1272,763,328,933,487,328,77,763,220,16,24,92,198,7,52,8,220,102486,17,15,198,7,53,8,220,102486,17,16,25,5046,1272,763,328,933,487,328,77,763,220,17,16,92,198,7,54,8,220,102486,17,17,198,7,55,8,220,102486,17,18,25,5046,1272,763,328,933,487,328,77,763,220,17,18,92,198,7,56,8,220,102486,17,19,198,7,57,8,220,102486,17,20,25,5046,1272,763,328,933,487,328,77,763,220,17,20,92,198,7,5840,8,220,102486,17,21,198,7,1803,8,220,102486,17,22,25,5046,1272,763,328,933,487,328,77,763,220,17,22,92,198,7,1646,8,220,102486,17,23,198,7,1745,8,220,102486,17,24,25,5046,1272,763,328,933,487,328,77,763,220,17,24,92,198,7,13276,8,220,102486,18,15,198,7,8018,8,220,102486,18,16,25,5046,1272,763,328,933,487,328,77,763,220,18,16,92,198,7,1825,8,220,102486,18,17,198,7,28946,8,220,102486,18,18,25,5046,1272,763,328,933,487,328,77,763,220,18,18,92,198,7,15015,8,220,102486,18,19,198,7,29595,8,220,102486,18,20,25,5046,1272,763,328,933,487,328,77,763,220,18,20,92,198,7,11568,8,220,102486,18,21,198,7,939,8,220,102486,18,22,25,5046,1272,763,328,933,487,328,77,763,220,18,22,92,198,7,1354,8,220,102486,18,23,198,7,1058,8,220,102486,18,24,25,5046,1272,763,328,933,487,328,77,763,220,18,24,92,198,7,18183,8,220,102486,19,15,198,7,2456,8,220,102486,19,16,25,5046,1272,763,328,933,487,328,77,763,220,19,16,92,198,7,88898,8,220,102486,19,17,198,7,905,8,220,102486,19,18,25,5046,1272,763,328,933,487,328,77,763,220,19,18,92,198,7,1846,8,220,102486,19,19,198,7,802,8,220,102486,19,20,25,5046,1272,763,328,933,487,328,77,763,220,19,20,92,198,7,33869,8,220,102486,19,21,198,7,7839,8,220,102486,19,22,25,5046,1272,763,328,933,487,328,77,763,220,19,22,92,198,7,14006,8,220,102486,19,23,198,7,2860,8,220,102486,19,24,25,5046,1272,763,328,933,487,328,77,763,220,19,24,92,198,7,2926,8,220,102486,20,15,198,7,22828,8,220,102486,20,16,25,5046,1272,763,328,933,487,328,77,763,220,20,16,92,198,7,6844,8,220,102486,20,17,198,7,9798,8,220,102486,20,18,25,5046,1272,763,328,933,487,328,77,763,220,20,18,92,198,7,4738,8,220,102486,20,19,198,7,9265,8,220,102486,20,20,25,5046,1272,763,328,933,487,328,77,763,220,20,20,92,198,7,11261,8,220,102486,20,21,198,7,19278,8,220,102486,20,22,25,5046,1272,763,328,933,487,328,77,763,220,20,22,92,198,7,36513,8,220,102486,20,23,198,7,93801,8,220,102486,20,24,25,5046,1272,763,328,933,487,328,77,763,220,20,24,92,198,7,8335,8,220,102486,21,15,198,7,14544,8,220,102486,21,16,25,5046,1272,763,328,933,487,328,77,763,220,21,16,92,198,7,85266,8,220,102486,21,17,198,7,9110,8,220,102486,21,18,25,5046,1272,763,328,933,487,328,77,763,220,21,18,92,198,7,28000,8,220,102486,21,19,198,7,15137,8,220,102486,21,20,25,5046,1272,763,328,933,487,328,77,763,220,21,20,92,198,7,4525,8,220,102486,21,21,198,7,25261,8,220,102486,21,22,25,5046,1272,763,328,933,487,328,77,763,220,21,22,92,198,7,12717,8,220,102486,21,23,198,7,7116,8,220,102486,21,24,25,5046,1272,763,328,933,487,328,77,763,220,21,24,92,198,7,17078,8,220,102486,22,15,198,7,14497,8,220,102486,22,16,25,5046,1272,763,328,933,487,328,77,763,220,22,16,92,198,7,57339,8,220,102486,22,17,198,7,74909,8,220,102486,22,18,25,5046,1272,763,328,933,487,328,77,763,220,22,18,92,198,7,52072,8,220,102486,22,19,198,7,19305,8,220,102486,22,20,25,5046,1272,763,328,933,487,328,77,763,220,22,20,92,198,7,4887,8,220,102486,22,21,198,7,12607,8,220,102486,22,22,25,5046,1272,763,328,933,487,328,77,763,220,22,22,92,198,7,3580,8,220,102486,22,23,198,7,6281,8,220,102486,22,24,25,5046,1272,763,328,933,487,328,77,763,220,22,24,92,198,7,2036,8,220,102486,23,15,198,7,9362,8,220,102486,23,16,25,5046,1272,763,328,933,487,328,77,763,220,23,16,92,198,7,8533,8,220,102486,23,17,198,7,2080,8,220,102486,23,18,25,5046,1272,763,328,933,487,328,77,763,220,23,18,92,198,7,10911,8,220,102486,23,19,198,7,2925,8,220,102486,23,20,25,5046,1272,763,328,933,487,328,77,763,220,23,20,92,198,7,3040,8,220,102486,23,21,198,7,9690,8,220,102486,23,22,25,5046,1272,763,328,933,487,328,77,763,220,23,22,92,198,7,27731,8,220,102486,23,23,198,7,8023,8,220,102486,23,24,25,5046,1272,763,328,933,487,328,77,763,220,23,24,92,198,7,6901,8,220,102486,24,15,198,7,8702,8,220,102486,24,16,25,5046,1272,763,328,933,487,328,77,763,220,24,16,92,198,7,6211,8,220,102486,24,17,198,7,1123,8,220,102486,24,18,25,5046,1272,763,328,933,487,328,77,763,220,24,18,92,198,7,16307,8,220,102486,24,19,198,7,18990,8,220,102486,24,20,25,5046,1272,763,328,933,487,328,77,763,220,24,20,92,198,7,64045,8,220,102486,24,21,198,7,63037,8,220,102486,24,22,25,5046,1272,763,328,933,487,328,77,763,220,24,22,92,198,7,33380,8,220,102486,24,23,198,7,6151,8,220,102486,24,24,25,5046,1272,763,328,933,487,328,77,763,220,24,24,92,198,7,3392,8,220,102486,16,15,15,198,7,5449,8,220,102486,16,15,16,25,5046,1272,763,328,933,487,328,77,763,220,16,15,16,92,198,7,3967,8,220,102486,16,15,17,198,7,1113,8,220,102486,16,15,18,25,5046,1272,763,328,933,487,328,77,763,220,16,15,18,92,198,7,5095,8,220,102486,16,15,19,198,7,51923,8,220,102486,16,15,20,25,5046,1272,763,328,933,487,328,77,763,220,16,15,20,92,198,7,49600,8,220,102486,16,15,21,198,7,17099,8,220,102486,16,15,22,25,5046,1272,763,328,933,487,328,77,763,220,16,15,22,92,198,7,51483,8,220,102486,16,15,23,198,7,17756,8,220,102486,16,15,24,25,5046,1272,763,328,933,487,328,77,763,220,16,15,24,92,198,7,16037,8,220,102486,16,16,15,198,7,8135,8,220,102486,16,16,16,25,5046,1272,763,328,933,487,328,77,763,220,16,16,16,92,198,7,30237,8,220,102486,16,16,17,198,7,5683,8,220,102486,16,16,18,25,5046,1272,763,328,933,487,328,77,763,220,16,16,18,92,198,7,9992,8,220,102486,16,16,19,198,7,7444,8,220,102486,16,16,20,25,5046,1272,763,328,933,487,328,77,763,220,16,16,20,92,198,7,5751,8,220,102486,16,16,21,198,7,10284,8,220,102486,16,16,22,25,5046,1272,763,328,933,487,328,77,763,220,16,16,22,92,198,7,20887,8,220,102486,16,16,23,198,7,59884,8,220,102486,16,16,24,25,5046,1272,763,328,933,487,328,77,763,220,16,16,24,92,198,7,52396,8,220,102486,16,17,15,198,7,16103,8,220,102486,16,17,16,25,5046,1272,763,328,933,487,328,77,763,220,16,17,16,92,198,7,67547,8,220,102486,16,17,17,198,7,18535,8,220,102486,16,17,18,25,5046,1272,763,328,933,487,328,77,763,220,16,17,18,92,198,7,8006,8,220,102486,16,17,19,198,7,7263,8,220,102486,16,17,20,25,5046,1272,763,328,933,487,328,77,763,220,16,17,20,92,198,7,1425,8,220,102486,16,17,21,198,7,6878,8,220,102486,16,17,22,25,5046,1272,763,328,933,487,328,77,763,220,16,17,22,92,198,7,14453,8,220,102486,16,17,23,198,7,9097,8,220,102486,16,17,24,25,5046,1272,763,328,933,487,328,77,763,220,16,17,24,92,198,7,44072,8,220,102486,16,18,15,198,7,76089,8,220,102486,16,18,16,25,5046,1272,763,328,933,487,328,77,763,220,16,18,16,92,198,7,68720,8,220,102486,16,18,17,198,7,2662,8,220,102486,16,18,18,25,5046,1272,763,328,933,487,328,77,763,220,16,18,18,92,198,7,2629,8,220,102486,16,18,19,198,7,923,8,220,102486,16,18,20,25,5046,1272,763,328,933,487,328,77,763,220,16,18,20,92,198,7,6548,8,220,102486,16,18,21,198,7,8924,8,220,102486,16,18,22,25,5046,1272,763,328,933,487,328,77,763,220,16,18,22,92,198,7,52194,8,220,102486,16,18,23,198,7,622,8,220,102486,16,18,24,25,5046,1272,763,328,933,487,328,77,763,220,16,18,24,92,198,7,1515,8,220,102486,16,19,15,198,7,1300,8,220,102486,16,19,16,25,5046,1272,763,328,933,487,328,77,763,220,16,19,16,92,198,7,37523,8,220,102486,16,19,17,198,7,44473,8,220,102486,16,19,18,25,5046,1272,763,328,933,487,328,77,763,220,16,19,18,92,198,7,36530,8,220,102486,16,19,19,198,7,3152,8,220,102486,16,19,20,25,5046,1272,763,328,933,487,328,77,763,220,16,19,20,92,198,7,93924,8,220,102486,16,19,21,198,7,3505,8,220,102486,16,19,22,25,5046,1272,763,328,933,487,328,77,763,220,16,19,22,92,198,7,15731,8,220,102486,16,19,23,198,7,6542,8,220,102486,16,19,24,25,5046,1272,763,328,933,487,328,77,763,220,16,19,24,92,198,7,14176,8,220,102486,16,20,15,198,7,11091,8,220,102486,16,20,16,25,5046,1272,763,328,933,487,328,77,763,220,16,20,16,92,198,7,1686,8,220,102486,16,20,17,198,7,11660,8,220,102486,16,20,18,25,5046,1272,763,328,933,487,328,77,763,220,16,20,18,92,198,7,80440,8,220,102486,16,20,19,198,7,18836,8,220,102486,16,20,20,25,5046,1272,763,328,933,487,328,77,763,220,16,20,20,92,198,7,26132,8,220,102486,16,20,21,198,7,5934,8,220,102486,16,20,22,25,5046,1272,763,328,933,487,328,77,763,220,16,20,22,92,198,7,24794,8,220,102486,16,20,23,198,7,40229,8,220,102486,16,20,24,25,5046,1272,763,328,933,487,328,77,763,220,16,20,24,92,198,7,3660,8,220,102486,16,21,15,198,7,11361,8,220,102486,16,21,16,25,5046,1272,763,328,933,487,328,77,763,220,16,21,16,92,198,7,10191,8,220,102486,16,21,17,198,7,8225,8,220,102486,16,21,18,25,5046,1272,763,328,933,487,328,77,763,220,16,21,18,92,198,7,3860,8,220,102486,16,21,19,198,7,78413,8,220,102486,16,21,20,25,5046,1272,763,328,933,487,328,77,763,220,16,21,20,92,198,7,17680,8,220,102486,16,21,21,198,7,15900,8,220,102486,16,21,22,25,5046,1272,763,328,933,487,328,77,763,220,16,21,22,92,198,7,78138,8,220,102486,16,21,23,198,7,15653,8,220,102486,16,21,24,25,5046,1272,763,328,933,487,328,77,763,220,16,21,24,92,198,7,5213,8,220,102486,16,22,15,198,7,22150,8,220,102486,16,22,16,25,5046,1272,763,328,933,487,328,77,763,220,16,22,16,92,198,7,39500,8,220,102486,16,22,17,198,7,10460,8,220,102486,16,22,18,25,5046,1272,763,328,933,487,328,77,763,220,16,22,18,92,198,7,35131,8,220,102486,16,22,19,198,7,21563,8,220,102486,16,22,20,25,5046,1272,763,328,933,487,328,77,763,220,16,22,20,92,198,7,43194,8,220,102486,16,22,21,198,7,27127,8,220,102486,16,22,22,25,5046,1272,763,328,933,487,328,77,763,220,16,22,22,92,198,7,3697,8,220,102486,16,22,23,198,7,20011,8,220,102486,16,22,24,25,5046,1272,763,328,933,487,328,77,763,220,16,22,24,92,198,7,24368,8,220,102486,16,23,15,198,7,15058,8,220,102486,16,23,16,25,5046,1272,763,328,933,487,328,77,763,220,16,23,16,92,198,7,23658,8,220,102486,16,23,17,198,7,8362,8,220,102486,16,23,18,25,5046,1272,763,328,933,487,328,77,763,220,16,23,18,92,198,7,16035,8,220,102486,16,23,19,198,7,24583,8,220,102486,16,23,20,25,5046,1272,763,328,933,487,328,77,763,220,16,23,20,92,198,7,52857,8,220,102486,16,23,21,198,7,38403,8,220,102486,16,23,22,25,5046,1272,763,328,933,487,328,77,763,220,16,23,22,92,198,7,60456,8,220,102486,16,23,23,198,7,81519,8,220,102486,16,23,24,25,5046,1272,763,328,933,487,328,77,763,220,16,23,24,92,198,7,40097,8,220,102486,16,24,15,198,7,16522,8,220,102486,16,24,16,25,5046,1272,763,328,933,487,328,77,763,220,16,24,16,92,198,7,29722,8,220,102486,16,24,17,198,7,21756,8,220,102486,16,24,18,25,5046,1272,763,328,933,487,328,77,763,220,16,24,18,92,198,7,18567,8,220,102486,16,24,19,198,7,1736,8,220,102486,16,24,20,25,5046,1272,763,328,933,487,328,77,763,220,16,24,20,92,198,7,48043,8,220,102486,16,24,21,198,7,87013,8,220,102486,16,24,22,25,5046,1272,763,328,933,487,328,77,763,220,16,24,22,92,198,7,22456,8,220,102486,16,24,23,198,7,23165,8,220,102486,16,24,24,25,5046,1272,763,328,933,487,328,77,763,220,16,24,24,92,198,7,55523,8,220,102486,17,15,15,198,7,13097,8,220,102486,17,15,16,25,5046,1272,763,328,933,487,328,77,763,220,17,15,16,92,198,7,50397,8,220,102486,17,15,17,198,7,41741,8,220,102486,17,15,18,25,5046,1272,763,328,933,487,328,77,763,220,17,15,18,92,198,7,23073,8,220,102486,17,15,19,198,7,6401,8,220,102486,17,15,20,25,5046,1272,763,328,933,487,328,77,763,220,17,15,20,92,198,7,86538,8,220,102486,17,15,21,198,7,16585,8,220,102486,17,15,22,25,5046,1272,763,328,933,487,328,77,763,220,17,15,22,92,198,7,11622,8,220,102486,17,15,23,198,7,2464,8,220,102486,17,15,24,25,5046,1272,763,328,933,487,328,77,763,220,17,15,24,92,198,7,84982,8,220,102486,17,16,15,198,7,75516,8,220,102486,17,16,16,25,5046,1272,763,328,933,487,328,77,763,220,17,16,16,92,198,7,36984,8,220,102486,17,16,17,198,7,58795,8,220,102486,17,16,18,25,5046,1272,763,328,933,487,328,77,763,220,17,16,18,92,198,7,47217,8,220,102486,17,16,19,198,7,59675,8,220,102486,17,16,20,25,5046,1272,763,328,933,487,328,77,763,220,17,16,20,92,198,7,5681,8,220,102486,17,16,21,198,7,3151,8,220,102486,17,16,22,25,5046,1272,763,328,933,487,328,77,763,220,17,16,22,92,198,7,1271,8,220,102486,17,16,23,198,7,887,8,220,102486,17,16,24,25,5046,1272,763,328,933,487,328,77,763,220,17,16,24,92,198,7,5203,8,220,102486,17,17,15,198,7,2685,8,220,102486,17,17,16,25,5046,1272,763,328,933,487,328,77,763,220,17,17,16,92,198,7,1849,8,220,102486,17,17,17,198,7,72470,8,220,102486,17,17,18,25,5046,1272,763,328,933,487,328,77,763,220,17,17,18,92,198,7,5370,8,220,102486,17,17,19,198,7,74063,8,220,102486,17,17,20,25,5046,1272,763,328,933,487,328,77,763,220,17,17,20,92,198,7,27629,8,220,102486,17,17,21,198,7,1655,8,220,102486,17,17,22,25,5046,1272,763,328,933,487,328,77,763,220,17,17,22,92,198,7,1728,8,220,102486,17,17,23,198,7,669,8,220,102486,17,17,24,25,5046,1272,763,328,933,487,328,77,763,220,17,17,24,92,198,7,3682,8,220,102486,17,18,15,198,7,3191,8,220,102486,17,18,16,25,5046,1272,763,328,933,487,328,77,763,220,17,18,16,92,198,7,59865,8,220,102486,17,18,17,198,7,2712,8,220,102486,17,18,18,25,5046,1272,763,328,933,487,328,77,763,220,17,18,18,92,198,7,1580,8,220,102486,17,18,19,198,7,922,8,220,102486,17,18,20,25,5046,1272,763,328,933,487,328,77,763,220,17,18,20,92,198,7,77243,8,220,102486,17,18,21,198,7,2990,8,220,102486,17,18,22,25,5046,1272,763,328,933,487,328,77,763,220,17,18,22,92,198,7,78493,8,220,102486,17,18,23,198,7,5228,8,220,102486,17,18,24,25,5046,1272,763,328,933,487,328,77,763,220,17,18,24,92,198,7,2750,8,220,102486,17,19,15,198,7,42711,8,220,102486,17,19,16,25,5046,1272,763,328,933,487,328,77,763,220,17,19,16,92,198,7,44568,8,220,102486,17,19,17,198,7,56402,8,220,102486,17,19,18,25,5046,1272,763,328,933,487,328,77,763,220,17,19,18,92,198,7,48236,8,220,102486,17,19,19,198,7,38993,8,220,102486,17,19,20,25,5046,1272,763,328,933,487,328,77,763,220,17,19,20,92,198,7,43559,8,220,102486,17,19,21,198,7,61250,8,220,102486,17,19,22,25,5046,1272,763,328,933,487,328,77,763,220,17,19,22,92,198,7,32942,8,220,102486,17,19,23,198,7,86302,8,220,102486,17,19,24,25,5046,1272,763,328,933,487,328,77,763,220,17,19,24,92,198,7,25310,8,220,102486,17,20,15,198,7,26313,8,220,102486,17,20,16,25,5046,1272,763,328,933,487,328,77,763,220,17,20,16,92,198,7,81770,8,220,102486,17,20,17,198,7,12185,8,220,102486,17,20,18,25,5046,1272,763,328,933,487,328,77,763,220,17,20,18,92,198,7,78382,8,220,102486,17,20,19,198,15666,25,318],"slots":[4055],"golds":[0],"nopts":[255],"perms":[[0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,77,78,79,80,81,82,83,84,85,86,87,88,89,90,91,92,93,94,95,96,97,98,99,100,101,102,103,104,105,106,107,108,109,110,111,112,113,114,115,116,117,118,119,120,121,122,123,124,125,126,127,128,129,130,131,132,133,134,135,136,137,138,139,140,141,142,143,144,145,146,147,148,149,150,151,152,153,154,155,156,157,158,159,160,161,162,163,164,165,166,167,168,169,170,171,172,173,174,175,176,177,178,179,180,181,182,183,184,185,186,187,188,189,190,191,192,193,194,195,196,197,198,199,200,201,202,203,204,205,206,207,208,209,210,211,212,213,214,215,216,217,218,219,220,221,222,223,224,225,226,227,228,229,230,231,232,233,234,235,236,237,238,239,240,241,242,243,244,245,246,247,248,249,250,251,252,253,254]],"types":["choice"]}],"context_tokens":8,"unique_tokens":4056,"processed_tokens":4056,"temperatures":[[1.164]]},{"name":"score_2","state":{"items":[1,{"_index":77,"text":"保留"},1,{"_index":77,"text":"保留"},1,{"_index":77,"text":"保留"},1,{"_index":77,"text":"保留"}]},"questions":{"score-id":{"type":"score","instructions":{"ask":"How severe?"},"criteria":[" 0: level 0"," 1: level 1"]}},"rendered_state":"{\"items\": [{\"_index\": 0, \"value\": 1}, {\"_index\": 77, \"text\": \"保留\"}, {\"_index\": 2, \"value\": 1}, {\"_index\": 77, \"text\": \"保留\"}, {\"_index\": 4, \"value\": 1}, {\"_index\": 77, \"text\": \"保留\"}, {\"_index\": 6, \"value\": 1}, {\"_index\": 77, \"text\": \"保留\"}]}","rendered_questions":{"score-id":{"question":"{\"ask\": \"How severe?\"}","options":["0: 0: level 0","1: 1: level 1"],"type":"score","names":[0,1],"legend":[" 0: level 0"," 1: level 1"],"isolated":true}},"index":[["score-id","iso",0,2]],"rows":[{"question":"{\"ask\": \"How severe?\"}\nProposed answer: level 0\nDoes the proposed answer fit?","options":["no","yes"]},{"question":"{\"ask\": \"How severe?\"}\nProposed answer: level 1\nDoes the proposed answer fit?","options":["no","yes"]}],"items":[{"ids":[1905,25,198,4754,3494,763,59670,62,1209,763,220,15,11,328,927,763,220,16,2069,5046,62,1209,763,220,22,22,11,328,1272,763,328,101780,13933,5046,62,1209,763,220,17,11,328,927,763,220,16,2069,5046,62,1209,763,220,22,22,11,328,1272,763,328,101780,13933,5046,62,1209,763,220,19,11,328,927,763,220,16,2069,5046,62,1209,763,220,22,22,11,328,1272,763,328,101780,13933,5046,62,1209,763,220,21,11,328,927,763,220,16,2069,5046,62,1209,763,220,22,22,11,328,1272,763,328,101780,8934,13587,271,14162,25,5046,1038,763,328,4199,14938,7285,92,198,1941,18988,4087,25,2119,220,15,198,20810,279,10660,4087,4786,30,198,3670,25,198,4205,8,874,198,5181,8,9542,198,15666,25,318],"slots":[155],"golds":[0],"nopts":[2],"perms":[[0,1]],"types":["score"]},{"ids":[1905,25,198,4754,3494,763,59670,62,1209,763,220,15,11,328,927,763,220,16,2069,5046,62,1209,763,220,22,22,11,328,1272,763,328,101780,13933,5046,62,1209,763,220,17,11,328,927,763,220,16,2069,5046,62,1209,763,220,22,22,11,328,1272,763,328,101780,13933,5046,62,1209,763,220,19,11,328,927,763,220,16,2069,5046,62,1209,763,220,22,22,11,328,1272,763,328,101780,13933,5046,62,1209,763,220,21,11,328,927,763,220,16,2069,5046,62,1209,763,220,22,22,11,328,1272,763,328,101780,8934,13587,271,14162,25,5046,1038,763,328,4199,14938,7285,92,198,1941,18988,4087,25,2119,220,16,198,20810,279,10660,4087,4786,30,198,3670,25,198,4205,8,874,198,5181,8,9542,198,15666,25,318],"slots":[155],"golds":[0],"nopts":[2],"perms":[[0,1]],"types":["score"]}],"context_tokens":115,"unique_tokens":179,"processed_tokens":312,"temperatures":[[1.124],[1.124]]},{"name":"score_3","state":{"items":[1,{"_index":77,"text":"保留"},1,{"_index":77,"text":"保留"},1,{"_index":77,"text":"保留"},1,{"_index":77,"text":"保留"}]},"questions":{"score-id":{"type":"score","instructions":{"ask":"How severe?"},"criteria":[" 0: level 0"," 1: level 1"," 2: level 2"]}},"rendered_state":"{\"items\": [{\"_index\": 0, \"value\": 1}, {\"_index\": 77, \"text\": \"保留\"}, {\"_index\": 2, \"value\": 1}, {\"_index\": 77, \"text\": \"保留\"}, {\"_index\": 4, \"value\": 1}, {\"_index\": 77, \"text\": \"保留\"}, {\"_index\": 6, \"value\": 1}, {\"_index\": 77, \"text\": \"保留\"}]}","rendered_questions":{"score-id":{"question":"{\"ask\": \"How severe?\"}","options":["0: 0: level 0","1: 1: level 1","2: 2: level 2"],"type":"score","names":[0,1,2],"legend":[" 0: level 0"," 1: level 1"," 2: level 2"],"isolated":true}},"index":[["score-id","iso",0,3]],"rows":[{"question":"{\"ask\": \"How severe?\"}\nProposed answer: level 0\nDoes the proposed answer fit?","options":["no","yes"]},{"question":"{\"ask\": \"How severe?\"}\nProposed answer: level 1\nDoes the proposed answer fit?","options":["no","yes"]},{"question":"{\"ask\": \"How severe?\"}\nProposed answer: level 2\nDoes the proposed answer fit?","options":["no","yes"]}],"items":[{"ids":[1905,25,198,4754,3494,763,59670,62,1209,763,220,15,11,328,927,763,220,16,2069,5046,62,1209,763,220,22,22,11,328,1272,763,328,101780,13933,5046,62,1209,763,220,17,11,328,927,763,220,16,2069,5046,62,1209,763,220,22,22,11,328,1272,763,328,101780,13933,5046,62,1209,763,220,19,11,328,927,763,220,16,2069,5046,62,1209,763,220,22,22,11,328,1272,763,328,101780,13933,5046,62,1209,763,220,21,11,328,927,763,220,16,2069,5046,62,1209,763,220,22,22,11,328,1272,763,328,101780,8934,13587,271,14162,25,5046,1038,763,328,4199,14938,7285,92,198,1941,18988,4087,25,2119,220,15,198,20810,279,10660,4087,4786,30,198,3670,25,198,4205,8,874,198,5181,8,9542,198,15666,25,318],"slots":[155],"golds":[0],"nopts":[2],"perms":[[0,1]],"types":["score"]},{"ids":[1905,25,198,4754,3494,763,59670,62,1209,763,220,15,11,328,927,763,220,16,2069,5046,62,1209,763,220,22,22,11,328,1272,763,328,101780,13933,5046,62,1209,763,220,17,11,328,927,763,220,16,2069,5046,62,1209,763,220,22,22,11,328,1272,763,328,101780,13933,5046,62,1209,763,220,19,11,328,927,763,220,16,2069,5046,62,1209,763,220,22,22,11,328,1272,763,328,101780,13933,5046,62,1209,763,220,21,11,328,927,763,220,16,2069,5046,62,1209,763,220,22,22,11,328,1272,763,328,101780,8934,13587,271,14162,25,5046,1038,763,328,4199,14938,7285,92,198,1941,18988,4087,25,2119,220,16,198,20810,279,10660,4087,4786,30,198,3670,25,198,4205,8,874,198,5181,8,9542,198,15666,25,318],"slots":[155],"golds":[0],"nopts":[2],"perms":[[0,1]],"types":["score"]},{"ids":[1905,25,198,4754,3494,763,59670,62,1209,763,220,15,11,328,927,763,220,16,2069,5046,62,1209,763,220,22,22,11,328,1272,763,328,101780,13933,5046,62,1209,763,220,17,11,328,927,763,220,16,2069,5046,62,1209,763,220,22,22,11,328,1272,763,328,101780,13933,5046,62,1209,763,220,19,11,328,927,763,220,16,2069,5046,62,1209,763,220,22,22,11,328,1272,763,328,101780,13933,5046,62,1209,763,220,21,11,328,927,763,220,16,2069,5046,62,1209,763,220,22,22,11,328,1272,763,328,101780,8934,13587,271,14162,25,5046,1038,763,328,4199,14938,7285,92,198,1941,18988,4087,25,2119,220,17,198,20810,279,10660,4087,4786,30,198,3670,25,198,4205,8,874,198,5181,8,9542,198,15666,25,318],"slots":[155],"golds":[0],"nopts":[2],"perms":[[0,1]],"types":["score"]}],"context_tokens":115,"unique_tokens":202,"processed_tokens":468,"temperatures":[[1.124],[1.124],[1.124]]},{"name":"score_10","state":{"items":[1,{"_index":77,"text":"保留"},1,{"_index":77,"text":"保留"},1,{"_index":77,"text":"保留"},1,{"_index":77,"text":"保留"}]},"questions":{"score-id":{"type":"score","instructions":{"ask":"How severe?"},"criteria":[" 0: level 0"," 1: level 1"," 2: level 2"," 3: level 3"," 4: level 4"," 5: level 5"," 6: level 6"," 7: level 7"," 8: level 8"," 9: level 9"]}},"rendered_state":"{\"items\": [{\"_index\": 0, \"value\": 1}, {\"_index\": 77, \"text\": \"保留\"}, {\"_index\": 2, \"value\": 1}, {\"_index\": 77, \"text\": \"保留\"}, {\"_index\": 4, \"value\": 1}, {\"_index\": 77, \"text\": \"保留\"}, {\"_index\": 6, \"value\": 1}, {\"_index\": 77, \"text\": \"保留\"}]}","rendered_questions":{"score-id":{"question":"{\"ask\": \"How severe?\"}","options":["0: 0: level 0","1: 1: level 1","2: 2: level 2","3: 3: level 3","4: 4: level 4","5: 5: level 5","6: 6: level 6","7: 7: level 7","8: 8: level 8","9: 9: level 9"],"type":"score","names":[0,1,2,3,4,5,6,7,8,9],"legend":[" 0: level 0"," 1: level 1"," 2: level 2"," 3: level 3"," 4: level 4"," 5: level 5"," 6: level 6"," 7: level 7"," 8: level 8"," 9: level 9"],"isolated":true}},"index":[["score-id","iso",0,10]],"rows":[{"question":"{\"ask\": \"How severe?\"}\nProposed answer: level 0\nDoes the proposed answer fit?","options":["no","yes"]},{"question":"{\"ask\": \"How severe?\"}\nProposed answer: level 1\nDoes the proposed answer fit?","options":["no","yes"]},{"question":"{\"ask\": \"How severe?\"}\nProposed answer: level 2\nDoes the proposed answer fit?","options":["no","yes"]},{"question":"{\"ask\": \"How severe?\"}\nProposed answer: level 3\nDoes the proposed answer fit?","options":["no","yes"]},{"question":"{\"ask\": \"How severe?\"}\nProposed answer: level 4\nDoes the proposed answer fit?","options":["no","yes"]},{"question":"{\"ask\": \"How severe?\"}\nProposed answer: level 5\nDoes the proposed answer fit?","options":["no","yes"]},{"question":"{\"ask\": \"How severe?\"}\nProposed answer: level 6\nDoes the proposed answer fit?","options":["no","yes"]},{"question":"{\"ask\": \"How severe?\"}\nProposed answer: level 7\nDoes the proposed answer fit?","options":["no","yes"]},{"question":"{\"ask\": \"How severe?\"}\nProposed answer: level 8\nDoes the proposed answer fit?","options":["no","yes"]},{"question":"{\"ask\": \"How severe?\"}\nProposed answer: level 9\nDoes the proposed answer fit?","options":["no","yes"]}],"items":[{"ids":[1905,25,198,4754,3494,763,59670,62,1209,763,220,15,11,328,927,763,220,16,2069,5046,62,1209,763,220,22,22,11,328,1272,763,328,101780,13933,5046,62,1209,763,220,17,11,328,927,763,220,16,2069,5046,62,1209,763,220,22,22,11,328,1272,763,328,101780,13933,5046,62,1209,763,220,19,11,328,927,763,220,16,2069,5046,62,1209,763,220,22,22,11,328,1272,763,328,101780,13933,5046,62,1209,763,220,21,11,328,927,763,220,16,2069,5046,62,1209,763,220,22,22,11,328,1272,763,328,101780,8934,13587,271,14162,25,5046,1038,763,328,4199,14938,7285,92,198,1941,18988,4087,25,2119,220,15,198,20810,279,10660,4087,4786,30,198,3670,25,198,4205,8,874,198,5181,8,9542,198,15666,25,318],"slots":[155],"golds":[0],"nopts":[2],"perms":[[0,1]],"types":["score"]},{"ids":[1905,25,198,4754,3494,763,59670,62,1209,763,220,15,11,328,927,763,220,16,2069,5046,62,1209,763,220,22,22,11,328,1272,763,328,101780,13933,5046,62,1209,763,220,17,11,328,927,763,220,16,2069,5046,62,1209,763,220,22,22,11,328,1272,763,328,101780,13933,5046,62,1209,763,220,19,11,328,927,763,220,16,2069,5046,62,1209,763,220,22,22,11,328,1272,763,328,101780,13933,5046,62,1209,763,220,21,11,328,927,763,220,16,2069,5046,62,1209,763,220,22,22,11,328,1272,763,328,101780,8934,13587,271,14162,25,5046,1038,763,328,4199,14938,7285,92,198,1941,18988,4087,25,2119,220,16,198,20810,279,10660,4087,4786,30,198,3670,25,198,4205,8,874,198,5181,8,9542,198,15666,25,318],"slots":[155],"golds":[0],"nopts":[2],"perms":[[0,1]],"types":["score"]},{"ids":[1905,25,198,4754,3494,763,59670,62,1209,763,220,15,11,328,927,763,220,16,2069,5046,62,1209,763,220,22,22,11,328,1272,763,328,101780,13933,5046,62,1209,763,220,17,11,328,927,763,220,16,2069,5046,62,1209,763,220,22,22,11,328,1272,763,328,101780,13933,5046,62,1209,763,220,19,11,328,927,763,220,16,2069,5046,62,1209,763,220,22,22,11,328,1272,763,328,101780,13933,5046,62,1209,763,220,21,11,328,927,763,220,16,2069,5046,62,1209,763,220,22,22,11,328,1272,763,328,101780,8934,13587,271,14162,25,5046,1038,763,328,4199,14938,7285,92,198,1941,18988,4087,25,2119,220,17,198,20810,279,10660,4087,4786,30,198,3670,25,198,4205,8,874,198,5181,8,9542,198,15666,25,318],"slots":[155],"golds":[0],"nopts":[2],"perms":[[0,1]],"types":["score"]},{"ids":[1905,25,198,4754,3494,763,59670,62,1209,763,220,15,11,328,927,763,220,16,2069,5046,62,1209,763,220,22,22,11,328,1272,763,328,101780,13933,5046,62,1209,763,220,17,11,328,927,763,220,16,2069,5046,62,1209,763,220,22,22,11,328,1272,763,328,101780,13933,5046,62,1209,763,220,19,11,328,927,763,220,16,2069,5046,62,1209,763,220,22,22,11,328,1272,763,328,101780,13933,5046,62,1209,763,220,21,11,328,927,763,220,16,2069,5046,62,1209,763,220,22,22,11,328,1272,763,328,101780,8934,13587,271,14162,25,5046,1038,763,328,4199,14938,7285,92,198,1941,18988,4087,25,2119,220,18,198,20810,279,10660,4087,4786,30,198,3670,25,198,4205,8,874,198,5181,8,9542,198,15666,25,318],"slots":[155],"golds":[0],"nopts":[2],"perms":[[0,1]],"types":["score"]},{"ids":[1905,25,198,4754,3494,763,59670,62,1209,763,220,15,11,328,927,763,220,16,2069,5046,62,1209,763,220,22,22,11,328,1272,763,328,101780,13933,5046,62,1209,763,220,17,11,328,927,763,220,16,2069,5046,62,1209,763,220,22,22,11,328,1272,763,328,101780,13933,5046,62,1209,763,220,19,11,328,927,763,220,16,2069,5046,62,1209,763,220,22,22,11,328,1272,763,328,101780,13933,5046,62,1209,763,220,21,11,328,927,763,220,16,2069,5046,62,1209,763,220,22,22,11,328,1272,763,328,101780,8934,13587,271,14162,25,5046,1038,763,328,4199,14938,7285,92,198,1941,18988,4087,25,2119,220,19,198,20810,279,10660,4087,4786,30,198,3670,25,198,4205,8,874,198,5181,8,9542,198,15666,25,318],"slots":[155],"golds":[0],"nopts":[2],"perms":[[0,1]],"types":["score"]},{"ids":[1905,25,198,4754,3494,763,59670,62,1209,763,220,15,11,328,927,763,220,16,2069,5046,62,1209,763,220,22,22,11,328,1272,763,328,101780,13933,5046,62,1209,763,220,17,11,328,927,763,220,16,2069,5046,62,1209,763,220,22,22,11,328,1272,763,328,101780,13933,5046,62,1209,763,220,19,11,328,927,763,220,16,2069,5046,62,1209,763,220,22,22,11,328,1272,763,328,101780,13933,5046,62,1209,763,220,21,11,328,927,763,220,16,2069,5046,62,1209,763,220,22,22,11,328,1272,763,328,101780,8934,13587,271,14162,25,5046,1038,763,328,4199,14938,7285,92,198,1941,18988,4087,25,2119,220,20,198,20810,279,10660,4087,4786,30,198,3670,25,198,4205,8,874,198,5181,8,9542,198,15666,25,318],"slots":[155],"golds":[0],"nopts":[2],"perms":[[0,1]],"types":["score"]},{"ids":[1905,25,198,4754,3494,763,59670,62,1209,763,220,15,11,328,927,763,220,16,2069,5046,62,1209,763,220,22,22,11,328,1272,763,328,101780,13933,5046,62,1209,763,220,17,11,328,927,763,220,16,2069,5046,62,1209,763,220,22,22,11,328,1272,763,328,101780,13933,5046,62,1209,763,220,19,11,328,927,763,220,16,2069,5046,62,1209,763,220,22,22,11,328,1272,763,328,101780,13933,5046,62,1209,763,220,21,11,328,927,763,220,16,2069,5046,62,1209,763,220,22,22,11,328,1272,763,328,101780,8934,13587,271,14162,25,5046,1038,763,328,4199,14938,7285,92,198,1941,18988,4087,25,2119,220,21,198,20810,279,10660,4087,4786,30,198,3670,25,198,4205,8,874,198,5181,8,9542,198,15666,25,318],"slots":[155],"golds":[0],"nopts":[2],"perms":[[0,1]],"types":["score"]},{"ids":[1905,25,198,4754,3494,763,59670,62,1209,763,220,15,11,328,927,763,220,16,2069,5046,62,1209,763,220,22,22,11,328,1272,763,328,101780,13933,5046,62,1209,763,220,17,11,328,927,763,220,16,2069,5046,62,1209,763,220,22,22,11,328,1272,763,328,101780,13933,5046,62,1209,763,220,19,11,328,927,763,220,16,2069,5046,62,1209,763,220,22,22,11,328,1272,763,328,101780,13933,5046,62,1209,763,220,21,11,328,927,763,220,16,2069,5046,62,1209,763,220,22,22,11,328,1272,763,328,101780,8934,13587,271,14162,25,5046,1038,763,328,4199,14938,7285,92,198,1941,18988,4087,25,2119,220,22,198,20810,279,10660,4087,4786,30,198,3670,25,198,4205,8,874,198,5181,8,9542,198,15666,25,318],"slots":[155],"golds":[0],"nopts":[2],"perms":[[0,1]],"types":["score"]},{"ids":[1905,25,198,4754,3494,763,59670,62,1209,763,220,15,11,328,927,763,220,16,2069,5046,62,1209,763,220,22,22,11,328,1272,763,328,101780,13933,5046,62,1209,763,220,17,11,328,927,763,220,16,2069,5046,62,1209,763,220,22,22,11,328,1272,763,328,101780,13933,5046,62,1209,763,220,19,11,328,927,763,220,16,2069,5046,62,1209,763,220,22,22,11,328,1272,763,328,101780,13933,5046,62,1209,763,220,21,11,328,927,763,220,16,2069,5046,62,1209,763,220,22,22,11,328,1272,763,328,101780,8934,13587,271,14162,25,5046,1038,763,328,4199,14938,7285,92,198,1941,18988,4087,25,2119,220,23,198,20810,279,10660,4087,4786,30,198,3670,25,198,4205,8,874,198,5181,8,9542,198,15666,25,318],"slots":[155],"golds":[0],"nopts":[2],"perms":[[0,1]],"types":["score"]},{"ids":[1905,25,198,4754,3494,763,59670,62,1209,763,220,15,11,328,927,763,220,16,2069,5046,62,1209,763,220,22,22,11,328,1272,763,328,101780,13933,5046,62,1209,763,220,17,11,328,927,763,220,16,2069,5046,62,1209,763,220,22,22,11,328,1272,763,328,101780,13933,5046,62,1209,763,220,19,11,328,927,763,220,16,2069,5046,62,1209,763,220,22,22,11,328,1272,763,328,101780,13933,5046,62,1209,763,220,21,11,328,927,763,220,16,2069,5046,62,1209,763,220,22,22,11,328,1272,763,328,101780,8934,13587,271,14162,25,5046,1038,763,328,4199,14938,7285,92,198,1941,18988,4087,25,2119,220,24,198,20810,279,10660,4087,4786,30,198,3670,25,198,4205,8,874,198,5181,8,9542,198,15666,25,318],"slots":[155],"golds":[0],"nopts":[2],"perms":[[0,1]],"types":["score"]}],"context_tokens":115,"unique_tokens":363,"processed_tokens":1560,"temperatures":[[1.124],[1.124],[1.124],[1.124],[1.124],[1.124],[1.124],[1.124],[1.124],[1.124]]},{"name":"noul_desc","state":true,"questions":{"invisible":{"type":"bool","criteria":{"true":{"ask":"refund"}}}},"rendered_state":"true","rendered_questions":{"invisible":{"question":"Which answer fits the context?","options":["no","yes: {\"ask\": \"refund\"}"],"type":"noul","names":[false,true],"legend":null,"isolated":true}},"index":[["invisible","list",0,1]],"rows":[{"question":"Which answer fits the context?","options":["no","yes: {\"ask\": \"refund\"}"]}],"items":[{"ids":[1905,25,198,1802,271,14162,25,15451,4087,17759,279,2193,30,198,3670,25,198,4205,8,874,198,5181,8,9542,25,5046,1038,763,328,66165,8934,198,15666,25,318],"slots":[34],"golds":[0],"nopts":[2],"perms":[[0,1]],"types":["noul"]}],"context_tokens":4,"unique_tokens":35,"processed_tokens":35,"temperatures":[[1.624]]},{"name":"noul_partial","state":"plain","questions":{"id":{"type":"noul","criteria":{"false":false}}},"rendered_state":"plain","rendered_questions":{"id":{"question":"Which answer fits the context?","options":["no: false","yes"],"type":"noul","names":[false,true],"legend":null,"isolated":true}},"index":[["id","list",0,1]],"rows":[{"question":"Which answer fits the context?","options":["no: false","yes"]}],"items":[{"ids":[1905,25,198,20139,271,14162,25,15451,4087,17759,279,2193,30,198,3670,25,198,4205,8,874,25,867,198,5181,8,9542,198,15666,25,318],"slots":[29],"golds":[0],"nopts":[2],"perms":[[0,1]],"types":["noul"]}],"context_tokens":4,"unique_tokens":30,"processed_tokens":30,"temperatures":[[1.624]]},{"name":"structured_mixed","state":{"b":"é","a":[-0.0,1e-05,1e+16,null]},"questions":{"first":{"instructions":["Which?",1],"options":["a","b"]},"second":{"type":"score","instructions":"Level","criteria":{"10":"bad","2":"fine"}},"third":{"type":"noul","instructions":null,"criteria":{"false":"fine"}}},"rendered_state":"{\"b\": \"é\", \"a\": [-0.0, 1e-05, 1e+16, null]}","rendered_questions":{"first":{"question":"[\"Which?\", 1]","options":["a","b"],"type":"choice","names":["a","b"],"legend":null,"isolated":true},"second":{"question":"Level","options":["0: fine","1: bad"],"type":"score","names":[0,1],"legend":["fine","bad"],"isolated":true},"third":{"question":"Which answer fits the context?","options":["no: fine","yes"],"type":"noul","names":[false,true],"legend":null,"isolated":true}},"index":[["first","list",0,1],["second","iso",1,2],["third","list",3,1]],"rows":[{"question":"[\"Which?\", 1]","options":["a","b"]},{"question":"Level\nProposed answer: fine\nDoes the proposed answer fit?","options":["no","yes"]},{"question":"Level\nProposed answer: bad\nDoes the proposed answer fit?","options":["no","yes"]},{"question":"Which answer fits the context?","options":["no: fine","yes"]}],"items":[{"ids":[1905,25,198,4754,65,763,328,933,487,328,64,763,9758,15,13,15,11,220,16,68,12,15,20,11,220,16,68,10,16,21,11,819,13587,271,14162,25,4241,22365,29993,220,16,60,198,3670,25,198,4205,8,264,198,5181,8,292,198,15666,25,318],"slots":[56],"golds":[0],"nopts":[2],"perms":[[0,1]],"types":["choice"]},{"ids":[1905,25,198,4754,65,763,328,933,487,328,64,763,9758,15,13,15,11,220,16,68,12,15,20,11,220,16,68,10,16,21,11,819,13587,271,14162,25,9116,198,1941,18988,4087,25,6699,198,20810,279,10660,4087,4786,30,198,3670,25,198,4205,8,874,198,5181,8,9542,198,15666,25,318],"slots":[64],"golds":[0],"nopts":[2],"perms":[[0,1]],"types":["score"]},{"ids":[1905,25,198,4754,65,763,328,933,487,328,64,763,9758,15,13,15,11,220,16,68,12,15,20,11,220,16,68,10,16,21,11,819,13587,271,14162,25,9116,198,1941,18988,4087,25,3743,198,20810,279,10660,4087,4786,30,198,3670,25,198,4205,8,874,198,5181,8,9542,198,15666,25,318],"slots":[64],"golds":[0],"nopts":[2],"perms":[[0,1]],"types":["score"]},{"ids":[1905,25,198,4754,65,763,328,933,487,328,64,763,9758,15,13,15,11,220,16,68,12,15,20,11,220,16,68,10,16,21,11,819,13587,271,14162,25,15451,4087,17759,279,2193,30,198,3670,25,198,4205,8,874,25,6699,198,5181,8,9542,198,15666,25,318],"slots":[58],"golds":[0],"nopts":[2],"perms":[[0,1]],"types":["noul"]}],"context_tokens":33,"unique_tokens":138,"processed_tokens":246,"temperatures":[[1.164],[1.124],[1.124],[1.624]]},{"name":"empty","state":{"n":1},"questions":{},"rendered_state":"{\"n\": 1}","rendered_questions":{},"index":[],"rows":[],"items":[],"context_tokens":9,"unique_tokens":0,"processed_tokens":0,"temperatures":[]}]} diff --git a/tests/decider/data/responses.json b/tests/decider/data/responses.json new file mode 100644 index 00000000..edcf13a4 --- /dev/null +++ b/tests/decider/data/responses.json @@ -0,0 +1 @@ +[{"name":"choice2","request":{"state":"x","questions":{"c":{"type":"choice","instructions":"q","criteria":{"0":null,"1":null}}}},"logits":[[0.0,0.25]],"probabilities":[[0.44651126861572266,0.5534886717796326]],"expected_answers":{"c":{"type":"choice","choice":"1","confidence":0.107,"x_p_max":0.5535,"certainty":0.0083,"probabilities":{"0":0.4465,"1":0.5535}}}},{"name":"choice3","request":{"state":"x","questions":{"c":{"type":"choice","instructions":"q","criteria":{"0":null,"1":null,"2":null}}}},"logits":[[0.0,0.25,0.5]],"probabilities":[[0.26481956243515015,0.3282663822174072,0.4069140553474426]],"expected_answers":{"c":{"type":"choice","choice":"2","confidence":0.1104,"x_p_max":0.4069,"certainty":0.0138,"probabilities":{"0":0.2648,"1":0.3283,"2":0.4069}}}},{"name":"choice10","request":{"state":"x","questions":{"c":{"type":"choice","instructions":"q","criteria":{"0":null,"1":null,"2":null,"3":null,"4":null,"5":null,"6":null,"7":null,"8":null,"9":null}}}},"logits":[[0.0,0.25,0.5,0.75,1.0,1.25,1.5,0.0,0.25,0.5]],"probabilities":[[0.05442853271961212,0.0674687922000885,0.08363329619169235,0.10367058217525482,0.12850849330425262,0.15929721295833588,0.19746242463588715,0.05442853271961212,0.0674687922000885,0.08363329619169235]],"expected_answers":{"c":{"type":"choice","choice":"6","confidence":0.1083,"x_p_max":0.1975,"certainty":0.0414,"probabilities":{"0":0.0544,"1":0.0675,"2":0.0836,"3":0.1037,"4":0.1285,"5":0.1593,"6":0.1975,"7":0.0544,"8":0.0675,"9":0.0836}}}},{"name":"choice11","request":{"state":"x","questions":{"c":{"type":"choice","instructions":"q","criteria":{"0":null,"1":null,"2":null,"3":null,"4":null,"5":null,"6":null,"7":null,"8":null,"9":null,"10":null}}}},"logits":[[0.0,0.25,0.5,0.75,1.0,1.25,1.5,0.0,0.25,0.5,0.75]],"probabilities":[[0.0493159219622612,0.06113127991557121,0.07577741146087646,0.09393253922462463,0.11643736809492111,0.14433401823043823,0.17891427874565125,0.0493159219622612,0.06113127991557121,0.07577741146087646,0.09393253922462463]],"expected_answers":{"c":{"type":"choice","choice":"6","confidence":0.0968,"x_p_max":0.1789,"certainty":0.036,"probabilities":{"0":0.0493,"1":0.0611,"2":0.0758,"3":0.0939,"4":0.1164,"5":0.1443,"6":0.1789,"7":0.0493,"8":0.0611,"9":0.0758,"10":0.0939}}}},{"name":"choice26","request":{"state":"x","questions":{"c":{"type":"choice","instructions":"q","criteria":{"0":null,"1":null,"2":null,"3":null,"4":null,"5":null,"6":null,"7":null,"8":null,"9":null,"10":null,"11":null,"12":null,"13":null,"14":null,"15":null,"16":null,"17":null,"18":null,"19":null,"20":null,"21":null,"22":null,"23":null,"24":null,"25":null}}}},"logits":[[0.0,0.25,0.5,0.75,1.0,1.25,1.5,0.0,0.25,0.5,0.75,1.0,1.25,1.5,0.0,0.25,0.5,0.75,1.0,1.25,1.5,0.0,0.25,0.5,0.75,1.0]],"probabilities":[[0.019293250516057014,0.023915624246001244,0.029645446687936783,0.03674805164337158,0.0455523319542408,0.056465987116098404,0.06999439001083374,0.019293250516057014,0.023915624246001244,0.029645446687936783,0.03674805164337158,0.0455523319542408,0.056465987116098404,0.06999439001083374,0.019293250516057014,0.023915624246001244,0.029645446687936783,0.03674805164337158,0.0455523319542408,0.056465987116098404,0.06999439001083374,0.019293250516057014,0.023915624246001244,0.029645446687936783,0.03674805164337158,0.0455523319542408]],"expected_answers":{"c":{"type":"choice","choice":"6","confidence":0.0328,"x_p_max":0.07,"certainty":0.0262,"probabilities":{"0":0.0193,"1":0.0239,"2":0.0296,"3":0.0367,"4":0.0456,"5":0.0565,"6":0.07,"7":0.0193,"8":0.0239,"9":0.0296,"10":0.0367,"11":0.0456,"12":0.0565,"13":0.07,"14":0.0193,"15":0.0239,"16":0.0296,"17":0.0367,"18":0.0456,"19":0.0565,"20":0.07,"21":0.0193,"22":0.0239,"23":0.0296,"24":0.0367,"25":0.0456}}}},{"name":"choice255","request":{"state":"x","questions":{"c":{"type":"choice","instructions":"q","criteria":{"0":null,"1":null,"2":null,"3":null,"4":null,"5":null,"6":null,"7":null,"8":null,"9":null,"10":null,"11":null,"12":null,"13":null,"14":null,"15":null,"16":null,"17":null,"18":null,"19":null,"20":null,"21":null,"22":null,"23":null,"24":null,"25":null,"26":null,"27":null,"28":null,"29":null,"30":null,"31":null,"32":null,"33":null,"34":null,"35":null,"36":null,"37":null,"38":null,"39":null,"40":null,"41":null,"42":null,"43":null,"44":null,"45":null,"46":null,"47":null,"48":null,"49":null,"50":null,"51":null,"52":null,"53":null,"54":null,"55":null,"56":null,"57":null,"58":null,"59":null,"60":null,"61":null,"62":null,"63":null,"64":null,"65":null,"66":null,"67":null,"68":null,"69":null,"70":null,"71":null,"72":null,"73":null,"74":null,"75":null,"76":null,"77":null,"78":null,"79":null,"80":null,"81":null,"82":null,"83":null,"84":null,"85":null,"86":null,"87":null,"88":null,"89":null,"90":null,"91":null,"92":null,"93":null,"94":null,"95":null,"96":null,"97":null,"98":null,"99":null,"100":null,"101":null,"102":null,"103":null,"104":null,"105":null,"106":null,"107":null,"108":null,"109":null,"110":null,"111":null,"112":null,"113":null,"114":null,"115":null,"116":null,"117":null,"118":null,"119":null,"120":null,"121":null,"122":null,"123":null,"124":null,"125":null,"126":null,"127":null,"128":null,"129":null,"130":null,"131":null,"132":null,"133":null,"134":null,"135":null,"136":null,"137":null,"138":null,"139":null,"140":null,"141":null,"142":null,"143":null,"144":null,"145":null,"146":null,"147":null,"148":null,"149":null,"150":null,"151":null,"152":null,"153":null,"154":null,"155":null,"156":null,"157":null,"158":null,"159":null,"160":null,"161":null,"162":null,"163":null,"164":null,"165":null,"166":null,"167":null,"168":null,"169":null,"170":null,"171":null,"172":null,"173":null,"174":null,"175":null,"176":null,"177":null,"178":null,"179":null,"180":null,"181":null,"182":null,"183":null,"184":null,"185":null,"186":null,"187":null,"188":null,"189":null,"190":null,"191":null,"192":null,"193":null,"194":null,"195":null,"196":null,"197":null,"198":null,"199":null,"200":null,"201":null,"202":null,"203":null,"204":null,"205":null,"206":null,"207":null,"208":null,"209":null,"210":null,"211":null,"212":null,"213":null,"214":null,"215":null,"216":null,"217":null,"218":null,"219":null,"220":null,"221":null,"222":null,"223":null,"224":null,"225":null,"226":null,"227":null,"228":null,"229":null,"230":null,"231":null,"232":null,"233":null,"234":null,"235":null,"236":null,"237":null,"238":null,"239":null,"240":null,"241":null,"242":null,"243":null,"244":null,"245":null,"246":null,"247":null,"248":null,"249":null,"250":null,"251":null,"252":null,"253":null,"254":null}}}},"logits":[[0.0,0.25,0.5,0.75,1.0,1.25,1.5,0.0,0.25,0.5,0.75,1.0,1.25,1.5,0.0,0.25,0.5,0.75,1.0,1.25,1.5,0.0,0.25,0.5,0.75,1.0,1.25,1.5,0.0,0.25,0.5,0.75,1.0,1.25,1.5,0.0,0.25,0.5,0.75,1.0,1.25,1.5,0.0,0.25,0.5,0.75,1.0,1.25,1.5,0.0,0.25,0.5,0.75,1.0,1.25,1.5,0.0,0.25,0.5,0.75,1.0,1.25,1.5,0.0,0.25,0.5,0.75,1.0,1.25,1.5,0.0,0.25,0.5,0.75,1.0,1.25,1.5,0.0,0.25,0.5,0.75,1.0,1.25,1.5,0.0,0.25,0.5,0.75,1.0,1.25,1.5,0.0,0.25,0.5,0.75,1.0,1.25,1.5,0.0,0.25,0.5,0.75,1.0,1.25,1.5,0.0,0.25,0.5,0.75,1.0,1.25,1.5,0.0,0.25,0.5,0.75,1.0,1.25,1.5,0.0,0.25,0.5,0.75,1.0,1.25,1.5,0.0,0.25,0.5,0.75,1.0,1.25,1.5,0.0,0.25,0.5,0.75,1.0,1.25,1.5,0.0,0.25,0.5,0.75,1.0,1.25,1.5,0.0,0.25,0.5,0.75,1.0,1.25,1.5,0.0,0.25,0.5,0.75,1.0,1.25,1.5,0.0,0.25,0.5,0.75,1.0,1.25,1.5,0.0,0.25,0.5,0.75,1.0,1.25,1.5,0.0,0.25,0.5,0.75,1.0,1.25,1.5,0.0,0.25,0.5,0.75,1.0,1.25,1.5,0.0,0.25,0.5,0.75,1.0,1.25,1.5,0.0,0.25,0.5,0.75,1.0,1.25,1.5,0.0,0.25,0.5,0.75,1.0,1.25,1.5,0.0,0.25,0.5,0.75,1.0,1.25,1.5,0.0,0.25,0.5,0.75,1.0,1.25,1.5,0.0,0.25,0.5,0.75,1.0,1.25,1.5,0.0,0.25,0.5,0.75,1.0,1.25,1.5,0.0,0.25,0.5,0.75,1.0,1.25,1.5,0.0,0.25,0.5,0.75,1.0,1.25,1.5,0.0,0.25,0.5]],"probabilities":[[0.0018894582754001021,0.0023421442601829767,0.002903286600485444,0.003598870476707816,0.004461105912923813,0.005529920104891062,0.006854805629700422,0.0018894582754001021,0.0023421442601829767,0.002903286600485444,0.003598870476707816,0.004461105912923813,0.005529920104891062,0.006854805629700422,0.0018894582754001021,0.0023421442601829767,0.002903286600485444,0.003598870476707816,0.004461105912923813,0.005529920104891062,0.006854805629700422,0.0018894582754001021,0.0023421442601829767,0.002903286600485444,0.003598870476707816,0.004461105912923813,0.005529920104891062,0.006854805629700422,0.0018894582754001021,0.0023421442601829767,0.002903286600485444,0.003598870476707816,0.004461105912923813,0.005529920104891062,0.006854805629700422,0.0018894582754001021,0.0023421442601829767,0.002903286600485444,0.003598870476707816,0.004461105912923813,0.005529920104891062,0.006854805629700422,0.0018894582754001021,0.0023421442601829767,0.002903286600485444,0.003598870476707816,0.004461105912923813,0.005529920104891062,0.006854805629700422,0.0018894582754001021,0.0023421442601829767,0.002903286600485444,0.003598870476707816,0.004461105912923813,0.005529920104891062,0.006854805629700422,0.0018894582754001021,0.0023421442601829767,0.002903286600485444,0.003598870476707816,0.004461105912923813,0.005529920104891062,0.006854805629700422,0.0018894582754001021,0.0023421442601829767,0.002903286600485444,0.003598870476707816,0.004461105912923813,0.005529920104891062,0.006854805629700422,0.0018894582754001021,0.0023421442601829767,0.002903286600485444,0.003598870476707816,0.004461105912923813,0.005529920104891062,0.006854805629700422,0.0018894582754001021,0.0023421442601829767,0.002903286600485444,0.003598870476707816,0.004461105912923813,0.005529920104891062,0.006854805629700422,0.0018894582754001021,0.0023421442601829767,0.002903286600485444,0.003598870476707816,0.004461105912923813,0.005529920104891062,0.006854805629700422,0.0018894582754001021,0.0023421442601829767,0.002903286600485444,0.003598870476707816,0.004461105912923813,0.005529920104891062,0.006854805629700422,0.0018894582754001021,0.0023421442601829767,0.002903286600485444,0.003598870476707816,0.004461105912923813,0.005529920104891062,0.006854805629700422,0.0018894582754001021,0.0023421442601829767,0.002903286600485444,0.003598870476707816,0.004461105912923813,0.005529920104891062,0.006854805629700422,0.0018894582754001021,0.0023421442601829767,0.002903286600485444,0.003598870476707816,0.004461105912923813,0.005529920104891062,0.006854805629700422,0.0018894582754001021,0.0023421442601829767,0.002903286600485444,0.003598870476707816,0.004461105912923813,0.005529920104891062,0.006854805629700422,0.0018894582754001021,0.0023421442601829767,0.002903286600485444,0.003598870476707816,0.004461105912923813,0.005529920104891062,0.006854805629700422,0.0018894582754001021,0.0023421442601829767,0.002903286600485444,0.003598870476707816,0.004461105912923813,0.005529920104891062,0.006854805629700422,0.0018894582754001021,0.0023421442601829767,0.002903286600485444,0.003598870476707816,0.004461105912923813,0.005529920104891062,0.006854805629700422,0.0018894582754001021,0.0023421442601829767,0.002903286600485444,0.003598870476707816,0.004461105912923813,0.005529920104891062,0.006854805629700422,0.0018894582754001021,0.0023421442601829767,0.002903286600485444,0.003598870476707816,0.004461105912923813,0.005529920104891062,0.006854805629700422,0.0018894582754001021,0.0023421442601829767,0.002903286600485444,0.003598870476707816,0.004461105912923813,0.005529920104891062,0.006854805629700422,0.0018894582754001021,0.0023421442601829767,0.002903286600485444,0.003598870476707816,0.004461105912923813,0.005529920104891062,0.006854805629700422,0.0018894582754001021,0.0023421442601829767,0.002903286600485444,0.003598870476707816,0.004461105912923813,0.005529920104891062,0.006854805629700422,0.0018894582754001021,0.0023421442601829767,0.002903286600485444,0.003598870476707816,0.004461105912923813,0.005529920104891062,0.006854805629700422,0.0018894582754001021,0.0023421442601829767,0.002903286600485444,0.003598870476707816,0.004461105912923813,0.005529920104891062,0.006854805629700422,0.0018894582754001021,0.0023421442601829767,0.002903286600485444,0.003598870476707816,0.004461105912923813,0.005529920104891062,0.006854805629700422,0.0018894582754001021,0.0023421442601829767,0.002903286600485444,0.003598870476707816,0.004461105912923813,0.005529920104891062,0.006854805629700422,0.0018894582754001021,0.0023421442601829767,0.002903286600485444,0.003598870476707816,0.004461105912923813,0.005529920104891062,0.006854805629700422,0.0018894582754001021,0.0023421442601829767,0.002903286600485444,0.003598870476707816,0.004461105912923813,0.005529920104891062,0.006854805629700422,0.0018894582754001021,0.0023421442601829767,0.002903286600485444,0.003598870476707816,0.004461105912923813,0.005529920104891062,0.006854805629700422,0.0018894582754001021,0.0023421442601829767,0.002903286600485444,0.003598870476707816,0.004461105912923813,0.005529920104891062,0.006854805629700422,0.0018894582754001021,0.0023421442601829767,0.002903286600485444,0.003598870476707816,0.004461105912923813,0.005529920104891062,0.006854805629700422,0.0018894582754001021,0.0023421442601829767,0.002903286600485444,0.003598870476707816,0.004461105912923813,0.005529920104891062,0.006854805629700422,0.0018894582754001021,0.0023421442601829767,0.002903286600485444]],"expected_answers":{"c":{"type":"choice","choice":"6","confidence":0.0029,"x_p_max":0.0069,"certainty":0.0158,"probabilities":{"0":0.0019,"1":0.0023,"2":0.0029,"3":0.0036,"4":0.0045,"5":0.0055,"6":0.0069,"7":0.0019,"8":0.0023,"9":0.0029,"10":0.0036,"11":0.0045,"12":0.0055,"13":0.0069,"14":0.0019,"15":0.0023,"16":0.0029,"17":0.0036,"18":0.0045,"19":0.0055,"20":0.0069,"21":0.0019,"22":0.0023,"23":0.0029,"24":0.0036,"25":0.0045,"26":0.0055,"27":0.0069,"28":0.0019,"29":0.0023,"30":0.0029,"31":0.0036,"32":0.0045,"33":0.0055,"34":0.0069,"35":0.0019,"36":0.0023,"37":0.0029,"38":0.0036,"39":0.0045,"40":0.0055,"41":0.0069,"42":0.0019,"43":0.0023,"44":0.0029,"45":0.0036,"46":0.0045,"47":0.0055,"48":0.0069,"49":0.0019,"50":0.0023,"51":0.0029,"52":0.0036,"53":0.0045,"54":0.0055,"55":0.0069,"56":0.0019,"57":0.0023,"58":0.0029,"59":0.0036,"60":0.0045,"61":0.0055,"62":0.0069,"63":0.0019,"64":0.0023,"65":0.0029,"66":0.0036,"67":0.0045,"68":0.0055,"69":0.0069,"70":0.0019,"71":0.0023,"72":0.0029,"73":0.0036,"74":0.0045,"75":0.0055,"76":0.0069,"77":0.0019,"78":0.0023,"79":0.0029,"80":0.0036,"81":0.0045,"82":0.0055,"83":0.0069,"84":0.0019,"85":0.0023,"86":0.0029,"87":0.0036,"88":0.0045,"89":0.0055,"90":0.0069,"91":0.0019,"92":0.0023,"93":0.0029,"94":0.0036,"95":0.0045,"96":0.0055,"97":0.0069,"98":0.0019,"99":0.0023,"100":0.0029,"101":0.0036,"102":0.0045,"103":0.0055,"104":0.0069,"105":0.0019,"106":0.0023,"107":0.0029,"108":0.0036,"109":0.0045,"110":0.0055,"111":0.0069,"112":0.0019,"113":0.0023,"114":0.0029,"115":0.0036,"116":0.0045,"117":0.0055,"118":0.0069,"119":0.0019,"120":0.0023,"121":0.0029,"122":0.0036,"123":0.0045,"124":0.0055,"125":0.0069,"126":0.0019,"127":0.0023,"128":0.0029,"129":0.0036,"130":0.0045,"131":0.0055,"132":0.0069,"133":0.0019,"134":0.0023,"135":0.0029,"136":0.0036,"137":0.0045,"138":0.0055,"139":0.0069,"140":0.0019,"141":0.0023,"142":0.0029,"143":0.0036,"144":0.0045,"145":0.0055,"146":0.0069,"147":0.0019,"148":0.0023,"149":0.0029,"150":0.0036,"151":0.0045,"152":0.0055,"153":0.0069,"154":0.0019,"155":0.0023,"156":0.0029,"157":0.0036,"158":0.0045,"159":0.0055,"160":0.0069,"161":0.0019,"162":0.0023,"163":0.0029,"164":0.0036,"165":0.0045,"166":0.0055,"167":0.0069,"168":0.0019,"169":0.0023,"170":0.0029,"171":0.0036,"172":0.0045,"173":0.0055,"174":0.0069,"175":0.0019,"176":0.0023,"177":0.0029,"178":0.0036,"179":0.0045,"180":0.0055,"181":0.0069,"182":0.0019,"183":0.0023,"184":0.0029,"185":0.0036,"186":0.0045,"187":0.0055,"188":0.0069,"189":0.0019,"190":0.0023,"191":0.0029,"192":0.0036,"193":0.0045,"194":0.0055,"195":0.0069,"196":0.0019,"197":0.0023,"198":0.0029,"199":0.0036,"200":0.0045,"201":0.0055,"202":0.0069,"203":0.0019,"204":0.0023,"205":0.0029,"206":0.0036,"207":0.0045,"208":0.0055,"209":0.0069,"210":0.0019,"211":0.0023,"212":0.0029,"213":0.0036,"214":0.0045,"215":0.0055,"216":0.0069,"217":0.0019,"218":0.0023,"219":0.0029,"220":0.0036,"221":0.0045,"222":0.0055,"223":0.0069,"224":0.0019,"225":0.0023,"226":0.0029,"227":0.0036,"228":0.0045,"229":0.0055,"230":0.0069,"231":0.0019,"232":0.0023,"233":0.0029,"234":0.0036,"235":0.0045,"236":0.0055,"237":0.0069,"238":0.0019,"239":0.0023,"240":0.0029,"241":0.0036,"242":0.0045,"243":0.0055,"244":0.0069,"245":0.0019,"246":0.0023,"247":0.0029,"248":0.0036,"249":0.0045,"250":0.0055,"251":0.0069,"252":0.0019,"253":0.0023,"254":0.0029}}}},{"name":"score2","request":{"state":"x","questions":{"s":{"type":"score","instructions":"q","criteria":["level0","level1"]}}},"logits":[[0.0,0.0],[0.0,0.25]],"probabilities":[[0.5,0.5],[0.44462311267852783,0.5553768873214722]],"expected_answers":{"s":{"type":"score","score":0.53,"confidence":0.0525,"x_p_max":0.5262,"certainty":0.002,"legend":{"0":"level0","1":"level1"},"probabilities":{"0":0.4738,"1":0.5262},"level_fit":{"0":0.5,"1":0.5554},"fit_mass":1.0554}}},{"name":"score3","request":{"state":"x","questions":{"s":{"type":"score","instructions":"q","criteria":["level0","level1","level2"]}}},"logits":[[0.0,0.0],[0.0,0.25],[0.0,0.5]],"probabilities":[[0.5,0.5],[0.44462311267852783,0.5553768873214722],[0.3905883729457855,0.6094117164611816]],"expected_answers":{"s":{"type":"score","score":1.07,"confidence":0.0,"x_p_max":0.3661,"certainty":0.003,"legend":{"0":"level0","1":"level1","2":"level2"},"probabilities":{"0":0.3003,"1":0.3336,"2":0.3661},"level_fit":{"0":0.5,"1":0.5554,"2":0.6094},"fit_mass":1.6648}}},{"name":"score10","request":{"state":"x","questions":{"s":{"type":"score","instructions":"q","criteria":["level0","level1","level2","level3","level4","level5","level6","level7","level8","level9"]}}},"logits":[[0.0,0.0],[0.0,0.25],[0.0,0.5],[0.0,0.75],[0.0,1.0],[0.0,1.25],[0.0,1.5],[0.0,1.75],[0.0,2.0],[0.0,2.25]],"probabilities":[[0.5,0.5],[0.44462311267852783,0.5553768873214722],[0.3905883729457855,0.6094117164611816],[0.33911067247390747,0.6608893275260925],[0.291175901889801,0.7088240385055542],[0.24747967720031738,0.7525203824043274],[0.2084127515554428,0.791587233543396],[0.17408624291419983,0.8259137868881226],[0.14438223838806152,0.8556177020072937],[0.11901620775461197,0.880983829498291]],"expected_answers":{"s":{"type":"score","score":4.99,"confidence":0.0,"x_p_max":0.1234,"certainty":0.0067,"legend":{"0":"level0","1":"level1","2":"level2","3":"level3","4":"level4","5":"level5","6":"level6","7":"level7","8":"level8","9":"level9"},"probabilities":{"0":0.07,"1":0.0778,"2":0.0853,"3":0.0925,"4":0.0993,"5":0.1054,"6":0.1108,"7":0.1157,"8":0.1198,"9":0.1234},"level_fit":{"0":0.5,"1":0.5554,"2":0.6094,"3":0.6609,"4":0.7088,"5":0.7525,"6":0.7916,"7":0.8259,"8":0.8556,"9":0.881},"fit_mass":7.1411}}},{"name":"score_zero","request":{"state":"x","questions":{"s":{"type":"score","instructions":"q","criteria":["a","b","c"]}}},"logits":[[1000.0,-1000.0],[1000.0,-1000.0],[1000.0,-1000.0]],"probabilities":[[1.0,0.0],[1.0,0.0],[1.0,0.0]],"expected_answers":{"s":{"type":"score","score":0.0,"confidence":0.0,"x_p_max":0.0,"certainty":1.0,"legend":{"0":"a","1":"b","2":"c"},"probabilities":{"0":0.0,"1":0.0,"2":0.0},"level_fit":{"0":0.0,"1":0.0,"2":0.0},"fit_mass":0.0}}},{"name":"noul","request":{"state":"x","questions":{"n":{"type":"noul","instructions":"q"}}},"logits":[[1.0,0.0]],"probabilities":[[0.6492544412612915,0.3507455885410309]],"expected_answers":{"n":{"type":"noul","noul":0.3507}}},{"name":"choice_uniform255","request":{"state":"x","questions":{"c":{"type":"choice","instructions":"q","criteria":{"0":null,"1":null,"2":null,"3":null,"4":null,"5":null,"6":null,"7":null,"8":null,"9":null,"10":null,"11":null,"12":null,"13":null,"14":null,"15":null,"16":null,"17":null,"18":null,"19":null,"20":null,"21":null,"22":null,"23":null,"24":null,"25":null,"26":null,"27":null,"28":null,"29":null,"30":null,"31":null,"32":null,"33":null,"34":null,"35":null,"36":null,"37":null,"38":null,"39":null,"40":null,"41":null,"42":null,"43":null,"44":null,"45":null,"46":null,"47":null,"48":null,"49":null,"50":null,"51":null,"52":null,"53":null,"54":null,"55":null,"56":null,"57":null,"58":null,"59":null,"60":null,"61":null,"62":null,"63":null,"64":null,"65":null,"66":null,"67":null,"68":null,"69":null,"70":null,"71":null,"72":null,"73":null,"74":null,"75":null,"76":null,"77":null,"78":null,"79":null,"80":null,"81":null,"82":null,"83":null,"84":null,"85":null,"86":null,"87":null,"88":null,"89":null,"90":null,"91":null,"92":null,"93":null,"94":null,"95":null,"96":null,"97":null,"98":null,"99":null,"100":null,"101":null,"102":null,"103":null,"104":null,"105":null,"106":null,"107":null,"108":null,"109":null,"110":null,"111":null,"112":null,"113":null,"114":null,"115":null,"116":null,"117":null,"118":null,"119":null,"120":null,"121":null,"122":null,"123":null,"124":null,"125":null,"126":null,"127":null,"128":null,"129":null,"130":null,"131":null,"132":null,"133":null,"134":null,"135":null,"136":null,"137":null,"138":null,"139":null,"140":null,"141":null,"142":null,"143":null,"144":null,"145":null,"146":null,"147":null,"148":null,"149":null,"150":null,"151":null,"152":null,"153":null,"154":null,"155":null,"156":null,"157":null,"158":null,"159":null,"160":null,"161":null,"162":null,"163":null,"164":null,"165":null,"166":null,"167":null,"168":null,"169":null,"170":null,"171":null,"172":null,"173":null,"174":null,"175":null,"176":null,"177":null,"178":null,"179":null,"180":null,"181":null,"182":null,"183":null,"184":null,"185":null,"186":null,"187":null,"188":null,"189":null,"190":null,"191":null,"192":null,"193":null,"194":null,"195":null,"196":null,"197":null,"198":null,"199":null,"200":null,"201":null,"202":null,"203":null,"204":null,"205":null,"206":null,"207":null,"208":null,"209":null,"210":null,"211":null,"212":null,"213":null,"214":null,"215":null,"216":null,"217":null,"218":null,"219":null,"220":null,"221":null,"222":null,"223":null,"224":null,"225":null,"226":null,"227":null,"228":null,"229":null,"230":null,"231":null,"232":null,"233":null,"234":null,"235":null,"236":null,"237":null,"238":null,"239":null,"240":null,"241":null,"242":null,"243":null,"244":null,"245":null,"246":null,"247":null,"248":null,"249":null,"250":null,"251":null,"252":null,"253":null,"254":null}}}},"logits":[[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0]],"probabilities":[[0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563,0.003921568859368563]],"expected_answers":{"c":{"type":"choice","choice":"0","confidence":0.0,"x_p_max":0.0039,"certainty":0.0,"probabilities":{"0":0.0039,"1":0.0039,"2":0.0039,"3":0.0039,"4":0.0039,"5":0.0039,"6":0.0039,"7":0.0039,"8":0.0039,"9":0.0039,"10":0.0039,"11":0.0039,"12":0.0039,"13":0.0039,"14":0.0039,"15":0.0039,"16":0.0039,"17":0.0039,"18":0.0039,"19":0.0039,"20":0.0039,"21":0.0039,"22":0.0039,"23":0.0039,"24":0.0039,"25":0.0039,"26":0.0039,"27":0.0039,"28":0.0039,"29":0.0039,"30":0.0039,"31":0.0039,"32":0.0039,"33":0.0039,"34":0.0039,"35":0.0039,"36":0.0039,"37":0.0039,"38":0.0039,"39":0.0039,"40":0.0039,"41":0.0039,"42":0.0039,"43":0.0039,"44":0.0039,"45":0.0039,"46":0.0039,"47":0.0039,"48":0.0039,"49":0.0039,"50":0.0039,"51":0.0039,"52":0.0039,"53":0.0039,"54":0.0039,"55":0.0039,"56":0.0039,"57":0.0039,"58":0.0039,"59":0.0039,"60":0.0039,"61":0.0039,"62":0.0039,"63":0.0039,"64":0.0039,"65":0.0039,"66":0.0039,"67":0.0039,"68":0.0039,"69":0.0039,"70":0.0039,"71":0.0039,"72":0.0039,"73":0.0039,"74":0.0039,"75":0.0039,"76":0.0039,"77":0.0039,"78":0.0039,"79":0.0039,"80":0.0039,"81":0.0039,"82":0.0039,"83":0.0039,"84":0.0039,"85":0.0039,"86":0.0039,"87":0.0039,"88":0.0039,"89":0.0039,"90":0.0039,"91":0.0039,"92":0.0039,"93":0.0039,"94":0.0039,"95":0.0039,"96":0.0039,"97":0.0039,"98":0.0039,"99":0.0039,"100":0.0039,"101":0.0039,"102":0.0039,"103":0.0039,"104":0.0039,"105":0.0039,"106":0.0039,"107":0.0039,"108":0.0039,"109":0.0039,"110":0.0039,"111":0.0039,"112":0.0039,"113":0.0039,"114":0.0039,"115":0.0039,"116":0.0039,"117":0.0039,"118":0.0039,"119":0.0039,"120":0.0039,"121":0.0039,"122":0.0039,"123":0.0039,"124":0.0039,"125":0.0039,"126":0.0039,"127":0.0039,"128":0.0039,"129":0.0039,"130":0.0039,"131":0.0039,"132":0.0039,"133":0.0039,"134":0.0039,"135":0.0039,"136":0.0039,"137":0.0039,"138":0.0039,"139":0.0039,"140":0.0039,"141":0.0039,"142":0.0039,"143":0.0039,"144":0.0039,"145":0.0039,"146":0.0039,"147":0.0039,"148":0.0039,"149":0.0039,"150":0.0039,"151":0.0039,"152":0.0039,"153":0.0039,"154":0.0039,"155":0.0039,"156":0.0039,"157":0.0039,"158":0.0039,"159":0.0039,"160":0.0039,"161":0.0039,"162":0.0039,"163":0.0039,"164":0.0039,"165":0.0039,"166":0.0039,"167":0.0039,"168":0.0039,"169":0.0039,"170":0.0039,"171":0.0039,"172":0.0039,"173":0.0039,"174":0.0039,"175":0.0039,"176":0.0039,"177":0.0039,"178":0.0039,"179":0.0039,"180":0.0039,"181":0.0039,"182":0.0039,"183":0.0039,"184":0.0039,"185":0.0039,"186":0.0039,"187":0.0039,"188":0.0039,"189":0.0039,"190":0.0039,"191":0.0039,"192":0.0039,"193":0.0039,"194":0.0039,"195":0.0039,"196":0.0039,"197":0.0039,"198":0.0039,"199":0.0039,"200":0.0039,"201":0.0039,"202":0.0039,"203":0.0039,"204":0.0039,"205":0.0039,"206":0.0039,"207":0.0039,"208":0.0039,"209":0.0039,"210":0.0039,"211":0.0039,"212":0.0039,"213":0.0039,"214":0.0039,"215":0.0039,"216":0.0039,"217":0.0039,"218":0.0039,"219":0.0039,"220":0.0039,"221":0.0039,"222":0.0039,"223":0.0039,"224":0.0039,"225":0.0039,"226":0.0039,"227":0.0039,"228":0.0039,"229":0.0039,"230":0.0039,"231":0.0039,"232":0.0039,"233":0.0039,"234":0.0039,"235":0.0039,"236":0.0039,"237":0.0039,"238":0.0039,"239":0.0039,"240":0.0039,"241":0.0039,"242":0.0039,"243":0.0039,"244":0.0039,"245":0.0039,"246":0.0039,"247":0.0039,"248":0.0039,"249":0.0039,"250":0.0039,"251":0.0039,"252":0.0039,"253":0.0039,"254":0.0039}}}}] diff --git a/tests/decider/executor.rs b/tests/decider/executor.rs new file mode 100644 index 00000000..c5a73532 --- /dev/null +++ b/tests/decider/executor.rs @@ -0,0 +1,87 @@ +use super::*; +use crate::Kind; +fn row(ids: Vec, candidates: Vec, position: usize) -> RowInput { + RowInput { + ids, + candidate_ids: candidates, + readout_position: position, + question_id: "q".into(), + level_index: None, + kind: Kind::Choice, + temperature: 1.164, + } +} +#[test] +fn validates_every_row_before_dispatch() { + let labels = [32, 33, 34]; + assert!(validate_rows(&[row(vec![1, 2], vec![32, 33], 1)], &labels).is_ok()); + for input in [ + row(vec![], vec![32, 33], 0), + row(vec![1, 2], vec![32, 33], 0), + row(vec![VOCAB as u32], vec![32, 33], 0), + row(vec![1], vec![33, 32], 0), + row(vec![1], vec![], 0), + row(vec![1], vec![32], 0), + row(vec![1; 36865], vec![32, 33], 36864), + ] { + assert!(validate_rows(&[row(vec![1], vec![32, 33], 0), input], &labels).is_err()); + } + assert!(validate_rows(&[], &labels).is_ok()); +} + +#[test] +#[ignore = "requires the built Qwen CUDA library and an available NVIDIA GPU"] +fn bf16_head_rounds_before_fp32_calibration() { + let library = std::env::var("DECIDER_CUDA_LIB").unwrap(); + cuda::load(Path::new(&library)).unwrap(); + cuda::set_device(0).unwrap(); + let mut weights = vec![0; PADDED_LABELS * HIDDEN * 2]; + for (offset, value) in [ + (0, 1.0), + (2, 0.00390625), + (HIDDEN * 2, 1.0), + (HIDDEN * 2 + 2, 0.015625), + ] { + weights[offset..offset + 2].copy_from_slice(&bf16::from_f32(value).to_le_bytes()); + } + let head = Head::new(&weights, 1).unwrap(); + let mut hidden = vec![0.0; HIDDEN]; + hidden[..2].copy_from_slice(&[1.0, 1.0]); + assert_eq!(head.project(&hidden, 2).unwrap(), vec![1.0, 1.015625]); + assert_eq!(head.project(&hidden, 255).unwrap()[254], 0.0); + hidden[0] = f32::NAN; + assert!(head.project(&hidden, 2).is_err()); +} + +#[test] +#[ignore = "requires the built Qwen CUDA library and an available NVIDIA GPU"] +fn batched_head_preserves_rows_counts_and_bf16_rounding() { + cuda::load(Path::new(&std::env::var("DECIDER_CUDA_LIB").unwrap())).unwrap(); + cuda::set_device(0).unwrap(); + let mut weights = vec![0; PADDED_LABELS * HIDDEN * 2]; + for (offset, value) in [ + (0, 1.0), + (2, 0.00390625), + (HIDDEN * 2, 1.0), + (HIDDEN * 2 + 2, 0.015625), + ] { + weights[offset..offset + 2].copy_from_slice(&bf16::from_f32(value).to_le_bytes()); + } + let head = Head::new(&weights, 2).unwrap(); + let mut a = vec![0.0; HIDDEN]; + a[..2].copy_from_slice(&[1.0, 1.0]); + let mut b = vec![0.0; HIDDEN]; + b[..2].copy_from_slice(&[2.0, 1.0]); + let first = head.project_batch(&[&a, &b], &[2, 255]).unwrap(); + assert_eq!(first[0], vec![1.0, 1.015625]); + assert_eq!(&first[1][..2], &[2.0, 2.015625]); + assert_eq!(first[1].len(), 255); + assert!(first[1][2..].iter().all(|&x| x == 0.0)); + let reversed = head.project_batch(&[&b, &a], &[255, 2]).unwrap(); + assert_eq!(reversed[0], first[1]); + assert_eq!(reversed[1], first[0]); + assert!(head.project_batch(&[&a, &b, &a], &[2, 2, 2]).is_err()); + assert!(head.project_batch(&[&a, &b], &[2]).is_err()); + b[0] = f32::NAN; + assert!(head.project_batch(&[&a, &b], &[2, 2]).is_err()); +} diff --git a/tests/decider/fail_head.c b/tests/decider/fail_head.c new file mode 100644 index 00000000..07418ff3 --- /dev/null +++ b/tests/decider/fail_head.c @@ -0,0 +1,18 @@ +// Test-only wrapper: real CUDA math, fail the second selected-head projection. +#define _GNU_SOURCE +#include +#include +#include "ops.h" + +typedef int (*gemm_fn)(void*, const void*, const void*, void*, int, int, int, int, void*); +int cs1_gemm(void* handle, const void* x, const void* w, void* y, + int m, int n, int k, int ldy, void* stream) { + static unsigned projection_calls; + gemm_fn original = (gemm_fn)dlsym(RTLD_NEXT, "cs1_gemm"); + if (!original) return 999; + if (m >= 1 && m <= 16 && n == 256 && k == 2048) { + fprintf(stderr, "Decider test projection %u rows %d\n", ++projection_calls, m); + if (projection_calls == 2) return 2; + } + return original(handle, x, w, y, m, n, k, ldy, stream); +} diff --git a/tests/decider/gpu.rs b/tests/decider/gpu.rs new file mode 100644 index 00000000..8bec730e --- /dev/null +++ b/tests/decider/gpu.rs @@ -0,0 +1,137 @@ +use omni_decider_native::engine::Engine; +use std::path::Path; + +#[tokio::test] +async fn missing_artifacts_fail_before_cuda_initialization() { + let dir = std::env::temp_dir().join(format!("missing-decider-{}", std::process::id())); + let result = Engine::load(&dir, Path::new("unavailable-cuda-library.so")).await; + assert!(result.is_err()); + assert!(format!("{:#}", result.err().unwrap()).contains("No such file")); +} + +#[tokio::test] +#[ignore = "requires pinned Decider-2B checkpoint and NVIDIA CUDA"] +async fn complete_eager_requests_preserve_order_and_recover_from_validation_errors() { + let dir = std::env::var("DECIDER_MODEL").unwrap(); + let library = std::env::var("DECIDER_CUDA_LIB").unwrap(); + let engine = Engine::load(Path::new(&dir), Path::new(&library)) + .await + .unwrap(); + let raw = br#"{"state":"The ticket requests a refund.","questions":{"category":{"type":"choice","instructions":"Choose the category.","criteria":{"refund":"Refund","shipping":"Shipping"}},"refund":{"type":"noul","instructions":"The customer requests a refund."},"urgency":{"type":"score","instructions":"How urgent is the ticket?","criteria":["low urgency","high urgency"]}}}"#; + let first = engine.predict(raw).await.unwrap(); + assert_eq!( + first["answers"] + .as_object() + .unwrap() + .keys() + .map(String::as_str) + .collect::>(), + vec!["category", "refund", "urgency"] + ); + assert_eq!(first["usage"]["output_tokens"], 0); + assert!( + engine + .predict(br#"{"state":0,"questions":{"bad":{"criteria":["one"]}}}"#) + .await + .is_err() + ); + assert!(engine.executor.is_ready()); + assert_eq!(engine.predict(raw).await.unwrap(), first); + let empty = engine + .predict(br#"{"state":{},"questions":{}}"#) + .await + .unwrap(); + assert!(empty["answers"].as_object().unwrap().is_empty()); + assert_eq!(empty["usage"]["input_tokens"], 0); + assert_eq!(engine.health()["status"], "ready"); +} + +#[tokio::test] +#[ignore = "requires pinned Decider-2B checkpoint and NVIDIA CUDA; near-limit allocation"] +async fn maximum_complete_row_grows_scratch_without_leaking_state() { + use omni_decider_native::Limits; + use serde_json::json; + let dir = std::env::var("DECIDER_MODEL").unwrap(); + let library = std::env::var("DECIDER_CUDA_LIB").unwrap(); + let engine = Engine::load(Path::new(&dir), Path::new(&library)) + .await + .unwrap(); + let short = br#"{"state":"A customer requests a refund.","questions":{"q":{"instructions":"Choose a team.","criteria":["billing","shipping"]}}}"#; + let baseline = engine.predict(short).await.unwrap(); + let request = |n: usize| { + json!({"state":"hello ".repeat(45000),"questions":{"q":{"instructions":format!("Choose.{}"," x".repeat(n)),"criteria":["billing","shipping"]}}}).to_string() + }; + let (mut low, mut high) = (0usize, 8000usize); + while low < high { + let mid = (low + high).div_ceil(2); + if engine.processor.prepare(request(mid).as_bytes()).is_ok() { + low = mid; + } else { + high = mid - 1; + } + } + let raw = request(low); + let prepared = engine.processor.prepare(raw.as_bytes()).unwrap(); + assert_eq!(prepared.rows[0].ids.len(), Limits::default().max_row_tokens); + assert!( + engine + .processor + .prepare(request(low + 1).as_bytes()) + .is_err() + ); + let first = engine.predict(raw.as_bytes()).await.unwrap(); + assert!( + first["answers"]["q"]["probabilities"] + .as_object() + .unwrap() + .values() + .all(|value| value.as_f64().unwrap().is_finite()) + ); + assert_eq!(engine.predict(raw.as_bytes()).await.unwrap(), first); + assert_eq!(engine.predict(short).await.unwrap(), baseline); + assert!(engine.executor.is_ready()); +} + +#[tokio::test] +#[ignore = "requires pinned Decider-2B checkpoint and NVIDIA CUDA; request packing"] +async fn packed_mixed_rows_preserve_order_usage_and_recovery() { + use omni_decider_native::batching::BatchLimits; + use serde_json::json; + let dir = std::env::var("DECIDER_MODEL").unwrap(); + let library = std::env::var("DECIDER_CUDA_LIB").unwrap(); + let engine = Engine::load_with_batch( + Path::new(&dir), + Path::new(&library), + BatchLimits::new(4, 4096).unwrap(), + ) + .await + .unwrap(); + let raw=json!({"state":"A customer requests a refund for a damaged item.","questions":{"first":{"instructions":"Choose the team.","criteria":["refund","shipping"]},"score":{"type":"score","instructions":"Severity.","criteria":["low","medium","high"]},"last":{"type":"noul","instructions":"The item is damaged."}}}).to_string(); + let first = engine.predict(raw.as_bytes()).await.unwrap(); + assert_eq!( + first["answers"] + .as_object() + .unwrap() + .keys() + .map(String::as_str) + .collect::>(), + vec!["first", "score", "last"] + ); + assert_eq!(engine.health()["batch_limits"]["rows"], 4); + let long=json!({"state":"The item is damaged. ".repeat(1000),"questions":{"q":{"instructions":"Choose the team.","criteria":["refund","shipping"]}}}).to_string(); + assert!( + engine.processor.prepare(long.as_bytes()).unwrap().rows[0] + .ids + .len() + > 4096 + ); + assert!(engine.predict(long.as_bytes()).await.is_ok()); + assert_eq!(engine.predict(raw.as_bytes()).await.unwrap(), first); + assert!( + engine + .predict(br#"{"state":0,"questions":{"q":{"criteria":["only"]}}}"#) + .await + .is_err() + ); + assert_eq!(engine.predict(raw.as_bytes()).await.unwrap(), first); +} diff --git a/tests/decider/runner.rs b/tests/decider/runner.rs new file mode 100644 index 00000000..4502c5fb --- /dev/null +++ b/tests/decider/runner.rs @@ -0,0 +1,35 @@ +//! Diagnostic JSON-lines runner retaining exact prepared rows and raw logits. +use anyhow::{Context, Result}; +use omni_decider_native::engine::Engine; +use serde_json::json; +use std::{ + io::{self, BufRead}, + path::Path, +}; + +#[tokio::main] +async fn main() -> Result<()> { + let args: Vec = std::env::args().collect(); + let model = args + .get(1) + .context("usage: decider-run MODEL_DIR CUDA_LIBRARY")?; + let library = args + .get(2) + .context("usage: decider-run MODEL_DIR CUDA_LIBRARY")?; + let engine = Engine::load(Path::new(model), Path::new(library)).await?; + for line in io::stdin().lock().lines() { + let line = line?; + let prepared = engine.processor.prepare(line.as_bytes())?; + let rows:Vec<_> = prepared.rows.iter().map(|row| json!({"ids":row.ids,"candidate_ids":row.candidate_ids,"readout_position":row.readout_position,"question_id":row.question_id,"level_index":row.level_index,"temperature":row.temperature})).collect(); + let logits = engine + .executor + .execute(&engine.scheduler, prepared.rows) + .await?; + let response = prepared.context.finish(logits.clone())?; + println!( + "{}", + json!({"rows":rows,"logits":logits,"response":response}) + ); + } + Ok(()) +} diff --git a/tests/decider/test_download.py b/tests/decider/test_download.py new file mode 100644 index 00000000..e193a8c5 --- /dev/null +++ b/tests/decider/test_download.py @@ -0,0 +1,28 @@ +import importlib.util +from pathlib import Path +import tempfile +import unittest + +MODULE_PATH = Path(__file__).parents[2] / "recipe/decider/download_weights.py" +spec = importlib.util.spec_from_file_location("decider_download", MODULE_PATH) +module = importlib.util.module_from_spec(spec) +spec.loader.exec_module(module) + + +class Verification(unittest.TestCase): + def test_rejects_content_and_size_mismatches(self): + expected = (3, "ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad") + with tempfile.TemporaryDirectory() as root: + path = Path(root) / "model" + path.write_bytes(b"abc") + module.verify(path, expected) + path.write_bytes(b"abd") + with self.assertRaisesRegex(ValueError, "checksum"): + module.verify(path, expected) + path.write_bytes(b"abcd") + with self.assertRaisesRegex(ValueError, "size"): + module.verify(path, expected) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/decider/test_reference.py b/tests/decider/test_reference.py new file mode 100644 index 00000000..2130f8c9 --- /dev/null +++ b/tests/decider/test_reference.py @@ -0,0 +1,29 @@ +import hashlib +import importlib.util +from pathlib import Path +import tempfile +import unittest + +spec = importlib.util.spec_from_file_location("decider_reference", Path(__file__).with_name("verify_reference.py")) +module = importlib.util.module_from_spec(spec) +spec.loader.exec_module(module) + + +class ReferenceProvenance(unittest.TestCase): + def test_source_hashes_reject_modified_and_missing_reference(self): + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + source = root / "model.py" + source.write_bytes(b"abc") + expected = {"model.py": hashlib.sha256(b"abc").hexdigest()} + self.assertEqual(module.verify_sources(root, expected), expected) + source.write_bytes(b"abd") + with self.assertRaisesRegex(ValueError, "reference checksum"): + module.verify_sources(root, expected) + source.unlink() + with self.assertRaises(FileNotFoundError): + module.verify_sources(root, expected) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/decider/verify_http.py b/tests/decider/verify_http.py new file mode 100644 index 00000000..595f6c64 --- /dev/null +++ b/tests/decider/verify_http.py @@ -0,0 +1,124 @@ +#!/usr/bin/env python3 +"""Exercise real native and frontend sockets, readiness, rejection and recovery.""" +import argparse +import concurrent.futures +import json +import os +from pathlib import Path +import subprocess +import time +import urllib.error +import urllib.request + + +def request(base, path, body=None, content_type="application/json"): + headers = {"Content-Type": content_type} if body is not None else {} + req = urllib.request.Request(base + path, data=body, headers=headers) + try: + with urllib.request.urlopen(req, timeout=120) as response: + return response.status, response.read() + except urllib.error.HTTPError as response: + return response.code, response.read() + + +def wait_ready(base, child, log): + deadline = time.monotonic() + 180 + while time.monotonic() < deadline: + if child.poll() is not None: + raise RuntimeError(f"worker exited: {log.read_text()[-3000:]}") + try: + status, raw = request(base, "/health") + if status == 200: + return json.loads(raw) + except (OSError, urllib.error.URLError): + pass + time.sleep(0.1) + raise TimeoutError("readiness timeout") + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--worker", type=Path, required=True) + parser.add_argument("--frontend", type=Path, required=True) + parser.add_argument("--model", type=Path, required=True) + parser.add_argument("--library", type=Path, required=True) + parser.add_argument("--parity-output", type=Path, required=True) + parser.add_argument("--output", type=Path, required=True) + parser.add_argument("--execution", default="eager") + parser.add_argument("--native-records", type=Path, help="verify_modes records.jsonl for the selected mode") + args = parser.parse_args() + args.output.mkdir(parents=True, exist_ok=True) + worker_base, frontend_base = "http://127.0.0.1:18110", "http://127.0.0.1:18111" + environment = os.environ.copy() + environment.update(DECIDER_MODEL=str(args.model), DECIDER_CUDA_LIB=str(args.library), DECIDER_PORT="18110", CUA_S1_GRAPH="0") + processes, logs, checks = [], [], [] + try: + path = args.output / "worker.log" + logs.append(path.open("w")) + worker = subprocess.Popen([str(args.worker)], env=environment, stdout=logs[-1], stderr=subprocess.STDOUT) + processes.append(worker) + metadata = wait_ready(worker_base, worker, path) + assert metadata["checkpoint_revision"] == "533964dae8be954c5b5e19fa4948e48408094c1e" + assert metadata["execution"] == args.execution and metadata["dtype"] == "bfloat16" + environment.update(OMNI_JEV_BIND="127.0.0.1:18111", OMNI_JEV_BACKEND_URL=worker_base) + path = args.output / "frontend.log" + logs.append(path.open("w")) + frontend = subprocess.Popen([str(args.frontend)], env=environment, stdout=logs[-1], stderr=subprocess.STDOUT) + processes.append(frontend) + wait_ready(frontend_base, frontend, path) + protocol = json.loads((args.parity_output / "protocol.json").read_text()) + native_records = [json.loads(line) for line in (args.parity_output / "native.jsonl").read_text().splitlines()] + if args.native_records: + native_records = [item["record"] for line in args.native_records.read_text().splitlines() + for item in [json.loads(line)] if item["phase"] == "parity"] + assert len(native_records) == len(protocol["cases"]) + for case, native in zip(protocol["cases"], native_records): + body = json.dumps(case["request"], ensure_ascii=False).encode() + direct, proxy = request(worker_base, "/v1/systemone", body), request(frontend_base, "/v1/systemone", body) + assert direct[0] == proxy[0] == 200 + assert direct[1] == proxy[1] + assert json.loads(direct[1]) == native["response"] + checks.append({"name": case["name"], "status": direct[0], "byte_parity": True, "response": json.loads(direct[1])}) + negative = [ + (b"{}", "text/plain", 415), + (b"{invalid", "application/json", 422), + (b'{"state":0,"state":1,"questions":{}}', "application/json", 422), + (b'{"state":0,"questions":{"q":{"instructions":"q","criteria":["one"]}}}', "application/json", 422), + (b'{"state":0,"questions":{},"independent":false}', "application/json", 422), + (b'{"state":0,"questions":{"q":{"type":"image","instructions":"q","criteria":["a","b"]}}}', "application/json", 422), + ] + for i, (body, content_type, expected) in enumerate(negative): + direct = request(worker_base, "/v1/systemone", body, content_type) + proxy = request(frontend_base, "/v1/systemone", body, content_type) + assert direct[0] == expected and proxy == direct, (i, direct[0], proxy[0], expected) + assert isinstance(json.loads(direct[1])["detail"], str) + checks.append({"name": f"invalid_{i}", "status": expected, "byte_parity": True}) + oversized = request(worker_base, "/v1/systemone", b" " * (8 * 1024 * 1024 + 1)) + assert oversized[0] == 413 + checks.append({"name": "body_limit", "status": 413}) + raw = json.dumps(protocol["cases"][13]["request"]).encode() + baseline = request(worker_base, "/v1/systemone", raw) + with concurrent.futures.ThreadPoolExecutor(max_workers=4) as pool: + concurrent_results = list(pool.map(lambda i: request(frontend_base if i % 2 else worker_base, "/v1/systemone", raw), range(12))) + assert all(result == baseline for result in concurrent_results) + checks.append({"name": "concurrent_direct_and_proxy", "requests": len(concurrent_results), "byte_parity": True}) + status, health = request(worker_base, "/health") + assert status == 200 and json.loads(health)["status"] == "ready" + checks.append({"name": "recovery_health", "status": status}) + (args.output / "summary.json").write_text(json.dumps({"passed": len(checks), "metadata": metadata, "checks": checks}, indent=2, ensure_ascii=False) + "\n") + print(json.dumps({"passed": len(checks), "concurrent_requests": len(concurrent_results)}), flush=True) + finally: + for child in reversed(processes): + if child.poll() is None: + child.terminate() + try: + child.wait(timeout=30) + except subprocess.TimeoutExpired: + child.kill() + child.wait() + for handle in logs: + handle.close() + + +if __name__ == "__main__": + main() diff --git a/tests/decider/verify_modes.py b/tests/decider/verify_modes.py new file mode 100644 index 00000000..bb5d439b --- /dev/null +++ b/tests/decider/verify_modes.py @@ -0,0 +1,104 @@ +#!/usr/bin/env python3 +"""Sequential matched diagnostic-runner comparisons; raw output and frozen plan retained.""" +import argparse +import hashlib +import json +import os +from pathlib import Path +import statistics +import subprocess +import time + +from verify_reference import GATES, compare + + +def digest(path): + return hashlib.sha256(Path(path).read_bytes()).hexdigest() + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--plan", type=Path, required=True) + parser.add_argument("--output", type=Path, required=True) + args = parser.parse_args() + plan = json.loads(args.plan.read_text()) + assert plan["repetitions"] == 2 + parity = Path(plan["parity_output"]) + protocol = json.loads((parity / "protocol.json").read_text()) + reference = [json.loads(line) for line in (parity / "reference.jsonl").read_text().splitlines()] + baseline = [json.loads(line) for line in (parity / "native.jsonl").read_text().splitlines()] + cases = protocol["cases"] + assert len(cases) == len(reference) == len(baseline) + selected = [i for i, case in enumerate(cases) if case["name"] in plan["timed_cases"]] + assert len(selected) == len(plan["timed_cases"]) + args.output.mkdir(parents=True, exist_ok=False) + frozen = {**plan, "gates": GATES, "baseline_protocol": protocol, + "input_hashes": {name: digest(parity / name) for name in ("protocol.json", "reference.jsonl", "native.jsonl")}, + "runner_sha256": digest(__file__), "comparison_sha256": digest(Path(__file__).with_name("verify_reference.py")), "library_sha256": digest(plan["library"]), + "binary_hashes": {mode["name"]: digest(mode["binary"]) for mode in plan["modes"]}, + "timing": "wall roundtrip: request JSON encoding/write through diagnostic stdout JSON decode; includes CPU preparation, GPU synchronization and serialization; excludes startup and one workload warmup; concurrency=1"} + (args.output / "protocol.json").write_text(json.dumps(frozen, indent=2) + "\n") + summaries = [] + for mode in plan["modes"]: + directory = args.output / mode["name"] + directory.mkdir() + env = dict(os.environ) + for key in ("CUA_S1_GRAPH", "DECIDER_GRAPH", "DECIDER_PREFIX", "DECIDER_FIXED", "DECIDER_BATCH_MAX_ROWS", "DECIDER_BATCH_MAX_TOKENS"): + env.pop(key, None) + env.update(mode["env"]) + with (directory / "stderr.log").open("w") as stderr, (directory / "records.jsonl").open("w") as records: + start = time.perf_counter() + child = subprocess.Popen([mode["binary"], plan["model"], plan["library"]], env=env, + stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=stderr, text=True) + def run(request): + begun = time.perf_counter() + child.stdin.write(json.dumps(request, ensure_ascii=False) + "\n") + child.stdin.flush() + line = child.stdout.readline() + if not line: + raise RuntimeError(f"{mode['name']} terminated: {child.poll()}; see stderr.log") + value = json.loads(line) + return value, (time.perf_counter() - begun) * 1000 + try: + run({"state": "", "questions": {}}) + startup_ms = (time.perf_counter() - start) * 1000 + checks, samples = [], [] + for i, case in enumerate(cases): + got, ms = run(case["request"]) + records.write(json.dumps({"phase": "parity", "name": case["name"], "elapsed_ms": ms, "record": got}, ensure_ascii=False) + "\n") + records.flush() + assert got["rows"] == baseline[i]["rows"], (mode["name"], case["name"], "prepared rows") + checks.append({"name": case["name"], **compare(reference[i]["response"], got["response"])}) + for i in selected: + got, ms = run(cases[i]["request"]) + records.write(json.dumps({"phase": "warmup", "name": cases[i]["name"], "elapsed_ms": ms, "record": got}, ensure_ascii=False) + "\n") + for repetition in range(plan["repetitions"]): + for i in selected: + got, ms = run(cases[i]["request"]) + sample = {"name": cases[i]["name"], "repetition": repetition, "elapsed_ms": ms} + samples.append(sample) + records.write(json.dumps({"phase": "measured", **sample, "record": got}, ensure_ascii=False) + "\n") + records.flush() + assert got["rows"] == baseline[i]["rows"], (mode["name"], cases[i]["name"], "measured rows") + compare(reference[i]["response"], got["response"]) + memory = subprocess.check_output(["nvidia-smi", "--query-gpu=memory.used", "--format=csv,noheader,nounits"], text=True).strip() + summary = {"name": mode["name"], "startup_to_empty_diagnostic_ms": startup_ms, + "checks": checks, "samples": samples, "device_used_mib_at_end": int(memory), + "warm_ms": {name: {"median": statistics.median(values), "min": min(values), "max": max(values)} + for name in plan["timed_cases"] + for values in [[s["elapsed_ms"] for s in samples if s["name"] == name]]}} + summaries.append(summary) + (directory / "summary.json").write_text(json.dumps(summary, indent=2) + "\n") + print(json.dumps(summary), flush=True) + finally: + child.stdin.close() + try: + child.wait(timeout=30) + except subprocess.TimeoutExpired: + child.kill() + child.wait() + (args.output / "summary.json").write_text(json.dumps({"modes": summaries}, indent=2) + "\n") + + +if __name__ == "__main__": + main() diff --git a/tests/decider/verify_reference.py b/tests/decider/verify_reference.py new file mode 100644 index 00000000..309504f0 --- /dev/null +++ b/tests/decider/verify_reference.py @@ -0,0 +1,176 @@ +#!/usr/bin/env python3 +"""Pinned full-checkpoint eager parity; no latency or task-quality claim.""" +import argparse +import gc +import hashlib +import json +import os +from pathlib import Path +import subprocess +import sys +import types + +GATES = {"max_probability_drift": 0.02, "max_score_drift": 0.1, "discrete_margin": 0.05} +REFERENCE_REVISION = "50d0be0d7cb43d2066965ce5fa7f3fe4e489a60f" +REFERENCE_FILES = { + "model.py": "74e0ca564bb52f7c2bd74c23cb34b16e0004ceadf7a13816f223c68829b5e1c5", + "prompt.py": "f21ae1016a11ca3500b5a8e2e8c3ef524434d196bbd4891199b96a33b3ad6314", + "prompt_fast.py": "fbfcc3a381a89785174cb9d29083ed14d404c7273d123ba8fe2b69584649ef25", + "systemone.py": "884e4b0bae36f5e09dbd689b4390e94320cbbd77f7b8d675e50490b1d7c61d34", + "temperature.py": "b940c253e5626145e806195cf5b2bc0de61461de633c07b174a924d18ceeab88", +} + + +def verify_sources(directory, expected): + verified = {} + for name, checksum in expected.items(): + actual = hashlib.sha256((directory / name).read_bytes()).hexdigest() + if actual != checksum: + raise ValueError(f"reference checksum mismatch: {name}") + verified[name] = actual + return verified + + +def workloads(): + cases = [] + for n in (2, 3, 10, 11, 26, 255): + cases.append((f"choice_{n}", {"state": "The customer asks for a refund for a damaged order.", "questions": { + "category": {"type": "choice", "instructions": "Choose the matching category.", "criteria": { + f"option_{i}": "Refund request" if i == n - 1 else f"Unrelated category {i}" for i in range(n)}}}})) + for n in (2, 3, 10): + cases.append((f"score_{n}", {"state": "The customer received a damaged item and requests help immediately.", "questions": { + "urgency": {"type": "score", "instructions": "How urgent is the customer request?", "criteria": [f"{i}. Urgency level {i}" for i in range(n)]}}})) + cases.extend([ + ("noul_true", {"state": "The customer requests a refund.", "questions": {"refund": {"type": "noul", "instructions": "The customer requests a refund."}}}), + ("noul_false", {"state": "The customer asks when the package will arrive.", "questions": {"refund": {"type": "noul", "instructions": "The customer requests a refund."}}}), + ("noul_descriptions", {"state": "The parcel is delivered.", "questions": {"delivered": {"type": "noul", "criteria": {"true": "The parcel is delivered.", "false": "The parcel is still in transit."}}}}), + ("structured_unicode", {"state": {"客户": "订单损坏，请退款", "orders": [{"id": i, "status": "damaged" if i == 7 else "delivered"} for i in range(9)]}, "questions": { + "z_last": {"type": "choice", "instructions": {"task": "Choose the status of orders[7]."}, "criteria": {"damaged": "Damaged", "delivered": "Delivered"}}, + "a_first": {"type": "score", "instructions": "Severity of the damage", "criteria": {"0": "Low", "1": "Medium", "2": "High"}}, + "m_middle": {"type": "noul", "instructions": "The customer requests a refund."}}}), + ("mixed", {"state": "A customer requests a refund because a delivery was damaged.", "questions": { + "route": {"type": "choice", "instructions": "Choose the team.", "criteria": {"billing": "Refund and billing", "delivery": "Shipping tracking", "sales": "New purchases"}}, + "refund": {"type": "noul", "instructions": "The customer requests a refund."}, + "severity": {"type": "score", "instructions": "Severity of the damage", "criteria": ["Minor", "Moderate", "Severe"]}}}), + ("long_context", {"state": "The item is damaged. " * 700 + "The customer requests a refund.", "questions": {"route": {"type": "choice", "instructions": "Choose the team.", "criteria": {"billing": "Refund and billing", "shipping": "Delivery status"}}}}), + ("empty", {"state": {}, "questions": {}}), + ]) + # Repetition and reverse question order test request/state isolation. + cases.append(("mixed_reordered", {"state": cases[13][1]["state"], "questions": dict(reversed(list(cases[13][1]["questions"].items())))})) + cases.append(("choice_repeat", cases[0][1])) + return cases + + +def compare(reference, native): + assert native["model"] == "decider-2b-v11" + assert native["usage"] == reference["usage"] + assert list(native["answers"]) == list(reference["answers"]) + worst_p, worst_score = 0.0, 0.0 + mismatches, low_margin = [], [] + for key, ref in reference["answers"].items(): + got = native["answers"][key] + assert set(got) == set(ref), (key, set(got), set(ref)) + assert got["type"] == ref["type"] + if ref["type"] == "noul": + drift = abs(got["noul"] - ref["noul"]) + else: + assert list(got["probabilities"]) == list(ref["probabilities"]) + drift = max(abs(got["probabilities"][k] - v) for k, v in ref["probabilities"].items()) + if ref["type"] == "choice": + probabilities = sorted(ref["probabilities"].values(), reverse=True) + if ref["choice"] != got["choice"]: + (mismatches if probabilities[0] - probabilities[1] >= GATES["discrete_margin"] else low_margin).append(key) + else: + assert got["legend"] == ref["legend"] + worst_score = max(worst_score, abs(got["score"] - ref["score"])) + drift = max(drift, max(abs(got["level_fit"][k] - v) for k, v in ref["level_fit"].items())) + worst_p = max(worst_p, drift) + assert worst_p <= GATES["max_probability_drift"], ("probability", worst_p) + assert worst_score <= GATES["max_score_drift"], ("score", worst_score) + assert not mismatches, ("discrete", mismatches) + return {"max_probability_drift": worst_p, "max_score_drift": worst_score, "low_margin_disagreements": low_margin} + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--model", type=Path, required=True) + parser.add_argument("--reference", type=Path, required=True) + parser.add_argument("--binary", type=Path, required=True) + parser.add_argument("--library", type=Path, required=True) + parser.add_argument("--output", type=Path, required=True) + args = parser.parse_args() + args.output.mkdir(parents=True, exist_ok=True) + sources = verify_sources(args.reference / "decider", REFERENCE_FILES) + cases = workloads() + manifest = {"reference_revision": REFERENCE_REVISION, "reference_source_sha256": sources, "gates": GATES, "runs": 1, "cases": [{"name": name, "request": request} for name, request in cases]} + # Freeze the manifest before executing either implementation. + (args.output / "protocol.json").write_text(json.dumps(manifest, ensure_ascii=False, indent=2) + "\n") + inputs = "".join(json.dumps(request, ensure_ascii=False) + "\n" for _, request in cases) + (args.output / "requests.jsonl").write_text(inputs) + with (args.output / "native.stderr.log").open("w") as stderr: + native = subprocess.run([str(args.binary), str(args.model), str(args.library)], input=inputs, capture_output=False, + stdout=subprocess.PIPE, stderr=stderr, text=True, check=True) + (args.output / "native.jsonl").write_text(native.stdout) + native = [json.loads(line) for line in native.stdout.splitlines()] + assert len(native) == len(cases) + + # Import only the pinned reference package, with no remote-code execution. + package = types.ModuleType("decider") + package.__path__ = [str(args.reference / "decider")] + sys.modules["decider"] = package + import torch + from transformers import AutoTokenizer + from decider import systemone as s1, prompt_fast, temperature as tt, prompt + from decider.model import DecisionModel + + torch.set_num_threads(4) + torch.backends.cuda.matmul.allow_tf32 = False + tokenizer = AutoTokenizer.from_pretrained(args.model, local_files_only=True) + scalar, temperatures = tt.from_config(json.loads((args.model / "decider_config.json").read_text()))[0] + del scalar + labels = prompt.label_table(tokenizer)[1] + model = DecisionModel(str(args.model), dtype=torch.bfloat16, grad_ckpt=False).to("cuda").eval() + model.lm.model.config.use_cache = False + results = [] + with (args.output / "reference.jsonl").open("w") as records, torch.inference_mode(): + for (name, request), got in zip(cases, native): + rendered = {key: s1.render_question(value) for key, value in request["questions"].items()} + plans, index = s1.plan_rows(rendered, True) + items, _ = prompt_fast.build_rows(tokenizer, s1.render_state(request["state"]), + [[(plan["question"], plan["options"])] for plan in plans], max_ctx_tokens=32768) + kinds = s1.row_types(rendered, index) + assert len(got["rows"]) == len(items), name + logits, probabilities = [], [] + max_logit_drift = 0.0 + for i, (item, kind) in enumerate(zip(items, kinds)): + count = item["nopts"][0] + row = got["rows"][i] + assert row["ids"] == item["ids"], (name, i, "tokens") + assert row["candidate_ids"] == labels[:count], (name, i, "candidate IDs") + assert row["readout_position"] == item["slots"][0] == len(item["ids"]) - 1 + ids = torch.tensor([item["ids"]], device="cuda", dtype=torch.long) + lg = model.slot_logits(ids, torch.ones_like(ids), torch.tensor([item["slots"][0]], device="cuda"), + torch.tensor([0], device="cuda"), torch.tensor([count], device="cuda"))[0, :count] + logits.append(lg.cpu().tolist()) + probabilities.append(tt.scaled_softmax(lg[None], [temperatures[kind]])[0].cpu().tolist()) + max_logit_drift = max(max_logit_drift, max(abs(a - b) for a, b in zip(logits[-1], got["logits"][i]))) + response = {"model": "decider-2b-v11", "answers": s1.assemble(rendered, index, probabilities), + "usage": {"input_tokens": s1.unique_tokens(items), "output_tokens": 0}} + records.write(json.dumps({"name": name, "logits": logits, "probabilities": probabilities, "response": response}, ensure_ascii=False) + "\n") + records.flush() + result = {"name": name, "rows": len(items), "processed_tokens": sum(len(item["ids"]) for item in items), + "max_logit_drift": max_logit_drift, **compare(response, got["response"])} + results.append(result) + print(json.dumps(result), flush=True) + del model + gc.collect() + torch.cuda.empty_cache() + summary = {"passed": len(results), "cases": results, "gates": GATES, + "torch": torch.__version__, "transformers": __import__("transformers").__version__, + "gpu": torch.cuda.get_device_name(), "native_sha256": hashlib.sha256(args.binary.read_bytes()).hexdigest(), + "library_sha256": hashlib.sha256(args.library.read_bytes()).hexdigest()} + (args.output / "summary.json").write_text(json.dumps(summary, indent=2) + "\n") + + +if __name__ == "__main__": + main() diff --git a/tests/decider/verify_retirement.py b/tests/decider/verify_retirement.py new file mode 100644 index 00000000..8bb5e125 --- /dev/null +++ b/tests/decider/verify_retirement.py @@ -0,0 +1,68 @@ +#!/usr/bin/env python3 +"""Fault-injection check using an isolated real worker after its genuine warmup.""" +import argparse +import json +import os +from pathlib import Path +import subprocess +import time +from verify_http import request, wait_ready + + +def memory(pid): + result = subprocess.run(["nvidia-smi", "--query-compute-apps=pid,used_gpu_memory", "--format=csv,noheader,nounits"], capture_output=True, text=True, check=True) + for line in result.stdout.splitlines(): + fields = line.split(",") + if fields[0].strip() == str(pid): + return int(fields[1].strip()) + return 0 + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--worker", type=Path, required=True) + parser.add_argument("--model", type=Path, required=True) + parser.add_argument("--wrapper", type=Path, required=True) + parser.add_argument("--output", type=Path, required=True) + parser.add_argument("--batched", action="store_true") + args = parser.parse_args() + args.output.mkdir(parents=True, exist_ok=True) + log = args.output / "worker.log" + environment = os.environ.copy() + environment.update(DECIDER_MODEL=str(args.model), DECIDER_CUDA_LIB=str(args.wrapper), DECIDER_PORT="18112", CUA_S1_GRAPH="0") + with log.open("w") as output: + child = subprocess.Popen([str(args.worker)], env=environment, stdout=output, stderr=subprocess.STDOUT) + try: + base = "http://127.0.0.1:18112" + health = wait_ready(base, child, log) + assert health["status"] == "ready" + before = memory(child.pid) + raw = b'{"state":"The item is damaged.","questions":{"q":{"instructions":"Choose a category.","criteria":["refund","shipping"]}}}' + if args.batched: + raw = b'{"state":"The item is damaged.","questions":{"q":{"type":"score","instructions":"Severity.","criteria":["low","high"]}}}' + status, body = request(base, "/v1/systemone", raw) + assert status == 503 and json.loads(body)["detail"] == "model inference failed" + status, body = request(base, "/health") + assert status == 503 and json.loads(body)["status"] == "unavailable" + for _ in range(2): + status, body = request(base, "/v1/systemone", raw) + assert status == 503 and json.loads(body)["detail"] == "model unavailable" + after = memory(child.pid) + # The checksum-pinned 3.76GB model allocation must actually be retired. + assert before - after > 3000, (before, after) + text = log.read_text() + assert text.count("Decider test projection") == 2 + assert "Decider test projection 2" in text + if args.batched: + assert "Decider test projection 2 rows 2" in text + result = {"passed": True, "projection_calls": 2, "health_after": "unavailable", "response_status": 503, + "sampled_before_mib": before, "sampled_after_mib": after, "later_requests_refused": 2} + (args.output / "summary.json").write_text(json.dumps(result, indent=2) + "\n") + print(json.dumps(result), flush=True) + finally: + child.terminate() + child.wait(timeout=30) + + +if __name__ == "__main__": + main() From 4eee3b020a2a3ad61e37a7b97962ac43bbbe2a3f Mon Sep 17 00:00:00 2001 From: levius <2114377220@qq.com> Date: Fri, 9 Oct 2026 22:38:54 +0800 Subject: [PATCH 4/4] test(decider): pin packing mode for retirement checks --- .github/workflows/ci.yml | 2 + tests/decider/test_retirement.py | 80 ++++++++++++++++++++++++++++++ tests/decider/verify_retirement.py | 6 ++- 3 files changed, 87 insertions(+), 1 deletion(-) create mode 100644 tests/decider/test_retirement.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 594ef0f0..c44863d2 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -42,3 +42,5 @@ jobs: - run: python -m pip install -r benchmarks/requirements.txt - run: python benchmarks/bench.py validate benchmarks/smoke.jsonl - run: python -m unittest discover -s tests/benchmarks -p 'test_*.py' -v + - name: Decider validation helpers + run: python -m unittest discover -s tests/decider -p 'test_*.py' -v diff --git a/tests/decider/test_retirement.py b/tests/decider/test_retirement.py new file mode 100644 index 00000000..30e2399a --- /dev/null +++ b/tests/decider/test_retirement.py @@ -0,0 +1,80 @@ +import contextlib +import importlib.util +import io +import json +import os +from pathlib import Path +import tempfile +import unittest +from unittest import mock + +spec = importlib.util.spec_from_file_location( + "decider_retirement", Path(__file__).with_name("verify_retirement.py") +) +module = importlib.util.module_from_spec(spec) +spec.loader.exec_module(module) + + +class WorkerLaunch(unittest.TestCase): + def run_cli(self, batched, inherited): + with tempfile.TemporaryDirectory() as directory: + output = Path(directory) + args = ["verify_retirement.py", "--worker", "worker", "--model", "model", + "--wrapper", "wrapper.so", "--output", str(output)] + if batched: + args.append("--batched") + + def launch(*args, **kwargs): + # Simulate external GPU evidence so this host-only test can + # inspect the actual CLI's worker launch and request boundary. + rows = 2 if batched else 1 + (output / "worker.log").write_text( + f"Decider test projection 1 rows 1\n" + f"Decider test projection 2 rows {rows}\n" + ) + return child + + child = mock.Mock(pid=123) + replies = [(503, b'{"detail":"model inference failed"}'), + (503, b'{"status":"unavailable"}'), + (503, b'{"detail":"model unavailable"}'), + (503, b'{"detail":"model unavailable"}')] + with mock.patch.dict(os.environ, inherited, clear=True), \ + mock.patch.object(module.subprocess, "Popen", side_effect=launch) as popen, \ + mock.patch.object(module, "wait_ready", return_value={"status": "ready"}), \ + mock.patch.object(module, "memory", side_effect=[4000, 0]), \ + mock.patch.object(module, "request", side_effect=replies) as request, \ + mock.patch("sys.argv", args), contextlib.redirect_stdout(io.StringIO()): + module.main() + child.terminate.assert_called_once() + child.wait.assert_called_once_with(timeout=30) + raw = json.loads(request.call_args_list[0].args[2]) + self.assertEqual(raw["questions"]["q"].get("type", "choice"), + "score" if batched else "choice") + self.assertEqual(len(raw["questions"]["q"]["criteria"]), 2) + return popen.call_args.kwargs["env"] + + def assert_mode(self, environment, rows): + self.assertEqual(environment["DECIDER_BATCH_MAX_ROWS"], str(rows)) + self.assertEqual(environment["DECIDER_BATCH_MAX_TOKENS"], "4096") + for name in ("CUA_S1_GRAPH", "DECIDER_GRAPH", "DECIDER_PREFIX", "DECIDER_FIXED"): + self.assertEqual(environment[name], "0", name) + + def test_batched_launch_without_exported_limits(self): + self.assert_mode(self.run_cli(True, {}), 2) + + def test_batched_launch_overrides_inherited_limits_and_modes(self): + inherited = {"DECIDER_BATCH_MAX_ROWS": "1", "DECIDER_BATCH_MAX_TOKENS": "1", + "CUA_S1_GRAPH": "1", "DECIDER_GRAPH": "1", + "DECIDER_PREFIX": "auto", "DECIDER_FIXED": "1"} + self.assert_mode(self.run_cli(True, inherited), 2) + + def test_unbatched_launch_overrides_inherited_limits_and_modes(self): + inherited = {"DECIDER_BATCH_MAX_ROWS": "4", "DECIDER_BATCH_MAX_TOKENS": "1", + "CUA_S1_GRAPH": "1", "DECIDER_GRAPH": "1", + "DECIDER_PREFIX": "1", "DECIDER_FIXED": "1"} + self.assert_mode(self.run_cli(False, inherited), 1) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/decider/verify_retirement.py b/tests/decider/verify_retirement.py index 8bb5e125..ca80bed6 100644 --- a/tests/decider/verify_retirement.py +++ b/tests/decider/verify_retirement.py @@ -29,7 +29,11 @@ def main(): args.output.mkdir(parents=True, exist_ok=True) log = args.output / "worker.log" environment = os.environ.copy() - environment.update(DECIDER_MODEL=str(args.model), DECIDER_CUDA_LIB=str(args.wrapper), DECIDER_PORT="18112", CUA_S1_GRAPH="0") + environment.update( + DECIDER_MODEL=str(args.model), DECIDER_CUDA_LIB=str(args.wrapper), DECIDER_PORT="18112", + DECIDER_BATCH_MAX_ROWS="2" if args.batched else "1", DECIDER_BATCH_MAX_TOKENS="4096", + CUA_S1_GRAPH="0", DECIDER_GRAPH="0", DECIDER_PREFIX="0", DECIDER_FIXED="0", + ) with log.open("w") as output: child = subprocess.Popen([str(args.worker)], env=environment, stdout=output, stderr=subprocess.STDOUT) try: