From ce802bb0c968875948c141245824f19ed73c52e8 Mon Sep 17 00:00:00 2001 From: baseRT sync Date: Thu, 22 Oct 2026 18:15:25 +1100 Subject: [PATCH 1/2] sync: open-core source for 0.4.0 Generated by tools/sync_public.py from the internal tree at 5ea5d3f0f. Only allow-listed paths are included; tuned quant profiles and internal tooling are excluded by rule. --- base-convert/Cargo.lock | 18 +- base-convert/Cargo.toml | 2 +- base-convert/crates/base-convert/Cargo.toml | 3 + base-convert/crates/base-convert/src/hub.rs | 15 +- base-convert/crates/base-convert/src/main.rs | 194 +++- base-convert/crates/base-convert/src/serve.rs | 937 ++++++++++++++++++ .../crates/base-convert/src/superfluid.rs | 908 +++++++++++++++++ .../base-convert/tests/serve_launcher.rs | 609 ++++++++++++ base-convert/crates/base-quant/src/profile.rs | 39 + .../crates/base-readers/src/gguf/dequant.rs | 80 ++ bindings/node/package.json | 2 +- bindings/python/baseRT/__init__.py | 2 +- bindings/python/setup.py | 2 +- bindings/rust/baseRT-sys/Cargo.toml | 2 +- bindings/rust/baseRT-sys/src/lib.rs | 5 + bindings/rust/baseRT/Cargo.toml | 2 +- .../swift/Sources/CBaseRT/include/baseRT.h | 34 +- include/baseRT/baseRT.h | 34 +- 18 files changed, 2844 insertions(+), 44 deletions(-) create mode 100644 base-convert/crates/base-convert/src/serve.rs create mode 100644 base-convert/crates/base-convert/src/superfluid.rs create mode 100644 base-convert/crates/base-convert/tests/serve_launcher.rs diff --git a/base-convert/Cargo.lock b/base-convert/Cargo.lock index 0bd9ad9..812f821 100644 --- a/base-convert/Cargo.lock +++ b/base-convert/Cargo.lock @@ -151,7 +151,7 @@ dependencies = [ [[package]] name = "base-arch" -version = "0.3.0" +version = "0.4.0" dependencies = [ "anyhow", "base-format", @@ -162,7 +162,7 @@ dependencies = [ [[package]] name = "base-awq" -version = "0.3.0" +version = "0.4.0" dependencies = [ "anyhow", "base-format", @@ -176,7 +176,7 @@ dependencies = [ [[package]] name = "base-convert" -version = "0.3.0" +version = "0.4.0" dependencies = [ "anyhow", "base-arch", @@ -187,6 +187,7 @@ dependencies = [ "base-readers", "base-sign", "clap", + "dirs 5.0.1", "ed25519-dalek", "half", "indicatif", @@ -195,11 +196,12 @@ dependencies = [ "serde_json", "sha2 0.10.9", "tempfile", + "ureq", ] [[package]] name = "base-format" -version = "0.3.0" +version = "0.4.0" dependencies = [ "anyhow", "bitflags", @@ -215,7 +217,7 @@ dependencies = [ [[package]] name = "base-hub" -version = "0.3.0" +version = "0.4.0" dependencies = [ "anyhow", "base-format", @@ -234,7 +236,7 @@ dependencies = [ [[package]] name = "base-quant" -version = "0.3.0" +version = "0.4.0" dependencies = [ "anyhow", "base-format", @@ -246,7 +248,7 @@ dependencies = [ [[package]] name = "base-readers" -version = "0.3.0" +version = "0.4.0" dependencies = [ "anyhow", "base-format", @@ -260,7 +262,7 @@ dependencies = [ [[package]] name = "base-sign" -version = "0.3.0" +version = "0.4.0" dependencies = [ "anyhow", "base-format", diff --git a/base-convert/Cargo.toml b/base-convert/Cargo.toml index c3b3d93..3d75485 100644 --- a/base-convert/Cargo.toml +++ b/base-convert/Cargo.toml @@ -12,7 +12,7 @@ members = [ ] [workspace.package] -version = "0.3.0" +version = "0.4.0" edition = "2021" license = "Apache-2.0" repository = "https://github.com/basecompute/baseRT" diff --git a/base-convert/crates/base-convert/Cargo.toml b/base-convert/crates/base-convert/Cargo.toml index 269e869..2a6a0f9 100644 --- a/base-convert/crates/base-convert/Cargo.toml +++ b/base-convert/crates/base-convert/Cargo.toml @@ -30,6 +30,9 @@ base-hub.workspace = true memmap2.workspace = true sha2.workspace = true tempfile.workspace = true +dirs.workspace = true +# `basert superfluid install` fetches superfluid's release tarball from GitHub. +ureq = "3" [dev-dependencies] tempfile.workspace = true diff --git a/base-convert/crates/base-convert/src/hub.rs b/base-convert/crates/base-convert/src/hub.rs index 69b5420..f94e52e 100644 --- a/base-convert/crates/base-convert/src/hub.rs +++ b/base-convert/crates/base-convert/src/hub.rs @@ -480,7 +480,8 @@ fn bundled_computearena_harness( /// Forward `basert [args…]` to the matching runtime binary. Searches for /// `basert-` (release layout) then `baseRT_` (local dev build), -/// looking next to this executable first and then on `PATH`. On success the +/// looking next to this executable first and then on `PATH`. `basert serve` +/// has no runtime binary: it runs superfluid (`crate::serve`). On success the /// current process is replaced (`exec`), so this only returns on failure. pub fn dispatch_external(argv: Vec) -> Result<()> { use std::os::unix::process::CommandExt; @@ -501,6 +502,9 @@ pub fn dispatch_external(argv: Vec) -> Result<()> { // hub-id resolution. let (variant_flag, rest) = extract_variant_flag(rest)?; let rest = resolve_model_args(&rest, variant_flag.as_deref())?; + if cmd == "serve" { + return crate::serve::run(&rest); + } (vec![format!("basert-{cmd}"), format!("baseRT_{cmd}")], rest) }; @@ -559,14 +563,7 @@ pub fn dispatch_external(argv: Vec) -> Result<()> { /// Commands served by the BaseRT runtime rather than this binary. Used only /// to shape the not-found error above; dispatch itself is name-driven, so /// commands absent from this list (or from `basert --help`) still dispatch. -const RUNTIME_COMMANDS: [&str; 6] = [ - "serve", - "chat", - "complete", - "bench", - "transcribe", - "profile", -]; +const RUNTIME_COMMANDS: [&str; 5] = ["chat", "complete", "bench", "transcribe", "profile"]; pub fn cmd_pull(args: PullArgs) -> Result<()> { let reg = MergedRegistry::load()?; diff --git a/base-convert/crates/base-convert/src/main.rs b/base-convert/crates/base-convert/src/main.rs index 91fdcd6..d506915 100644 --- a/base-convert/crates/base-convert/src/main.rs +++ b/base-convert/crates/base-convert/src/main.rs @@ -5,6 +5,8 @@ use std::path::PathBuf; mod gpt_oss; mod hub; mod lora; +mod serve; +mod superfluid; /// Hand-written top-level help: the runtime commands are dispatched via /// clap's external-subcommand catch-all (invisible to clap's auto listing), @@ -15,7 +17,8 @@ const HELP_TEMPLATE: &str = "\ {usage-heading} {usage} Run models: - serve Start an OpenAI-compatible HTTP server + serve Start an OpenAI-compatible HTTP server (superfluid, installed on first use) + superfluid Install or check superfluid, the server `basert serve` runs chat Chat with a model interactively complete Generate a one-shot completion bench Measure throughput @@ -75,6 +78,8 @@ enum Cmd { CatalogScan(CatalogScanArgs), /// Write the manifest a Hub-split `.base` needs beside its parts. CatalogManifest(CatalogManifestArgs), + /// Install or check superfluid, the server `basert serve` runs. + Superfluid(superfluid::SuperfluidArgs), /// Runtime commands — `serve`, `chat`, `complete`, `bench`, … — handled /// by the BaseRT runtime (dispatched in `hub::dispatch_external`). #[command(external_subcommand)] @@ -161,12 +166,13 @@ struct ConvertArgs { /// halves it, q4 quarters it, at a small cost in acceptance. #[arg(long)] quantize_drafter: bool, - /// GGUF sources only: copy Q4_K / Q5_K / Q6_K super-blocks into the - /// bundle VERBATIM instead of dequantizing and re-packing them. + /// GGUF sources only: copy Q3_K / Q4_K / Q5_K / Q6_K / IQ4_XS and IQ2_XS / + /// IQ2_S / IQ3_XXS / IQ3_S super-blocks into the bundle VERBATIM instead + /// of dequantizing and re-packing them. /// /// This is lossless — the exact block bytes land in the `.base` /// weights blob, tagged with `layout = gguf_super`, `group_size = - /// 256` and the source `ggml_type` (12/13/14) so the runtime can + /// 256` and the source `ggml_type` (11/12/13/14/23, 17/18/21/22) so the runtime can /// dispatch a native k-quant kernel. It is therefore NOT /// "quant-from-quant" and does not need `--allow-quant-from-quant`; /// the canonical-quant spec forbids dequant→requant precisely @@ -418,7 +424,7 @@ struct ListArgs { /// White→lime vertical gradient anchored on brand Lime #E8FFBD (the brand /// tone alone is too close to white to read as terminal text). Printed only /// on an interactive terminal; NO_COLOR drops the colors. -fn print_banner() { +pub(crate) fn print_banner() { use std::io::IsTerminal; if !std::io::stdout().is_terminal() { return; @@ -485,6 +491,7 @@ fn main() -> Result<()> { Cmd::List(a) => hub::cmd_list(a), Cmd::CatalogScan(a) => hub::cmd_catalog_scan(a.org, a.out, a.dry_run), Cmd::CatalogManifest(a) => hub::cmd_catalog_manifest(a.bundle), + Cmd::Superfluid(a) => superfluid::cmd(a), Cmd::External(argv) => hub::dispatch_external(argv), } } @@ -668,14 +675,38 @@ fn convert_gguf( // are the layouts `Layout::GgufSuper` + `source_ggml_type` describe // and the ones the runtime knows how to dispatch natively. let kquant_passthrough = ctx.kquant_passthrough; - let is_kquant = |t: GgmlType| matches!(t, GgmlType::Q4K | GgmlType::Q5K | GgmlType::Q6K); - let has_kquant = gguf.tensors.iter().any(|t| is_kquant(t.ggml_type)); + let has_kquant = gguf.tensors.iter().any(kquant_passes_through); if kquant_passthrough && !has_kquant { eprintln!( - " warning: --kquant-passthrough set but no Q4_K/Q5_K/Q6_K tensors \ - in this GGUF; every tensor takes the normal convert path" + " warning: --kquant-passthrough set but no Q3_K/Q4_K/Q5_K/Q6_K/IQ4_XS/IQ2_XS/\ + IQ2_S/IQ3_XXS/IQ3_S tensors outside the MoE expert stacks in this GGUF; every \ + tensor takes the normal convert path" ); } + let kquant_experts = gguf + .tensors + .iter() + .filter(|t| is_passthrough_kquant(t.ggml_type) && is_gguf_expert_stack(&t.name)) + .count(); + if kquant_passthrough && kquant_experts > 0 { + eprintln!( + " kquant: {} MoE expert tensors take the normal convert path \ + (the expert GEMM has no k-quant kernels)", + kquant_experts + ); + } + if kquant_passthrough { + if let Some(t) = unconvertible_expert_stack(&gguf.tensors) { + bail!( + "MoE expert tensor `{}` is {}: --kquant-passthrough leaves expert stacks \ + to the normal convert path (the expert GEMM has no k-quant kernels), \ + and that path cannot dequantize {}", + t.name, + ggml_type_name(t.ggml_type), + ggml_type_name(t.ggml_type) + ); + } + } // Per CANONICAL_QUANT_SPEC.md: profile-driven canonical-quant // requires fp16/bf16/fp32 source. A GGUF with quantized weight @@ -688,10 +719,11 @@ fn convert_gguf( // Tensors headed for k-quant PASSTHROUGH are exempt: they are never // dequantized, so there is no compounded error to acknowledge. The // guard still fires for anything else quantized in the same file - // (a stray Q4_0 / Q8_0 tensor does get dequant→requant). + // (a stray Q4_0 / Q8_0 tensor, or a k-quant MoE expert stack, does get + // dequant→requant). if ctx.profile.is_some() && !ctx.allow_quant_from_quant { for tensor in gguf.tensors.iter() { - if kquant_passthrough && is_kquant(tensor.ggml_type) { + if kquant_passthrough && kquant_passes_through(tensor) { continue; } if !matches!( @@ -860,7 +892,7 @@ fn convert_gguf( // so a k-quant embedding yields a head at the SAME precision as the // embedding — the profile's head rule never runs. Say so. let copy = match src_idx { - Some(idx) if kquant_passthrough && is_kquant(gguf.tensors[idx].ggml_type) => { + Some(idx) if kquant_passthrough && kquant_passes_through(&gguf.tensors[idx]) => { TiedHeadCopy::Verbatim { ggml_type: ggml_type_name(gguf.tensors[idx].ggml_type), } @@ -908,7 +940,7 @@ fn convert_gguf( // INSIDE each 256-element block, so there are no separate // scale/bias regions — `scale_offset` / `bias_offset` stay None // and the runtime keys off `source_ggml_type` + `Layout::GgufSuper`. - if kquant_passthrough && is_kquant(info.ggml_type) { + if kquant_passthrough && kquant_passes_through(info) { let (entry, data) = kquant_passthrough_entry(info, bytes, canonical); passthrough_tensors += 1; writer.add_tensor(TensorPayload { entry, data }); @@ -1264,7 +1296,7 @@ fn convert_gguf( || canonical.contains("patch_embed") || canonical.ends_with(".bias"); - if !must_stay_raw && kquant_passthrough && is_kquant(info.ggml_type) { + if !must_stay_raw && kquant_passthrough && kquant_passes_through(info) { let (entry, data) = kquant_passthrough_entry(info, raw, canonical); mm_passthrough += 1; mm_kept += 1; @@ -1501,6 +1533,52 @@ fn unpermute_rope_rows( Ok(out) } +/// The GGUF super-block types `--kquant-passthrough` can copy verbatim. +fn is_passthrough_kquant(t: base_readers::gguf::GgmlType) -> bool { + use base_readers::gguf::GgmlType; + matches!( + t, + GgmlType::Q3K + | GgmlType::Q4K + | GgmlType::Q5K + | GgmlType::Q6K + | GgmlType::Iq4Xs + | GgmlType::Iq2Xs + | GgmlType::Iq2S + | GgmlType::Iq3Xxs + | GgmlType::Iq3S + ) +} + +/// A GGUF MoE expert stack (`ffn_{gate,up,down,gate_up}_exps`). +fn is_gguf_expert_stack(name: &str) -> bool { + name.contains("_exps") +} + +/// The first expert stack in a type `--kquant-passthrough` copies but +/// `dequant_to_f32` cannot read (the grid i-quants). Expert stacks never pass +/// through, so such a tensor has no way into the bundle. +fn unconvertible_expert_stack( + tensors: &[base_readers::gguf::TensorInfo], +) -> Option<&base_readers::gguf::TensorInfo> { + use base_readers::gguf::GgmlType; + tensors.iter().find(|t| { + is_gguf_expert_stack(&t.name) + && matches!( + t.ggml_type, + GgmlType::Iq2Xs | GgmlType::Iq2S | GgmlType::Iq3Xxs | GgmlType::Iq3S + ) + }) +} + +/// Whether `--kquant-passthrough` copies this tensor verbatim. Expert +/// stacks never do: the MoE expert GEMM has no kernel for any GGUF +/// super-block dtype, so a copied stack would convert and then abort at +/// the first expert dispatch. They take the normal convert path instead. +fn kquant_passes_through(t: &base_readers::gguf::TensorInfo) -> bool { + is_passthrough_kquant(t.ggml_type) && !is_gguf_expert_stack(&t.name) +} + /// Build the `.base` entry for a GGUF k-quant tensor copied through /// verbatim. The payload is the source super-block bytes unchanged. /// @@ -1520,10 +1598,20 @@ fn kquant_passthrough_entry( use base_readers::gguf::GgmlType; let (dtype, ggml_code) = match info.ggml_type { + // 3.4375 bits per element: base_q3 sizes it from below. + GgmlType::Q3K => (TensorDtype::BaseQ3, 11u32), GgmlType::Q4K => (TensorDtype::BaseQ4, 12u32), GgmlType::Q5K => (TensorDtype::BaseQ5, 13u32), GgmlType::Q6K => (TensorDtype::BaseQ6, 14u32), - // Unreachable: the caller gates on the same three types. + // 4.25 bits per element: base_q4 sizes it from below, as for Q4_K. + GgmlType::Iq4Xs => (TensorDtype::BaseQ4, 23u32), + // The grid i-quants (2.31 / 2.56 / 3.06 / 3.44 bits per element): + // base_q2 / base_q3 size them from below, as base_q4 does Q4_K. + GgmlType::Iq2Xs => (TensorDtype::BaseQ2, 17u32), + GgmlType::Iq2S => (TensorDtype::BaseQ2, 22u32), + GgmlType::Iq3Xxs => (TensorDtype::BaseQ3, 18u32), + GgmlType::Iq3S => (TensorDtype::BaseQ3, 21u32), + // Unreachable: the caller gates on the same types. other => unreachable!("kquant passthrough called with {other:?}"), }; // Same role-based routing the re-quantizing path uses: the @@ -6252,7 +6340,8 @@ struct QuantContext { /// vocabulary (`mtp.draft_rank`), and `--mtp-draft-vocab`, its default K. mtp_draft_ranking: Option>, mtp_draft_vocab: Option, - /// Copy GGUF Q4_K/Q5_K/Q6_K super-blocks through verbatim. + /// Copy GGUF Q3_K/Q4_K/Q5_K/Q6_K/IQ4_XS/IQ2_XS/IQ2_S/IQ3_XXS/IQ3_S super-blocks + /// through verbatim. kquant_passthrough: bool, /// Materialize `lm_head.weight` for tied models (`plan_tied_lm_head`); /// false under `--no-synth-tied-head`. @@ -7592,7 +7681,10 @@ mod rope_permute_tests { #[cfg(test)] mod gguf_passthrough_tests { - use super::{kquant_passthrough_entry, rope_permute_rows, unpermute_rope_rows}; + use super::{ + kquant_passes_through, kquant_passthrough_entry, rope_permute_rows, + unconvertible_expert_stack, unpermute_rope_rows, + }; use base_readers::gguf::{GgmlType, TensorInfo}; /// GGUF dimension order: ne[0] is the fastest-varying (in-features) @@ -7706,9 +7798,15 @@ mod gguf_passthrough_tests { fn passthrough_entry_describes_gguf_super_blocks() { use base_format::{ComputeRegion, Layout, TensorDtype}; for (ty, want_dtype, want_code) in [ + (GgmlType::Q3K, TensorDtype::BaseQ3, 11u32), (GgmlType::Q4K, TensorDtype::BaseQ4, 12u32), (GgmlType::Q5K, TensorDtype::BaseQ5, 13u32), (GgmlType::Q6K, TensorDtype::BaseQ6, 14u32), + (GgmlType::Iq4Xs, TensorDtype::BaseQ4, 23u32), + (GgmlType::Iq2Xs, TensorDtype::BaseQ2, 17u32), + (GgmlType::Iq2S, TensorDtype::BaseQ2, 22u32), + (GgmlType::Iq3Xxs, TensorDtype::BaseQ3, 18u32), + (GgmlType::Iq3S, TensorDtype::BaseQ3, 21u32), ] { let bytes: Vec = (0..64u8).collect(); let (entry, data) = kquant_passthrough_entry( @@ -7731,6 +7829,68 @@ mod gguf_passthrough_tests { } } + /// MoE expert stacks stay off the passthrough path whatever their type + /// (the expert GEMM has no k-quant kernels); dense and shared-expert + /// weights of the same types still pass through. + #[test] + fn expert_stacks_are_not_passed_through() { + for ty in [ + GgmlType::Q3K, + GgmlType::Q4K, + GgmlType::Q5K, + GgmlType::Q6K, + GgmlType::Iq4Xs, + GgmlType::Iq2Xs, + GgmlType::Iq2S, + GgmlType::Iq3Xxs, + GgmlType::Iq3S, + ] { + for (name, want) in [ + ("blk.0.attn_q.weight", true), + ("blk.0.ffn_gate_shexp.weight", true), + ("blk.0.ffn_gate_exps.weight", false), + ("blk.0.ffn_up_exps.weight", false), + ("blk.0.ffn_down_exps.weight", false), + ("blk.0.ffn_gate_up_exps.weight", false), + ] { + let t = TensorInfo { + name: name.into(), + ..info(256, 4, ty) + }; + assert_eq!(kquant_passes_through(&t), want, "{ty:?} {name}"); + } + } + assert!(!kquant_passes_through(&info(256, 4, GgmlType::F16))); + assert!(!kquant_passes_through(&info(256, 4, GgmlType::Q8_0))); + } + + /// A grid i-quant expert stack is refused up front: it cannot pass + /// through and cannot be dequantized. A dequantizable one (Q4_K) and a + /// grid i-quant dense weight are not. + #[test] + fn grid_iquant_expert_stacks_are_refused() { + let named = |name: &str, ty: GgmlType| TensorInfo { + name: name.into(), + ..info(256, 4, ty) + }; + let ok = vec![ + named("blk.0.attn_q.weight", GgmlType::Iq2Xs), + named("blk.0.ffn_gate_exps.weight", GgmlType::Q4K), + ]; + assert!(unconvertible_expert_stack(&ok).is_none()); + for ty in [ + GgmlType::Iq2Xs, + GgmlType::Iq2S, + GgmlType::Iq3Xxs, + GgmlType::Iq3S, + ] { + let mut tensors = ok.clone(); + tensors.push(named("blk.0.ffn_down_exps.weight", ty)); + let found = unconvertible_expert_stack(&tensors).map(|t| t.name.as_str()); + assert_eq!(found, Some("blk.0.ffn_down_exps.weight"), "{ty:?}"); + } + } + /// embed/lm_head keep the hot GPU-region routing the re-quantizing /// path gives them. #[test] diff --git a/base-convert/crates/base-convert/src/serve.rs b/base-convert/crates/base-convert/src/serve.rs new file mode 100644 index 0000000..f93342c --- /dev/null +++ b/base-convert/crates/base-convert/src/serve.rs @@ -0,0 +1,937 @@ +//! `basert serve`: the command line basert-serve has always had (the C++ +//! server's, flag for flag), run on superfluid. The launcher reads the +//! command line, finds the engine library and the pinned superfluid release +//! beside itself (installing it if asked, see [`crate::superfluid`]), and +//! execs `superfluid serve` with baseRT's models only, nothing fetched, and a +//! request log that starts fresh with each start: the server basert-serve has +//! always been. + +use std::path::{Path, PathBuf}; +use std::process::Command; + +use anyhow::{bail, Context, Result}; + +use crate::superfluid; + +pub const USAGE: &str = "\ +Usage: basert serve [--model ...] [options] + + a .base file path, or a hub id (e.g. basecompute/Qwen3-4B) that + `basert serve` pulls/converts on first use. The first non-flag + argument is treated as a model (same as `basert chat`). + + The server is superfluid (https://github.com/basecompute/superfluid). basert + runs the one installed beside it or in its own data directory, installing + the release it was tested with when there is none, or when the one found + cannot serve this engine: asked first on a terminal, without asking + otherwise, never with BASERT_SUPERFLUID_AUTO_INSTALL=0. + `basert superfluid install` does it ahead of time, and + `basert superfluid status` shows what is installed and whether it can serve. + +Options: + --model Additional .base model or hub id. Repeatable (multi-model serving). + --verbose, -v Show engine diagnostics (load, GPU, dispatch table) + --host Bind address (default: 127.0.0.1) + --port Bind port (default: 8453) + --max-context Max context window in tokens. Default: auto-sized for this + machine — the GPU memory budget less the weights and + scratch headroom, over what a token of KV costs across the + configured lanes, capped only at the model's trained window. + Determines KV cache allocation up front; requests past it + are rejected with context_length_exceeded. + --max-tokens Cap on generated tokens for a request that names none. + Default: fill the rest of the context window. + --rate-limit Requests per minute per client (0 = unlimited) + --api-key Require Bearer token authentication + --sessions Request-log directory (default: per-port under the state directory). + The request log is rotated at startup; Local owns chat history. + --idle-timeout Auto-unload idle models after N seconds (0 = disabled) + --model-dir Autoload: register every *.base in as a known-but- + unloaded model, loaded into VRAM on first request (and freed + again by --idle-timeout). Pairs well with --idle-timeout. + --request-timeout Accepted; it bounds FIM completions, not chat completions, + which this server runs to their token limit. + --drain-timeout Seconds to wait for in-flight requests on SIGUSR1 (default: 60) + SIGUSR1 stops accepting connections and drains; pair with an + external supervisor for rolling restarts. SIGINT/SIGTERM still + exit immediately without draining. + --files-max-bytes Reject uploads that would push total stored bytes above N + --files-expiry Auto-remove files older than N seconds (background sweep) + --files-sweep Sweep cadence in seconds (default 300, when --files-expiry set) + --log-file Redirect stderr (access log + diagnostics) to this file. + Use external rotation: `logrotate copytruncate` (Linux) or + `newsyslog` with the F flag (macOS). Default: stderr. + --kv-bits 4|8|16|84 KV cache element width (84 = mixed K=Q8/V=Q4) (default: per-model auto) + --max-batch-size N Max concurrent sequences (default 1) + --max-batch N Alias for --max-batch-size (used by Local) + --continuous-batching [N] Serve up to N requests at once through one shared + forward pass (default 8). Set N >= your peak concurrent + requests, or the surplus queues (higher TTFT, no extra + throughput). + --prefill-budget N Max prompt tokens ingested per engine tick (64-4096). Lower + it to trade first-token latency for smoother concurrent decode. + --basert-lib The directory holding libbaseRT to serve with (default: beside + basert, or ../lib, or $BASERT_LIB) + --print-command Print the superfluid command line this start would run, and exit + --version Print the version and exit + +Any other --flag is handed to `superfluid serve` as it is, with the word after +it as its value unless that word is a model (`superfluid serve --help` lists +them); everything after `--` is handed over untouched. + +Accepted so that existing command lines keep working, and no longer used: + --paged-kv, --prefix-cache Paged KV and prefix reuse are always on. + --prefix-cache-file , --prefix-cache-save-interval + The prefix cache is not kept on disk. + --files-dir Uploaded files are kept in the server's state directory. + --media-dir Images and audio arrive as data: URIs or uploads; no local + path is read. + --metallib , --prefill-chunk N, --paged-weights, --no-paged-weights-retry, + --gpu-wait-timeout-ms N, --decode-replay 0|1|2, --no-baked-decode + Engine settings this server does not expose. + +State (the request log, uploaded files) is kept under $BASERT_STATE_DIR, by +default the serve/ directory beside the model cache, one directory per port. +The engine library is found beside basert (or in ../lib); BASERT_LIB names +another directory. BASERT_SUPERFLUID_DIR names where superfluid is installed; +BASERT_SUPERFLUID_SOURCE a directory holding its release tarball, read instead +of GitHub; BASERT_SUPERFLUID_AUTO_INSTALL=0 forbids a download at a start. +"; + +/// Flags every start hands superfluid; one given on the command line too is +/// not handed over twice. +const ALWAYS: [&str; 5] = [ + "--offline", + "--no-park", + "--sessions-cache", + "--no-tui", + "--no-http-qos-header", +]; + +/// How deep `--model-dir` is looked into for a model to start with; a hub +/// cache nests one three levels down (`///model.base`). +const MODEL_DIR_DEPTH: usize = 8; + +/// What a command line asks for. +#[derive(Debug, Default, PartialEq)] +pub struct Options { + pub models: Vec, + pub model_dir: Option, + pub host: Option, + pub port: Option, + /// `None` sizes the window for this device. + pub max_context: Option, + /// `None` fills the rest of the window. + pub max_tokens: Option, + pub rate_limit: u32, + pub api_key: Option, + pub sessions: Option, + pub idle_timeout: u64, + pub request_timeout_ms: u64, + pub drain_timeout: Option, + pub files_max_bytes: Option, + pub files_expiry: Option, + pub files_sweep: Option, + pub log_file: Option, + pub kv_bits: Option, + pub lanes: u32, + pub prefill_budget: Option, + pub verbose: bool, + /// `--basert-lib`: the engine library directory, superfluid's own flag, + /// in place of the one found beside basert. + pub basert_lib: Option, + /// superfluid's own flags, handed over after the translated ones. + pub passthrough: Vec, + /// Print the superfluid command line and exit. + pub print_command: bool, + /// Said on stderr before serving: flags that are no longer used, and + /// values read as something else. + pub notes: Vec, +} + +#[derive(Debug, PartialEq)] +pub enum Parsed { + Serve(Box), + Help, + Version, +} + +fn number(flag: &str, raw: &str) -> Result { + raw.trim() + .parse() + .map_err(|_| format!("Error: {flag} expects a number, got {raw:?}")) +} + +/// Reads a command line (after `serve`) the way the C++ server did. +pub fn parse(args: &[String]) -> Result { + let mut o = Options::default(); + // The C++ server's two batch settings: `--max-batch-size` set the lanes + // the load installs, `--continuous-batching [N]` (default 8) set them + // too, and the larger of the two won. + let mut max_batch_size: u32 = 1; + let mut continuous: Option = None; + let mut it = args.iter().peekable(); + while let Some(a) = it.next() { + let flag = a.as_str(); + // A flag that takes a value and has none was an unknown option. + let value = + |it: &mut std::iter::Peekable>| -> Result { + it.next() + .cloned() + .ok_or_else(|| format!("Unknown option: {flag} (see `basert serve --help`)")) + }; + match flag { + "--help" | "-h" => return Ok(Parsed::Help), + "--version" | "-V" => return Ok(Parsed::Version), + "--model" => o.models.push(value(&mut it)?), + "--model-dir" => o.model_dir = Some(value(&mut it)?), + "--host" => o.host = Some(value(&mut it)?), + "--port" => o.port = Some(number(flag, &value(&mut it)?)?), + "--max-context" => { + let v = value(&mut it)?; + let n: i64 = if v == "auto" { 0 } else { number(flag, &v)? }; + if n > 0 { + o.max_context = Some(n.min(u32::MAX as i64) as u32); + } else { + if v != "auto" { + o.notes.push(format!("WARNING: --max-context {v} is not a positive token count; auto-sizing instead")); + } + o.max_context = None; + } + } + "--max-tokens" => { + let n: i64 = number(flag, &value(&mut it)?)?; + o.max_tokens = (n > 0).then(|| n.min(u32::MAX as i64) as u32); + } + "--rate-limit" => { + o.rate_limit = + number::(flag, &value(&mut it)?)?.clamp(0, u32::MAX as i64) as u32 + } + "--api-key" => o.api_key = Some(value(&mut it)?), + "--sessions" => o.sessions = Some(value(&mut it)?), + "--idle-timeout" => { + o.idle_timeout = number::(flag, &value(&mut it)?)?.max(0) as u64 + } + "--request-timeout" => { + o.request_timeout_ms = number::(flag, &value(&mut it)?)?.max(0) as u64; + // superfluid's completion deadline bounds its FIM path; a + // chat completion has no deadline, and the C++ server's did. + if o.request_timeout_ms > 0 { + o.notes.push(format!( + "basert serve: {flag} bounds FIM completions only: a chat completion runs to its token limit" + )); + } + } + "--drain-timeout" => { + o.drain_timeout = Some(number::(flag, &value(&mut it)?)?.max(0) as u64) + } + "--files-max-bytes" => { + o.files_max_bytes = Some(number::(flag, &value(&mut it)?)?.max(0) as u64) + } + "--files-expiry" => { + o.files_expiry = Some(number::(flag, &value(&mut it)?)?.max(0) as u64) + } + "--files-sweep" => { + o.files_sweep = Some(number::(flag, &value(&mut it)?)?.max(0) as u64) + } + "--log-file" => o.log_file = Some(value(&mut it)?), + "--kv-bits" => { + let bits: i64 = number(flag, &value(&mut it)?)?; + if ![4, 8, 16, 84].contains(&bits) { + return Err(format!("Error: --kv-bits must be 4, 8, 16, or 84 (84 = mixed K=Q8/V=Q4) (got {bits})")); + } + o.kv_bits = Some(bits as u32); + } + "--max-batch-size" | "--max-batch" => { + let b: i64 = number(flag, &value(&mut it)?)?; + if b < 1 { + return Err(format!("Error: {flag} must be >= 1 (got {b})")); + } + max_batch_size = b.min(u32::MAX as i64) as u32; + } + "--continuous-batching" => { + // The width is optional: taken when the next word starts with a digit. + let mut lanes = continuous.unwrap_or(8); + if let Some(next) = it + .peek() + .filter(|n| n.starts_with(|c: char| c.is_ascii_digit())) + { + let b: i64 = number(flag, next)?; + it.next(); + if b >= 1 { + lanes = b.min(u32::MAX as i64) as u32; + } + } + continuous = Some(lanes); + max_batch_size = lanes; + } + "--prefill-budget" => { + let v: i64 = number(flag, &value(&mut it)?)?; + if v > 0 { + let kept = v.clamp(64, 4096) as u32; + if kept as i64 != v { + o.notes.push(format!( + "basert serve: --prefill-budget {v} is outside 64-4096; using {kept}" + )); + } + o.prefill_budget = Some(kept); + } + } + "--verbose" | "-v" => o.verbose = true, + "--paged-kv" | "--prefix-cache" => o.notes.push(format!( + "basert serve: {flag} is no longer used: paged KV and prefix reuse are always on" + )), + "--prefix-cache-file" | "--prefix-cache-save-interval" => { + value(&mut it)?; + o.notes.push(format!( + "basert serve: {flag} is no longer used: the prefix cache is not kept on disk" + )); + } + "--files-dir" => { + value(&mut it)?; + o.notes.push(format!("basert serve: {flag} is no longer used: uploaded files are kept in the server's state directory")); + } + "--media-dir" => { + value(&mut it)?; + o.notes.push(format!( + "basert serve: {flag} is no longer used: images and audio arrive as data: URIs or uploads, and no local path is read" + )); + } + "--metallib" | "--prefill-chunk" | "--gpu-wait-timeout-ms" | "--decode-replay" => { + value(&mut it)?; + o.notes.push(format!("basert serve: {flag} is no longer used: an engine setting this server does not expose")); + } + "--paged-weights" | "--no-paged-weights-retry" | "--no-baked-decode" => { + o.notes.push(format!("basert serve: {flag} is no longer used: an engine setting this server does not expose")); + } + "--print-command" => o.print_command = true, + "--basert-lib" => o.basert_lib = Some(value(&mut it)?), + "--runtime" => { + return Err("Error: --runtime is not accepted: basert serve always serves through superfluid's basert runtime".into()) + } + "--" => o.passthrough.extend(it.by_ref().cloned()), + other if !other.starts_with('-') => o.models.push(other.to_string()), + other if ALWAYS.contains(&other) => {} + other => { + // superfluid's own flag: handed over as it is, with the word + // after it as its value unless that word is a model. + o.passthrough.push(other.to_string()); + if let Some(next) = it + .peek() + .map(|n| n.as_str()) + .filter(|n| !n.starts_with('-') && !looks_like_model(n)) + { + o.passthrough.push(next.to_string()); + it.next(); + } + } + } + } + if o.models.is_empty() && o.model_dir.is_none() { + return Err("Error: at least one --model (or --model-dir) is required".into()); + } + o.lanes = max_batch_size.max(continuous.unwrap_or(1)); + Ok(Parsed::Serve(Box::new(o))) +} + +/// The `superfluid serve` command line for `o`, after `serve`. `sessions` is +/// this server's state directory. +pub fn superfluid_args(o: &Options, sessions: &Path) -> Vec { + let mut a: Vec = Vec::new(); + let mut push = |k: &str, v: String| { + a.push(k.to_string()); + a.push(v); + }; + for m in &o.models { + push("--model", m.clone()); + } + if let Some(d) = &o.model_dir { + push("--model-dir", d.clone()); + } + // baseRT's models only, never fetched, and a request log that starts + // fresh with each start: the server basert-serve has always been. + push("--runtime", "basert".into()); + push( + "--sessions", + o.sessions + .clone() + .unwrap_or_else(|| sessions.display().to_string()), + ); + push( + "--host", + o.host.clone().unwrap_or_else(|| "127.0.0.1".into()), + ); + push("--port", o.port.unwrap_or(8453).to_string()); + push("--max-batch", o.lanes.to_string()); + if let Some(n) = o.max_context { + push("--max-context", n.to_string()); + } + if let Some(n) = o.max_tokens { + push("--max-tokens", n.to_string()); + } + if o.rate_limit > 0 { + push("--rate-limit", o.rate_limit.to_string()); + } + if let Some(k) = &o.api_key { + push("--api-key", k.clone()); + } + if o.idle_timeout > 0 { + push("--idle-timeout", o.idle_timeout.to_string()); + } + if o.request_timeout_ms > 0 { + push("--completion-deadline-ms", o.request_timeout_ms.to_string()); + } + if let Some(n) = o.drain_timeout { + push("--drain-timeout", n.to_string()); + } + if let Some(n) = o.files_max_bytes { + push("--files-max-bytes", n.to_string()); + } + if let Some(n) = o.files_expiry { + push("--files-expiry", n.to_string()); + } + if let Some(n) = o.files_sweep { + push("--files-sweep", n.to_string()); + } + if let Some(p) = &o.log_file { + push("--log-file", p.clone()); + } + if let Some(b) = o.kv_bits { + push("--kv-bits", b.to_string()); + } + if let Some(b) = o.prefill_budget { + push("--prefill-budget", b.to_string()); + } + if let Some(dir) = &o.basert_lib { + push("--basert-lib", dir.clone()); + } + a.extend(ALWAYS.map(String::from)); + a.extend(o.passthrough.iter().cloned()); + a +} + +/// A word on the command line that is a model, not a flag's value: a `.base` +/// path, or a hub id (`org/name`, with a variant after `:` or not). +fn looks_like_model(word: &str) -> bool { + word.ends_with(".base") + || (!word.starts_with(['/', '.', '~']) && word.matches('/').count() == 1) +} + +/// `basert serve `, the args' hub ids already resolved to paths. Returns +/// only when superfluid could not be started: on success the process is +/// replaced (`exec`). +pub fn run(rest: &[String]) -> Result<()> { + let parsed = parse(rest); + if parsed == Ok(Parsed::Version) { + println!("basert serve {}", env!("CARGO_PKG_VERSION")); + match superfluid::Install::locate()? { + Some(i) => println!( + "superfluid {} ({})", + i.installed_release() + .unwrap_or_else(|| "of a release it does not say".to_string()), + i.binary().display() + ), + None => println!( + "superfluid {} pinned, not installed (`basert superfluid install`)", + superfluid::RELEASE + ), + } + return Ok(()); + } + crate::print_banner(); + if rest.is_empty() { + print!("{USAGE}"); + std::process::exit(1); + } + let mut o = match parsed { + Ok(Parsed::Serve(o)) => *o, + Ok(_) => { + print!("{USAGE}"); + return Ok(()); + } + Err(e) => fail(&e), + }; + for note in &o.notes { + eprintln!("{note}"); + } + for m in &o.models { + if !Path::new(m).exists() { + fail(&format!( + "Failed to load model {m}: no such file (a .base path, or a hub id that `basert serve` pulls on first use)" + )); + } + } + if o.models.is_empty() { + // superfluid starts with a model loaded; the C++ server could start + // with none and load the first on first request. + let dir = o.model_dir.clone().unwrap_or_default(); + let Some(first) = first_model_under(Path::new(&dir), MODEL_DIR_DEPTH) else { + fail(&format!("Error: --model-dir {dir} holds no .base model")); + }; + eprintln!( + "basert serve: --model-dir without --model: {} is loaded at startup, the others on first use", + first.display() + ); + o.models.push(first.display().to_string()); + } + let state = state_dir()?; + let sessions = o + .sessions + .as_ref() + .map(PathBuf::from) + .unwrap_or_else(|| state.join(format!("basert-serve-{}", o.port.unwrap_or(8453)))); + std::fs::create_dir_all(&sessions).with_context(|| { + format!( + "Error: could not create the state directory {}", + sessions.display() + ) + })?; + // The engine library: the user's `--basert-lib`, else the one found + // beside basert, which superfluid is told of the same way. + let engine = match &o.basert_lib { + Some(dir) => Some(PathBuf::from(dir)), + None => engine_dir()?, + }; + // superfluid keeps what it installs and its session socket under + // SUPERFLUID_HOME: here, beside this server's state, never in + // ~/.superfluid, so a superfluid of the user's own is never touched. + let home = state.join("superfluid"); + let mut tail: Vec = Vec::new(); + if let (Some(dir), None) = (&engine, &o.basert_lib) { + tail.push("--basert-lib".to_string()); + tail.push(dir.display().to_string()); + } + if o.print_command { + let binary = match superfluid::Install::locate()? { + Some(i) => i.binary().display().to_string(), + None => format!( + "superfluid (not installed; {} would be)", + superfluid::RELEASE + ), + }; + let mut words = vec![binary, "serve".to_string()]; + words.extend(superfluid_args(&o, &sessions)); + words.extend(tail); + println!( + "SUPERFLUID_HOME={} SUPERFLUID_NO_UPDATE_CHECK=1{} {}", + shell_word(&home.display().to_string()), + if o.verbose { + " SUPERFLUID_ENGINE_VERBOSE=1" + } else { + "" + }, + words + .iter() + .map(|w| shell_word(w)) + .collect::>() + .join(" ") + ); + return Ok(()); + } + let binary = superfluid::ensure(engine.as_deref(), &home, &mut |line| { + eprintln!("basert serve: {line}") + })?; + std::fs::create_dir_all(&home) + .with_context(|| format!("Error: could not create {}", home.display()))?; + + let mut command = Command::new(&binary); + command + .arg("serve") + .args(superfluid_args(&o, &sessions)) + .args(&tail); + // A superfluid of basert's is updated by basert, never by itself: no + // daily check, no notice in this server's output. + command + .env("SUPERFLUID_HOME", &home) + .env("SUPERFLUID_NO_UPDATE_CHECK", "1"); + if o.verbose { + command.env("SUPERFLUID_ENGINE_VERBOSE", "1"); + } + use std::os::unix::process::CommandExt; + let err = command.exec(); + Err(err).with_context(|| format!("launching {}", binary.display())) +} + +/// `word` as a shell would need it: quoted when it holds a space or a quote. +fn shell_word(word: &str) -> String { + if word.is_empty() || word.contains(|c: char| c.is_whitespace() || c == '\'' || c == '"') { + format!("'{}'", word.replace('\'', "'\\''")) + } else { + word.to_string() + } +} + +fn fail(message: &str) -> ! { + eprintln!("{message}"); + std::process::exit(1); +} + +/// `$BASERT_STATE_DIR`, else `serve/` beside the model cache, as `basert +/// serve` has always had it. +pub(crate) fn superfluid_home() -> Result { + Ok(state_dir()?.join("superfluid")) +} + +fn state_dir() -> Result { + if let Some(dir) = std::env::var_os("BASERT_STATE_DIR").filter(|v| !v.is_empty()) { + return Ok(PathBuf::from(dir)); + } + let models = base_hub::models_dir() + .context("no cache directory to keep the server's state in (set BASERT_STATE_DIR)")?; + Ok(models + .parent() + .map(Path::to_path_buf) + .unwrap_or(models) + .join("serve")) +} + +/// Where libbaseRT is, for superfluid's basert worker: `None` when `$BASERT_LIB` +/// names it (superfluid reads that itself), else beside this launcher (the +/// release bundle), in `../lib` (the SDK bundle), or in `build/` of the source +/// tree this launcher was built in. +pub(crate) fn engine_dir() -> Result> { + if std::env::var_os("BASERT_LIB").is_some_and(|v| !v.is_empty()) { + return Ok(None); + } + let exe = std::env::current_exe().context("this program's own path")?; + let dir = exe.parent().context("this program's own directory")?; + let mut candidates = vec![dir.to_path_buf(), dir.join("..").join("lib")]; + candidates.extend( + dir.ancestors() + .filter(|a| a.join("tools/base-convert/Cargo.toml").is_file()) + .map(|a| a.join("build")), + ); + for candidate in candidates { + if ["libbaseRT.dylib", "libbaseRT.so"] + .iter() + .any(|name| candidate.join(name).exists()) + { + return Ok(Some(std::fs::canonicalize(&candidate).unwrap_or(candidate))); + } + } + bail!( + "libbaseRT was not found beside {} or in its ../lib; set BASERT_LIB to the directory holding it", + exe.display() + ) +} + +/// The first `.base` under `dir` in name order, looking into directories. +fn first_model_under(dir: &Path, depth: usize) -> Option { + let mut entries: Vec = std::fs::read_dir(dir) + .ok()? + .flatten() + .map(|e| e.path()) + .collect(); + entries.sort(); + entries.into_iter().find_map(|p| { + if p.is_file() && p.extension().is_some_and(|e| e == "base") { + Some(p) + } else if depth > 0 && p.is_dir() { + first_model_under(&p, depth - 1) + } else { + None + } + }) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn line(args: &[&str]) -> Vec { + args.iter().map(|a| a.to_string()).collect() + } + + fn serve(args: &[&str]) -> Options { + match parse(&line(args)) { + Ok(Parsed::Serve(o)) => *o, + other => panic!("{args:?}: {other:?}"), + } + } + + #[test] + fn a_model_is_a_bare_word_or_a_model_flag_in_order() { + assert_eq!( + serve(&["a.base", "--model", "b.base", "c.base"]).models, + ["a.base", "b.base", "c.base"] + ); + assert_eq!( + serve(&["--model-dir", "/m"]).model_dir.as_deref(), + Some("/m") + ); + } + + #[test] + fn what_the_cpp_server_refused_is_refused_in_its_words() { + for (args, said) in [ + ( + &[][..], + "Error: at least one --model (or --model-dir) is required", + ), + ( + &["m.base", "--runtime", "llamacpp"], + "Error: --runtime is not accepted: basert serve always serves through superfluid's basert runtime", + ), + ( + &["m.base", "--port"], + "Unknown option: --port (see `basert serve --help`)", + ), + ( + &["m.base", "--kv-bits", "5"], + "Error: --kv-bits must be 4, 8, 16, or 84 (84 = mixed K=Q8/V=Q4) (got 5)", + ), + ( + &["m.base", "--max-batch-size", "0"], + "Error: --max-batch-size must be >= 1 (got 0)", + ), + ( + &["m.base", "--port", "nope"], + "Error: --port expects a number, got \"nope\"", + ), + ] { + assert_eq!(parse(&line(args)).unwrap_err(), said, "{args:?}"); + } + } + + #[test] + fn lanes_are_one_unless_batching_asks_for_more() { + assert_eq!(serve(&["m.base"]).lanes, 1); + assert_eq!(serve(&["m.base", "--continuous-batching"]).lanes, 8); + assert_eq!(serve(&["m.base", "--continuous-batching", "4"]).lanes, 4); + assert_eq!(serve(&["m.base", "--max-batch-size", "3"]).lanes, 3); + // The larger of the two settings, whichever came last. + assert_eq!( + serve(&[ + "m.base", + "--continuous-batching", + "4", + "--max-batch-size", + "2" + ]) + .lanes, + 4 + ); + assert_eq!( + serve(&[ + "m.base", + "--max-batch-size", + "16", + "--continuous-batching", + "4" + ]) + .lanes, + 4 + ); + // A model named after the flag is not taken for its width. + let o = serve(&["--continuous-batching", "m.base"]); + assert_eq!( + (o.lanes, o.models.as_slice()), + (8, &["m.base".to_string()][..]) + ); + } + + #[test] + fn auto_and_unset_values_leave_the_default() { + let o = serve(&[ + "m.base", + "--max-context", + "auto", + "--max-tokens", + "0", + "--prefill-budget", + "0", + ]); + assert_eq!( + (o.max_context, o.max_tokens, o.prefill_budget), + (None, None, None) + ); + assert!(o.notes.is_empty(), "{:?}", o.notes); + let o = serve(&["m.base", "--max-context", "-5"]); + assert_eq!(o.max_context, None); + assert_eq!( + o.notes, + ["WARNING: --max-context -5 is not a positive token count; auto-sizing instead"] + ); + } + + #[test] + fn local_launches_keep_the_session_path_and_batch_width() { + for lanes in ["1", "2", "4"] { + let o = serve(&[ + "/cache/model.base", + "--port", + "8453", + "--host", + "0.0.0.0", + "--api-key", + "local-test", + "--sessions", + "/app support/Local/engine", + "--max-batch", + lanes, + "--max-context", + "8192", + "--kv-bits", + "84", + ]); + assert_eq!(o.sessions.as_deref(), Some("/app support/Local/engine")); + assert_eq!(o.lanes, lanes.parse::().unwrap()); + let args = superfluid_args(&o, Path::new("/fallback")); + assert_eq!(args.iter().filter(|a| *a == "--sessions").count(), 1); + assert!(args + .windows(2) + .any(|a| a == ["--sessions", "/app support/Local/engine"])); + assert!(args.windows(2).any(|a| a == ["--max-batch", lanes])); + } + assert!(parse(&line(&["m.base", "--max-batch", "0"])).is_err()); + assert!(parse(&line(&["m.base", "--sessions"])).is_err()); + assert_eq!( + serve(&["m.base", "--continuous-batching", "4", "--max-batch", "2"]).lanes, + 4 + ); + } + + #[test] + fn retired_flags_are_taken_with_their_values_and_said_to_be_unused() { + let o = serve(&[ + "m.base", + "--paged-kv", + "--prefix-cache", + "--prefix-cache-file", + "/c", + "--media-dir", + "/m", + "--files-dir", + "/f", + "--metallib", + "x.metallib", + "--prefill-chunk", + "256", + "--decode-replay", + "1", + "--no-baked-decode", + ]); + assert_eq!(o.models, ["m.base"]); + assert_eq!(o.notes.len(), 9, "{:?}", o.notes); + assert!( + o.notes.iter().all(|n| n.contains(" is no longer used: ")), + "{:?}", + o.notes + ); + } + + #[test] + fn the_superfluid_command_line_serves_basert_models_only_and_keeps_nothing() { + let o = serve(&[ + "m.base", + "--port", + "9000", + "--api-key", + "k", + "--request-timeout", + "60000", + "--kv-bits", + "8", + ]); + let a = superfluid_args(&o, Path::new("/state/basert-serve-9000")); + let joined = a.join(" "); + for want in [ + "--model m.base", + "--runtime basert", + "--sessions /state/basert-serve-9000", + "--host 127.0.0.1", + "--port 9000", + "--max-batch 1", + "--api-key k", + "--completion-deadline-ms 60000", + "--kv-bits 8", + "--offline", + "--no-park", + "--sessions-cache", + "--no-tui", + ] { + assert!(joined.contains(want), "{want:?} in {joined}"); + } + assert!( + !joined.contains("--max-context"), + "auto sizing is superfluid's default: {joined}" + ); + } + + #[test] + fn superfluids_own_flags_are_handed_over_as_they_are() { + let o = serve(&[ + "m.base", + "--prefix-blob-budget-mb", + "512", + "--speculate", + "auto", + "--no-tui", + "--offline", + "--basert-lib", + "/engine", + "--", + "--anything", + "goes", + "--here", + ]); + assert_eq!(o.models, ["m.base"]); + assert_eq!(o.basert_lib.as_deref(), Some("/engine")); + assert_eq!( + o.passthrough, + [ + "--prefix-blob-budget-mb", + "512", + "--speculate", + "auto", + "--anything", + "goes", + "--here" + ] + ); + // A model after a flag that takes no value stays a model; a flag every + // start hands over anyway is not handed over twice. + let o = serve(&[ + "--no-http-qos-header", + "--tui", + "basecompute/Qwen3-4B:default-q8", + "x.base", + ]); + assert_eq!(o.models, ["basecompute/Qwen3-4B:default-q8", "x.base"]); + assert_eq!(o.passthrough, ["--tui"]); + assert!(looks_like_model("/models/m.base") && looks_like_model("org/name")); + assert!( + !looks_like_model("/tmp/log") && !looks_like_model("auto") && !looks_like_model("512") + ); + // Handed over last, after the flags every start sets; the user's + // engine directory is superfluid's flag, said once. + let o = serve(&["m.base", "--basert-lib", "/engine", "--speculate", "auto"]); + let joined = superfluid_args(&o, Path::new("/s")).join(" "); + assert!( + joined.ends_with("--no-http-qos-header --speculate auto"), + "{joined}" + ); + assert_eq!( + joined.matches("--basert-lib /engine").count(), + 1, + "{joined}" + ); + assert_eq!(shell_word("plain"), "plain"); + assert_eq!(shell_word("with space"), "'with space'"); + assert_eq!(shell_word("it's"), "'it'\\''s'"); + } + + #[test] + fn the_first_model_under_a_directory_is_found_through_a_hub_cache_layout() { + let tmp = tempfile::tempdir().unwrap(); + let deep = tmp.path().join("org").join("repo").join("variant"); + std::fs::create_dir_all(&deep).unwrap(); + std::fs::write(deep.join("model.base"), b"").unwrap(); + std::fs::write(tmp.path().join("zzz.txt"), b"").unwrap(); + assert_eq!( + first_model_under(tmp.path(), MODEL_DIR_DEPTH), + Some(deep.join("model.base")) + ); + assert_eq!(first_model_under(tmp.path(), 0), None); + } +} diff --git a/base-convert/crates/base-convert/src/superfluid.rs b/base-convert/crates/base-convert/src/superfluid.rs new file mode 100644 index 0000000..c1eecdb --- /dev/null +++ b/base-convert/crates/base-convert/src/superfluid.rs @@ -0,0 +1,908 @@ +//! superfluid, the server `basert serve` runs. +//! +//! basert ships no server of its own. `basert serve` runs a superfluid +//! release: the one the one-shot installer put beside basert, or one basert +//! installs for itself at a start. Which superfluid may serve this engine is +//! superfluid's own call: before a start the launcher asks each install it +//! finds (`superfluid runtimes --json`, the engine library named) whether its +//! basert runtime is ready, and runs the first that is. One that is not, and +//! is not the pinned release, is brought to the pin. The pin ([`RELEASE`]) is +//! the release this basert was tested with; a newer one the probe accepts +//! runs as it is, and nothing is ever downgraded. +//! +//! Where superfluid is looked for, in order: `$BASERT_SUPERFLUID_DIR`; +//! `superfluid/` beside basert, the one-shot installer's layout; a directory +//! of the user's own, where basert installs when the one beside it cannot be +//! written (a package manager's, an app bundle's). Each holds superfluid's +//! own layout (`bin/superfluid`, the runtime workers in `libexec/superfluid/`) +//! off PATH: a superfluid the user installed for themselves is neither +//! shadowed nor run in its place. +//! +//! A download at a start is asked for on a terminal and made without asking +//! otherwise, since an older Local calls `basert serve` without knowing +//! superfluid exists; `$BASERT_SUPERFLUID_AUTO_INSTALL=0` forbids it, and the +//! command to run is named instead. `basert superfluid install` installs +//! ahead of time; `$BASERT_SUPERFLUID_SOURCE` names a directory holding the +//! release's tarball and checksum, read instead of GitHub. + +use std::fmt; +use std::fs; +use std::io::{BufRead, IsTerminal, Write}; +use std::path::{Path, PathBuf}; +use std::process::{Command, Stdio}; +use std::time::Duration; + +use anyhow::{bail, Context, Result}; +use clap::{Args, Subcommand}; +use sha2::{Digest, Sha256}; + +/// The superfluid release this basert was tested with, a tag of +/// : installed when +/// none is found, or when the one found cannot serve this engine. +pub const RELEASE: &str = "v0.1.3"; + +const REPO: &str = "basecompute/superfluid"; +/// Where superfluid is installed, instead of the places looked in by default. +pub const DIR_ENV: &str = "BASERT_SUPERFLUID_DIR"; +/// A directory holding the release's tarball and its `.sha256`, read instead +/// of GitHub: an install on a machine without internet access, and the tests. +pub const SOURCE_ENV: &str = "BASERT_SUPERFLUID_SOURCE"; +/// `0` forbids `basert serve` any download; `1` lets it download without +/// asking, even on a terminal. +pub const AUTO_INSTALL_ENV: &str = "BASERT_SUPERFLUID_AUTO_INSTALL"; +/// Written into an install with the release it holds. +const STAMP: &str = ".release"; +/// The name superfluid's basert runtime answers to in `runtimes --json`. +const RUNTIME_ID: &str = "basert"; + +/// `basert superfluid `. +#[derive(Args, Debug)] +pub struct SuperfluidArgs { + #[command(subcommand)] + pub action: Action, +} + +#[derive(Subcommand, Debug)] +pub enum Action { + /// Install the pinned superfluid release where `basert serve` looks for + /// it (nothing to do when it is already there). + Install { + /// Install again even when the pinned release is already there. + #[arg(long)] + force: bool, + }, + /// Show the pinned release, what is installed where, and whether it can + /// serve the engine beside basert. Exits 0 when an installed superfluid + /// can, 1 when none is installed, 2 when the installed one cannot. + Status, +} + +pub fn cmd(args: SuperfluidArgs) -> Result<()> { + match args.action { + Action::Install { force } => { + let dest = match Install::locate()? { + Some(i) if !force && i.installed_release().as_deref() == Some(RELEASE) => { + println!("superfluid {RELEASE} is installed at {}", i.root.display()); + return Ok(()); + } + Some(i) if i.writable() => i, + _ => Install::for_install()?, + }; + dest.install(&Source::locate(), &mut |line| eprintln!("{line}"))?; + println!( + "superfluid {RELEASE} is installed at {}", + dest.root.display() + ); + Ok(()) + } + Action::Status => { + println!("pinned: {RELEASE}"); + let code = match Install::locate()? { + None => { + let dest = Install::for_install()?; + println!( + "installed: nothing (`basert superfluid install` puts {RELEASE} at {})", + dest.root.display() + ); + 1 + } + Some(i) => { + let have = i + .installed_release() + .unwrap_or_else(|| "a release that does not say".to_string()); + println!( + "installed: {have} ({}, {})", + i.root.display(), + i.origin.said() + ); + let engine = crate::serve::engine_dir().ok().flatten(); + let home = crate::serve::superfluid_home().ok(); + match i.probe(engine.as_deref(), home.as_deref()).ready { + Ok(()) => { + println!("engine: ready to serve"); + 0 + } + Err(why) => { + println!("engine: cannot serve: {why}"); + 2 + } + } + } + }; + std::process::exit(code) + } + } +} + +/// The release tarball's target triple for this machine. +pub fn target() -> Result<&'static str> { + match (std::env::consts::OS, std::env::consts::ARCH) { + ("macos", "aarch64") => Ok("aarch64-apple-darwin"), + ("linux", "aarch64") => Ok("aarch64-unknown-linux-gnu"), + ("linux", "x86_64") => Ok("x86_64-unknown-linux-gnu"), + (os, arch) => bail!( + "superfluid, which basert serve runs, publishes no release for {os}/{arch}; build it \ + from https://github.com/{REPO} and set {DIR_ENV} to a directory holding bin/superfluid" + ), + } +} + +/// basert's own place for a superfluid it installs when the directory beside +/// it cannot be written: under the user's local data directory +/// (`~/Library/Application Support/baseRT` on macOS, `$XDG_DATA_HOME/baseRT` +/// or `~/.local/share/baseRT` on Linux). +pub fn user_data_dir() -> Result { + let data = dirs::data_local_dir() + .context("no data directory to install superfluid in (set BASERT_SUPERFLUID_DIR)")?; + Ok(data.join("baseRT").join("superfluid")) +} + +/// Where an install was looked for. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Origin { + /// `$BASERT_SUPERFLUID_DIR`. + Env, + /// `superfluid/` beside basert: the one-shot installer's layout. + BesideBasert, + /// [`user_data_dir`]. + UserData, +} + +impl Origin { + /// For a message: where this is. + fn said(self) -> &'static str { + match self { + Origin::Env => "named by BASERT_SUPERFLUID_DIR", + Origin::BesideBasert => "beside basert", + Origin::UserData => "basert's data directory", + } + } +} + +/// A superfluid install: `root/bin/superfluid`, with the runtime workers in +/// `root/libexec/superfluid/`, where superfluid looks for them. +#[derive(Debug, Clone)] +pub struct Install { + pub root: PathBuf, + pub origin: Origin, +} + +impl Install { + /// Where superfluid is looked for, in order: `$BASERT_SUPERFLUID_DIR` + /// alone when set; else beside basert, then the user's data directory. + pub fn candidates() -> Result> { + if let Some(dir) = std::env::var_os(DIR_ENV).filter(|d| !d.is_empty()) { + return Ok(vec![Install { + root: PathBuf::from(dir), + origin: Origin::Env, + }]); + } + let exe = std::env::current_exe().context("this program's own path")?; + let dir = exe.parent().context("this program's own directory")?; + Ok(vec![ + Install { + root: dir.join("superfluid"), + origin: Origin::BesideBasert, + }, + Install { + root: user_data_dir()?, + origin: Origin::UserData, + }, + ]) + } + + /// The first place with a superfluid installed. + pub fn locate() -> Result> { + Ok(Self::candidates()?.into_iter().find(Install::is_installed)) + } + + /// Where an install goes: the first place that can be written, else the + /// last, whose own error then says why not. + pub fn for_install() -> Result { + let candidates = Self::candidates()?; + let last = candidates.len() - 1; + Ok(candidates + .into_iter() + .enumerate() + .find(|(k, c)| *k == last || c.writable()) + .map(|(_, c)| c) + .expect("at least one candidate")) + } + + /// Whether an install can be made or replaced here: its parent takes a + /// new directory. + pub fn writable(&self) -> bool { + let parent = self.parent(); + if !parent.is_dir() && fs::create_dir_all(&parent).is_err() { + return false; + } + tempfile::Builder::new() + .prefix(".superfluid-probe-") + .tempdir_in(&parent) + .is_ok() + } + + fn parent(&self) -> PathBuf { + match self.root.parent() { + Some(p) if !p.as_os_str().is_empty() => p.to_path_buf(), + _ => PathBuf::from("."), + } + } + + pub fn binary(&self) -> PathBuf { + self.root.join("bin").join("superfluid") + } + + pub fn is_installed(&self) -> bool { + is_executable(&self.binary()) + } + + /// The release installed here: the stamp an install writes, else what the + /// binary says of itself (a copy put here by hand). `None` when nothing is + /// installed. + pub fn installed_release(&self) -> Option { + if !self.is_installed() { + return None; + } + if let Ok(stamp) = fs::read_to_string(self.root.join(STAMP)) { + let stamp = stamp.trim(); + if !stamp.is_empty() { + return Some(stamp.to_string()); + } + } + let out = Command::new(self.binary()).arg("--version").output().ok()?; + // `superfluid --version` prints "superfluid 0.1.3". + let text = String::from_utf8_lossy(&out.stdout); + let version = text.split_whitespace().nth(1)?; + Some(match version.strip_prefix('v') { + Some(_) => version.to_string(), + None => format!("v{version}"), + }) + } + + /// Ask this superfluid whether its basert runtime can serve the engine + /// library in `engine` (`None`: the one `BASERT_LIB` or superfluid's own + /// search finds), with `home` as its `SUPERFLUID_HOME`. The answer is + /// superfluid's: `runtimes --json` reports the runtime ready, or why not. + pub fn probe(&self, engine: Option<&Path>, home: Option<&Path>) -> Probe { + let mut command = Command::new(self.binary()); + command + .args(["runtimes", "--json"]) + .env("SUPERFLUID_NO_UPDATE_CHECK", "1") + .stdin(Stdio::null()); + if let Some(dir) = engine { + command.env("BASERT_LIB", dir); + } + if let Some(dir) = home { + command.env("SUPERFLUID_HOME", dir); + } + match command.output() { + Ok(out) => Probe::from_report(&out.stdout, &out.stderr, out.status.success()), + Err(e) => Probe::failed(format!("{} could not be run: {e}", self.binary().display())), + } + } + + /// Download the pinned release, check its SHA-256 against the one the + /// release publishes, unpack it beside the install and swap it in. An + /// install that is there stays until the new one is complete, and comes + /// back if the swap fails. + pub fn install(&self, source: &Source, say: &mut dyn FnMut(&str)) -> Result<()> { + let name = format!("superfluid-{}", target()?); + let parent = self.parent(); + fs::create_dir_all(&parent).with_context(|| format!("creating {}", parent.display()))?; + // Beside the install, so the final rename stays on one filesystem. + let stage = tempfile::Builder::new() + .prefix(".superfluid-new-") + .tempdir_in(&parent) + .with_context(|| format!("creating a staging directory in {}", parent.display()))?; + let tarball = stage.path().join(format!("{name}.tar.gz")); + let checksum = stage.path().join(format!("{name}.tar.gz.sha256")); + + say(&format!( + "downloading superfluid {RELEASE} ({name}.tar.gz) from {source}" + )); + source.fetch(&format!("{name}.tar.gz"), &tarball)?; + source.fetch(&format!("{name}.tar.gz.sha256"), &checksum)?; + let want = fs::read_to_string(&checksum) + .with_context(|| format!("reading {}", checksum.display()))? + .split_whitespace() + .next() + .unwrap_or_default() + .to_ascii_lowercase(); + let have = sha256_of(&tarball)?; + if want.len() != 64 || want != have { + bail!( + "{name}.tar.gz: SHA-256 is {have}, the release's checksum file says {want:?}: \ + the download is not the published file" + ); + } + + let unpacked = stage.path().join("unpacked"); + fs::create_dir(&unpacked)?; + let status = Command::new("tar") + .arg("-xzf") + .arg(&tarball) + .arg("-C") + .arg(&unpacked) + .status() + .context("running tar")?; + if !status.success() { + bail!("tar -xzf {} failed ({status})", tarball.display()); + } + let fresh = unpacked.join(&name); + if !is_executable(&fresh.join("bin").join("superfluid")) { + bail!("{name}.tar.gz holds no {name}/bin/superfluid"); + } + fs::write(fresh.join(STAMP), format!("{RELEASE}\n"))?; + + let old = parent.join(format!(".superfluid-old-{}", std::process::id())); + let had_old = self.root.exists(); + if had_old { + fs::rename(&self.root, &old).with_context(|| { + format!( + "moving the installed superfluid aside ({})", + self.root.display() + ) + })?; + } + if let Err(e) = fs::rename(&fresh, &self.root) { + if had_old { + let _ = fs::rename(&old, &self.root); + } + return Err(e) + .with_context(|| format!("installing superfluid into {}", self.root.display())); + } + if had_old { + let _ = fs::remove_dir_all(&old); + } + say(&format!( + "installed superfluid {RELEASE} at {}", + self.root.display() + )); + Ok(()) + } +} + +/// What a superfluid says of its basert runtime against an engine. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Probe { + /// Ready, or why not, in superfluid's words. + pub ready: Result<(), String>, + /// The engine ABI number in use, when superfluid reports one (0.1.3 on). + pub abi: Option, +} + +impl Probe { + fn failed(why: String) -> Probe { + Probe { + ready: Err(why), + abi: None, + } + } + + /// The basert row of `superfluid runtimes --json`: its `ready` is `true` + /// or the reason it is not, from 0.1.2 on; `abi` arrives with 0.1.3. + pub fn from_report(stdout: &[u8], stderr: &[u8], ok: bool) -> Probe { + let text = String::from_utf8_lossy(stdout); + let report: serde_json::Value = match serde_json::from_str(text.trim()) { + Ok(v) => v, + Err(_) => { + let err = String::from_utf8_lossy(stderr); + let said = [err.trim(), text.trim()] + .into_iter() + .find(|s| !s.is_empty()) + .and_then(|s| s.lines().last()) + .unwrap_or("nothing"); + let what = if ok { "answered no report" } else { "failed" }; + return Probe::failed(format!("`superfluid runtimes --json` {what}: {said}")); + } + }; + let Some(row) = report + .as_array() + .into_iter() + .flatten() + .find(|r| r["id"] == RUNTIME_ID) + else { + return Probe::failed("this superfluid ships no basert runtime".to_string()); + }; + let ready = match &row["ready"] { + serde_json::Value::Bool(true) => Ok(()), + serde_json::Value::String(why) => Err(why.clone()), + _ => Err(row["state"].as_str().unwrap_or("not ready").to_string()), + }; + Probe { + ready, + abi: row["abi"].as_u64(), + } + } +} + +/// Where a release's files come from. +pub enum Source { + /// The release's assets on GitHub. + GitHub, + /// A directory holding the tarball and its `.sha256` ([`SOURCE_ENV`]). + Dir(PathBuf), +} + +impl Source { + pub fn locate() -> Source { + match std::env::var_os(SOURCE_ENV).filter(|d| !d.is_empty()) { + Some(dir) => Source::Dir(PathBuf::from(dir)), + None => Source::GitHub, + } + } + + fn fetch(&self, file: &str, dest: &Path) -> Result<()> { + match self { + Source::Dir(dir) => { + let from = dir.join(file); + fs::copy(&from, dest) + .with_context(|| format!("copying {}", from.display())) + .map(|_| ()) + } + Source::GitHub => { + let url = format!("https://github.com/{REPO}/releases/download/{RELEASE}/{file}"); + // GitHub answers an asset with a redirect to its storage, which + // the agent follows. A server that goes quiet fails the install + // rather than hanging a `basert serve` start. + let agent: ureq::Agent = ureq::Agent::config_builder() + .timeout_connect(Some(Duration::from_secs(30))) + .timeout_recv_response(Some(Duration::from_secs(60))) + .timeout_recv_body(Some(Duration::from_secs(600))) + .build() + .into(); + let response = agent + .get(&url) + .call() + .with_context(|| format!("downloading {url}"))?; + let mut body = response.into_body().into_reader(); + let mut out = fs::File::create(dest) + .with_context(|| format!("creating {}", dest.display()))?; + std::io::copy(&mut body, &mut out).with_context(|| format!("reading {url}"))?; + Ok(()) + } + } + } +} + +impl fmt::Display for Source { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Source::GitHub => write!(f, "https://github.com/{REPO}/releases/tag/{RELEASE}"), + Source::Dir(dir) => write!(f, "{}", dir.display()), + } + } +} + +/// Whether a start may download superfluid. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Download { + /// Ask first: a terminal is there to answer. + Ask, + /// Download without asking, saying so. + Auto, + /// Never: name the command instead. + Never, +} + +/// The policy for this start: `$BASERT_SUPERFLUID_AUTO_INSTALL` decides when +/// set (`0` never, `1` without asking); otherwise a terminal is asked, and a +/// start with none (Local's, a service's) downloads and says so. +pub fn policy() -> Download { + match std::env::var(AUTO_INSTALL_ENV) + .ok() + .as_deref() + .map(str::trim) + .map(str::to_ascii_lowercase) + .as_deref() + { + Some("0") | Some("no") | Some("false") => Download::Never, + Some("1") | Some("yes") | Some("true") => Download::Auto, + _ if std::io::stdin().is_terminal() && std::io::stderr().is_terminal() => Download::Ask, + _ => Download::Auto, + } +} + +/// The superfluid a `basert serve` start runs, as a path to its binary: the +/// first install whose basert runtime superfluid itself reports ready for +/// the engine in `engine`, under `home` as its `SUPERFLUID_HOME`. `say` gets +/// a line for each step taken on the user's behalf. +pub fn ensure(engine: Option<&Path>, home: &Path, say: &mut dyn FnMut(&str)) -> Result { + ensure_with(policy(), &Source::locate(), engine, home, say) +} + +pub fn ensure_with( + policy: Download, + source: &Source, + engine: Option<&Path>, + home: &Path, + say: &mut dyn FnMut(&str), +) -> Result { + // Every install found is asked; the first that can serve runs. An install + // that cannot and is the pin itself makes the engine the problem, which no + // download mends. + let mut unready: Vec = Vec::new(); + for install in Install::candidates()? + .into_iter() + .filter(Install::is_installed) + { + let why = match install.probe(engine, Some(home)).ready { + Ok(()) => return Ok(install.binary()), + Err(why) => why, + }; + let have = install.installed_release(); + if have.as_deref() == Some(RELEASE) { + bail!( + "superfluid {RELEASE} at {} cannot serve this engine: {why}", + install.root.display() + ); + } + say(&format!( + "superfluid {} at {} cannot serve this engine ({why}); {RELEASE} can", + have.as_deref().unwrap_or("of a release it does not say"), + install.root.display() + )); + unready.push(install); + } + let dest = match unready.iter().find(|i| i.writable()) { + Some(found) => found.clone(), + None => Install::for_install()?, + }; + let hint = format!( + "install it with\n basert superfluid install\n(superfluid {RELEASE} from \ + https://github.com/{REPO}/releases, into {})", + dest.root.display() + ); + match policy { + Download::Never => { + if unready.is_empty() { + bail!("superfluid is not installed, and {AUTO_INSTALL_ENV}=0 forbids a download at a start; {hint}"); + } + bail!("{AUTO_INSTALL_ENV}=0 forbids a download at a start; {hint}"); + } + Download::Ask => { + if unready.is_empty() { + eprintln!("basert serve runs superfluid, which is not installed."); + } + if !confirm(&format!( + "Download and install superfluid {RELEASE} into {} now? [Y/n] ", + dest.root.display() + ))? { + bail!("superfluid {RELEASE} is not installed; {hint}"); + } + } + Download::Auto => say(&format!( + "installing superfluid {RELEASE} into {} ({AUTO_INSTALL_ENV}=0 forbids this)", + dest.root.display() + )), + } + dest.install(source, say)?; + match dest.probe(engine, Some(home)).ready { + Ok(()) => Ok(dest.binary()), + Err(why) => bail!( + "superfluid {RELEASE}, installed at {}, cannot serve this engine: {why}", + dest.root.display() + ), + } +} + +/// A yes/no question on stderr, answered on stdin; an empty answer is yes. +fn confirm(question: &str) -> Result { + eprint!("{question}"); + std::io::stderr().flush()?; + let mut answer = String::new(); + std::io::stdin().lock().read_line(&mut answer)?; + Ok(matches!( + answer.trim().to_ascii_lowercase().as_str(), + "" | "y" | "yes" + )) +} + +fn sha256_of(path: &Path) -> Result { + let mut file = fs::File::open(path).with_context(|| format!("opening {}", path.display()))?; + let mut hasher = Sha256::new(); + std::io::copy(&mut file, &mut hasher)?; + Ok(format!("{:x}", hasher.finalize())) +} + +fn is_executable(path: &Path) -> bool { + use std::os::unix::fs::PermissionsExt; + fs::metadata(path).is_ok_and(|m| m.is_file() && m.permissions().mode() & 0o111 != 0) +} + +#[cfg(test)] +mod tests { + use super::*; + + /// A release directory holding `superfluid-.tar.gz` and its + /// checksum, the tarball unpacking to superfluid's layout with a shell + /// script for a binary: `--version` answers `version`, `runtimes --json` + /// reports the basert runtime ready unless `FAKE_SUPERFLUID_UNREADY` + /// names a reason for the version `FAKE_SUPERFLUID_UNREADY_VERSION`. + fn release(dir: &Path, version: &str, checksum_ok: bool) -> PathBuf { + let name = format!("superfluid-{}", target().unwrap()); + let tree = dir.join("tree"); + let bin = tree.join(&name).join("bin"); + let libexec = tree.join(&name).join("libexec").join("superfluid"); + fs::create_dir_all(&bin).unwrap(); + fs::create_dir_all(&libexec).unwrap(); + fs::write(bin.join("superfluid"), fake_superfluid(version)).unwrap(); + fs::write(libexec.join("superfluid-worker-basert"), "#!/bin/sh\n").unwrap(); + for p in [ + bin.join("superfluid"), + libexec.join("superfluid-worker-basert"), + ] { + use std::os::unix::fs::PermissionsExt; + fs::set_permissions(&p, fs::Permissions::from_mode(0o755)).unwrap(); + } + let assets = dir.join("assets"); + fs::create_dir_all(&assets).unwrap(); + let tarball = assets.join(format!("{name}.tar.gz")); + assert!(Command::new("tar") + .arg("-czf") + .arg(&tarball) + .arg("-C") + .arg(&tree) + .arg(&name) + .status() + .unwrap() + .success()); + let sum = if checksum_ok { + sha256_of(&tarball).unwrap() + } else { + "0".repeat(64) + }; + fs::write( + assets.join(format!("{name}.tar.gz.sha256")), + format!("{sum} {name}.tar.gz\n"), + ) + .unwrap(); + assets + } + + pub(super) fn fake_superfluid(version: &str) -> String { + format!( + "#!/bin/sh\n\ + if [ \"$1\" = \"--version\" ]; then echo \"superfluid {version}\"; exit 0; fi\n\ + if [ \"$1\" = \"runtimes\" ]; then\n\ + if [ -n \"${{FAKE_SUPERFLUID_UNREADY:-}}\" ] && [ \"${{FAKE_SUPERFLUID_UNREADY_VERSION:-}}\" = \"{version}\" ]; then\n\ + echo \"[{{\\\"id\\\":\\\"basert\\\",\\\"state\\\":\\\"not ready\\\",\\\"ready\\\":\\\"$FAKE_SUPERFLUID_UNREADY\\\",\\\"version\\\":\\\"{version}\\\"}}]\"\n\ + else\n\ + echo \"[{{\\\"id\\\":\\\"basert\\\",\\\"state\\\":\\\"ready\\\",\\\"ready\\\":true,\\\"version\\\":\\\"{version}\\\",\\\"abi\\\":1}}]\"\n\ + fi\n\ + exit 0\n\ + fi\n\ + printf 'ARGS:'; for a in \"$@\"; do printf ' %s' \"$a\"; done; printf '\\n'\n" + ) + } + + fn install_at(root: PathBuf) -> Install { + Install { + root, + origin: Origin::Env, + } + } + + #[test] + fn an_install_unpacks_the_release_and_stamps_it() { + let tmp = tempfile::tempdir().unwrap(); + let assets = release(tmp.path(), "0.1.3", true); + let install = install_at(tmp.path().join("bin").join("superfluid")); + assert!(!install.is_installed()); + assert_eq!(install.installed_release(), None); + let mut said = Vec::new(); + install + .install(&Source::Dir(assets), &mut |l| said.push(l.to_string())) + .unwrap(); + assert!(install.is_installed()); + assert!(install + .root + .join("libexec") + .join("superfluid") + .join("superfluid-worker-basert") + .is_file()); + assert_eq!(install.installed_release().as_deref(), Some(RELEASE)); + assert_eq!(said.len(), 2, "{said:?}"); + assert!(said[0].starts_with(&format!("downloading superfluid {RELEASE} ("))); + // Nothing of the staging is left behind. + let left: Vec<_> = fs::read_dir(tmp.path().join("bin")) + .unwrap() + .map(|e| e.unwrap().file_name().into_string().unwrap()) + .collect(); + assert_eq!(left, ["superfluid"]); + } + + #[test] + fn a_wrong_checksum_refuses_the_download_and_keeps_what_is_installed() { + let tmp = tempfile::tempdir().unwrap(); + let good = release(&tmp.path().join("good"), "0.1.3", true); + let bad = release(&tmp.path().join("bad"), "0.1.3", false); + let install = install_at(tmp.path().join("superfluid")); + install.install(&Source::Dir(good), &mut |_| {}).unwrap(); + fs::write(install.root.join("marker"), "kept").unwrap(); + let err = install + .install(&Source::Dir(bad), &mut |_| {}) + .unwrap_err() + .to_string(); + assert!( + err.contains("the download is not the published file"), + "{err}" + ); + assert_eq!( + fs::read_to_string(install.root.join("marker")).unwrap(), + "kept" + ); + assert_eq!(install.installed_release().as_deref(), Some(RELEASE)); + } + + #[test] + fn a_missing_asset_is_an_error_that_names_it() { + let tmp = tempfile::tempdir().unwrap(); + let install = install_at(tmp.path().join("superfluid")); + let err = install + .install(&Source::Dir(tmp.path().join("nowhere")), &mut |_| {}) + .unwrap_err() + .to_string(); + assert!(err.contains("copying") && err.contains(".tar.gz"), "{err}"); + assert!(!install.root.exists()); + } + + #[test] + fn the_release_installed_is_the_stamp_else_what_the_binary_says() { + let tmp = tempfile::tempdir().unwrap(); + let assets = release(tmp.path(), "0.0.9", true); + let install = install_at(tmp.path().join("superfluid")); + install.install(&Source::Dir(assets), &mut |_| {}).unwrap(); + assert_eq!(install.installed_release().as_deref(), Some(RELEASE)); + // A copy put here by hand has no stamp: the binary is asked. + fs::remove_file(install.root.join(STAMP)).unwrap(); + assert_eq!(install.installed_release().as_deref(), Some("v0.0.9")); + // A second install replaces it, stamp included. + let newer = release(&tmp.path().join("again"), "0.1.3", true); + install.install(&Source::Dir(newer), &mut |_| {}).unwrap(); + assert_eq!(install.installed_release().as_deref(), Some(RELEASE)); + assert!(!tmp + .path() + .join(format!(".superfluid-old-{}", std::process::id())) + .exists()); + } + + #[test] + fn the_target_is_one_superfluid_publishes() { + let t = target().unwrap(); + assert!( + [ + "aarch64-apple-darwin", + "aarch64-unknown-linux-gnu", + "x86_64-unknown-linux-gnu" + ] + .contains(&t), + "{t}" + ); + } + + #[test] + fn the_probe_reads_the_basert_row_of_either_generation() { + // 0.1.2: ready is true or the reason; no abi. + let p = Probe::from_report( + br#"[{"id":"llamacpp","state":"not installed","ready":"no worker"},{"id":"basert","state":"ready","ready":true,"version":"0.1.2"}]"#, + b"", + true, + ); + assert_eq!( + p, + Probe { + ready: Ok(()), + abi: None + } + ); + // 0.1.3: the abi comes along. + let p = Probe::from_report( + br#"[{"id":"basert","state":"ready","ready":true,"version":"0.1.3","abi":1,"requires":{"abi":1}}]"#, + b"", + true, + ); + assert_eq!( + p, + Probe { + ready: Ok(()), + abi: Some(1) + } + ); + let p = Probe::from_report( + br#"[{"id":"basert","state":"not ready","ready":"libbaseRT 0.4.0 states no ABI number and is not a 0.3 library"}]"#, + b"", + true, + ); + assert_eq!( + p.ready.unwrap_err(), + "libbaseRT 0.4.0 states no ABI number and is not a 0.3 library" + ); + // No basert row, no report, a failure: each says what it saw. + assert!( + Probe::from_report(br#"[{"id":"mlx","ready":true}]"#, b"", true) + .ready + .unwrap_err() + .contains("no basert runtime") + ); + let why = Probe::from_report(b"", b"error: unknown flag --json\n", false) + .ready + .unwrap_err(); + assert!( + why.contains("failed") && why.contains("unknown flag --json"), + "{why}" + ); + let why = Probe::from_report(b"not json", b"", true) + .ready + .unwrap_err(); + assert!(why.contains("answered no report"), "{why}"); + } + + #[test] + fn the_probe_runs_the_binary_with_the_engine_and_home_named() { + let tmp = tempfile::tempdir().unwrap(); + let assets = release(tmp.path(), "0.1.3", true); + let install = install_at(tmp.path().join("superfluid")); + install.install(&Source::Dir(assets), &mut |_| {}).unwrap(); + let p = install.probe(Some(Path::new("/lib")), Some(Path::new("/home"))); + assert_eq!( + p, + Probe { + ready: Ok(()), + abi: Some(1) + } + ); + let p = install_at(tmp.path().join("nowhere")).probe(None, None); + assert!(p.ready.unwrap_err().contains("could not be run")); + } + + /// `ensure_with` against installs named by `BASERT_SUPERFLUID_DIR` is + /// covered end to end in tests/serve_launcher.rs, where the environment + /// is the launched process's own; here the pieces that need none. + #[test] + fn the_policy_words_are_read_as_set() { + // The environment is process-wide: this test alone touches the variable, + // and only to values, never to unset it mid-flight. + for (value, want) in [ + ("0", Download::Never), + ("no", Download::Never), + ("1", Download::Auto), + ("yes", Download::Auto), + ] { + std::env::set_var(AUTO_INSTALL_ENV, value); + assert_eq!(policy(), want, "{value}"); + } + std::env::remove_var(AUTO_INSTALL_ENV); + // Unset: a terminal is asked, and a start without one downloads. + let terminal = std::io::stdin().is_terminal() && std::io::stderr().is_terminal(); + assert_eq!( + policy(), + if terminal { + Download::Ask + } else { + Download::Auto + } + ); + } +} diff --git a/base-convert/crates/base-convert/tests/serve_launcher.rs b/base-convert/crates/base-convert/tests/serve_launcher.rs new file mode 100644 index 0000000..8b3a3ba --- /dev/null +++ b/base-convert/crates/base-convert/tests/serve_launcher.rs @@ -0,0 +1,609 @@ +//! `basert serve` and `basert superfluid`, end to end against a fake superfluid +//! release: the launcher installs it from a directory of release assets, asks +//! it whether it can serve the engine, brings one that cannot to the pin, and +//! execs it with the C++ server's command line translated to superfluid's. + +use std::fs; +use std::path::{Path, PathBuf}; +use std::process::{Command, Output, Stdio}; + +use sha2::{Digest, Sha256}; +use tempfile::TempDir; + +fn target() -> &'static str { + match (std::env::consts::OS, std::env::consts::ARCH) { + ("macos", "aarch64") => "aarch64-apple-darwin", + ("linux", "aarch64") => "aarch64-unknown-linux-gnu", + ("linux", "x86_64") => "x86_64-unknown-linux-gnu", + other => panic!("no superfluid release for {other:?}"), + } +} + +/// A release directory, an install directory for it, a model file, a state +/// directory and an engine directory, all under one temporary directory. +struct Fixture { + tmp: TempDir, + assets: PathBuf, + install: PathBuf, + model: PathBuf, + state: PathBuf, + lib: PathBuf, +} + +fn executable(path: &Path, text: &str) { + use std::os::unix::fs::PermissionsExt; + fs::write(path, text).unwrap(); + fs::set_permissions(path, fs::Permissions::from_mode(0o755)).unwrap(); +} + +/// Release assets whose `superfluid` is a shell script: `--version` answers +/// `version`; `runtimes --json` reports the basert runtime ready, unless +/// `FAKE_SUPERFLUID_UNREADY` names a reason and `FAKE_SUPERFLUID_UNREADY_VERSION` +/// is this version, and notes the environment it was probed under in the +/// file `FAKE_SUPERFLUID_PROBE_LOG` names; anything else echoes its arguments +/// and the environment the launcher sets. +fn release_assets(dir: &Path, version: &str) -> PathBuf { + let name = format!("superfluid-{}", target()); + let tree = dir.join("tree"); + let bin = tree.join(&name).join("bin"); + let libexec = tree.join(&name).join("libexec").join("superfluid"); + fs::create_dir_all(&bin).unwrap(); + fs::create_dir_all(&libexec).unwrap(); + executable( + &bin.join("superfluid"), + &format!( + "#!/bin/sh\n\ + if [ \"$1\" = \"--version\" ]; then echo \"superfluid {version}\"; exit 0; fi\n\ + if [ \"$1\" = \"runtimes\" ]; then\n\ + if [ -n \"${{FAKE_SUPERFLUID_PROBE_LOG:-}}\" ]; then echo \"PROBE: BASERT_LIB=${{BASERT_LIB:-unset}} SUPERFLUID_HOME=${{SUPERFLUID_HOME:-unset}} SUPERFLUID_NO_UPDATE_CHECK=${{SUPERFLUID_NO_UPDATE_CHECK:-unset}}\" >> \"$FAKE_SUPERFLUID_PROBE_LOG\"; fi\n\ + if [ -n \"${{FAKE_SUPERFLUID_UNREADY:-}}\" ] && [ \"${{FAKE_SUPERFLUID_UNREADY_VERSION:-}}\" = \"{version}\" ]; then\n\ + echo \"[{{\\\"id\\\":\\\"basert\\\",\\\"state\\\":\\\"not ready\\\",\\\"ready\\\":\\\"$FAKE_SUPERFLUID_UNREADY\\\",\\\"version\\\":\\\"{version}\\\"}}]\"\n\ + else\n\ + echo \"[{{\\\"id\\\":\\\"basert\\\",\\\"state\\\":\\\"ready\\\",\\\"ready\\\":true,\\\"version\\\":\\\"{version}\\\",\\\"abi\\\":1}}]\"\n\ + fi\n\ + exit 0\n\ + fi\n\ + printf 'ARGS:'; for a in \"$@\"; do printf ' %s' \"$a\"; done; printf '\\n'\n\ + echo \"SUPERFLUID_HOME=$SUPERFLUID_HOME\"\n\ + echo \"SUPERFLUID_NO_UPDATE_CHECK=${{SUPERFLUID_NO_UPDATE_CHECK:-unset}}\"\n\ + echo \"SUPERFLUID_ENGINE_VERBOSE=${{SUPERFLUID_ENGINE_VERBOSE:-unset}}\"\n" + ), + ); + executable(&libexec.join("superfluid-worker-basert"), "#!/bin/sh\n"); + let assets = dir.join("assets"); + fs::create_dir_all(&assets).unwrap(); + let tarball = assets.join(format!("{name}.tar.gz")); + assert!(Command::new("tar") + .arg("-czf") + .arg(&tarball) + .arg("-C") + .arg(&tree) + .arg(&name) + .status() + .unwrap() + .success()); + let sum = format!("{:x}", Sha256::digest(fs::read(&tarball).unwrap())); + fs::write( + assets.join(format!("{name}.tar.gz.sha256")), + format!("{sum} {name}.tar.gz\n"), + ) + .unwrap(); + assets +} + +fn fixture() -> Fixture { + let tmp = tempfile::tempdir().unwrap(); + let assets = release_assets(&tmp.path().join("release"), "9.9.9"); + let model = tmp.path().join("m.base"); + fs::write(&model, b"").unwrap(); + let lib = tmp.path().join("lib"); + fs::create_dir(&lib).unwrap(); + Fixture { + install: tmp.path().join("superfluid"), + state: tmp.path().join("state"), + assets, + model, + lib, + tmp, + } +} + +impl Fixture { + /// `basert` with its superfluid, its release source, its state and the + /// engine directory all under the fixture, stdin closed (no terminal), + /// and no policy set: a start without a terminal downloads. + fn basert(&self) -> Command { + let mut c = Command::new(env!("CARGO_BIN_EXE_basert")); + c.env("BASERT_SUPERFLUID_DIR", &self.install) + .env("BASERT_SUPERFLUID_SOURCE", &self.assets) + .env("BASERT_STATE_DIR", &self.state) + .env("BASERT_LIB", &self.lib) + .env("FAKE_SUPERFLUID_PROBE_LOG", self.probe_log()) + .env_remove("BASERT_SUPERFLUID_AUTO_INSTALL") + .env_remove("FAKE_SUPERFLUID_UNREADY") + .env_remove("FAKE_SUPERFLUID_UNREADY_VERSION") + .stdin(Stdio::null()); + c + } + + fn model(&self) -> &str { + self.model.to_str().unwrap() + } + + fn probe_log(&self) -> PathBuf { + self.tmp.path().join("probes.log") + } + + /// What the fake superfluid was probed under, one line per probe. + fn probes(&self) -> String { + fs::read_to_string(self.probe_log()).unwrap_or_default() + } + + fn stamp(&self) -> String { + fs::read_to_string(self.install.join(".release")) + .unwrap() + .trim() + .to_string() + } + + /// Install, from assets of `version`, stamped as that version: a copy of + /// another release than the pin. + fn install_old(&self, version: &str) -> PathBuf { + let assets = release_assets(&self.tmp.path().join(format!("release-{version}")), version); + let (code, out, err) = run(self + .basert() + .env("BASERT_SUPERFLUID_SOURCE", &assets) + .args(["superfluid", "install"])); + assert_eq!(code, 0, "{out}{err}"); + fs::write(self.install.join(".release"), format!("v{version}\n")).unwrap(); + assets + } +} + +fn run(command: &mut Command) -> (i32, String, String) { + let Output { + status, + stdout, + stderr, + } = command.output().unwrap(); + ( + status.code().unwrap_or(-1), + String::from_utf8_lossy(&stdout).into_owned(), + String::from_utf8_lossy(&stderr).into_owned(), + ) +} + +/// The pinned release, as `basert superfluid status` prints it. +fn pinned(f: &Fixture) -> String { + let (_, out, _) = run(f.basert().args(["superfluid", "status"])); + out.lines() + .find_map(|l| l.strip_prefix("pinned:")) + .map(|v| v.trim().to_string()) + .unwrap_or_else(|| panic!("no pinned line in {out:?}")) +} + +#[test] +fn superfluid_install_puts_the_pinned_release_in_place_and_status_says_so() { + let f = fixture(); + let (code, out, _) = run(f.basert().args(["superfluid", "status"])); + assert_eq!(code, 1, "{out}"); + assert!(out.contains("installed: nothing"), "{out}"); + let pin = pinned(&f); + assert!(pin.starts_with('v'), "{pin}"); + + let (code, out, err) = run(f.basert().args(["superfluid", "install"])); + assert_eq!(code, 0, "{out}{err}"); + assert!( + err.contains(&format!("downloading superfluid {pin} (")), + "{err}" + ); + assert!( + out.contains(&format!("superfluid {pin} is installed at")), + "{out}" + ); + assert!(f.install.join("bin").join("superfluid").is_file()); + assert!(f + .install + .join("libexec") + .join("superfluid") + .join("superfluid-worker-basert") + .is_file()); + assert_eq!(f.stamp(), pin); + + let (code, out, _) = run(f.basert().args(["superfluid", "status"])); + assert_eq!(code, 0, "{out}"); + assert!(out.contains(&format!("installed: {pin} (")), "{out}"); + assert!(out.contains("engine: ready to serve"), "{out}"); + + // Installed at the pin: nothing to download. + let (code, out, err) = run(f.basert().args(["superfluid", "install"])); + assert_eq!(code, 0, "{out}{err}"); + assert!(err.is_empty(), "{err}"); + assert!(out.contains("is installed at"), "{out}"); + + // One that cannot serve the engine: status says so and exits 2. + let (code, out, _) = run(f + .basert() + .env("FAKE_SUPERFLUID_UNREADY", "libbaseRT states no ABI number") + .env("FAKE_SUPERFLUID_UNREADY_VERSION", "9.9.9") + .args(["superfluid", "status"])); + assert_eq!(code, 2, "{out}"); + assert!( + out.contains("engine: cannot serve: libbaseRT states no ABI number"), + "{out}" + ); +} + +#[test] +fn serve_without_superfluid_and_without_a_terminal_installs_it_and_says_so() { + let f = fixture(); + let pin = pinned(&f); + let (code, out, err) = run(f.basert().args(["serve", f.model()])); + assert_eq!(code, 0, "{out}{err}"); + assert!( + err.contains(&format!("basert serve: installing superfluid {pin} into")), + "{err}" + ); + assert!( + err.contains("BASERT_SUPERFLUID_AUTO_INSTALL=0 forbids this"), + "{err}" + ); + assert!( + err.contains(&format!("downloading superfluid {pin} (")), + "{err}" + ); + assert_eq!(f.stamp(), pin); + assert!(out.contains("ARGS: serve"), "{out}"); + // The probe ran against the engine, under the server's own home, with + // superfluid's update check off. + let probes = f.probes(); + assert!( + probes.contains(&format!( + "PROBE: BASERT_LIB={} SUPERFLUID_HOME={} SUPERFLUID_NO_UPDATE_CHECK=1", + f.lib.display(), + f.state.join("superfluid").display() + )), + "{probes}" + ); +} + +#[test] +fn a_start_forbidden_to_download_names_the_install_command() { + let f = fixture(); + let (code, out, err) = run(f + .basert() + .env("BASERT_SUPERFLUID_AUTO_INSTALL", "0") + .args(["serve", f.model()])); + assert_eq!(code, 1, "{out}{err}"); + assert!(err.contains("superfluid is not installed"), "{err}"); + assert!(err.contains("basert superfluid install"), "{err}"); + assert!(!f.install.exists(), "nothing is downloaded when forbidden"); +} + +#[test] +fn serve_execs_superfluid_with_the_cpp_servers_command_line_translated() { + let f = fixture(); + assert_eq!(run(f.basert().args(["superfluid", "install"])).0, 0); + let sessions = f.state.join("local"); + let (code, out, err) = run(f.basert().args([ + "serve", + f.model(), + "--port", + "9000", + "--api-key", + "k", + "--max-batch", + "2", + "--sessions", + sessions.to_str().unwrap(), + "--paged-kv", + "--verbose", + ])); + assert_eq!(code, 0, "{out}{err}"); + let want = format!( + "ARGS: serve --model {} --runtime basert --sessions {} --host 127.0.0.1 --port 9000 \ + --max-batch 2 --api-key k --offline --no-park --sessions-cache --no-tui --no-http-qos-header", + f.model(), + sessions.display() + ); + assert!(out.lines().any(|l| l == want), "{out}\nwant {want}"); + assert!( + out.contains(&format!( + "SUPERFLUID_HOME={}", + f.state.join("superfluid").display() + )), + "{out}" + ); + assert!(out.contains("SUPERFLUID_NO_UPDATE_CHECK=1"), "{out}"); + assert!(out.contains("SUPERFLUID_ENGINE_VERBOSE=1"), "{out}"); + assert!(err.contains("--paged-kv is no longer used"), "{err}"); + // BASERT_LIB names the engine: superfluid reads it itself. + assert!(!out.contains("--basert-lib"), "{out}"); + // No download was needed: nothing said of one. + assert!(!err.contains("installing superfluid"), "{err}"); +} + +#[test] +fn superfluids_own_flags_pass_through_and_print_command_shows_the_start() { + let f = fixture(); + assert_eq!(run(f.basert().args(["superfluid", "install"])).0, 0); + let (code, out, err) = run(f.basert().args([ + "serve", + f.model(), + "--speculate", + "auto", + "--prefix-blob-budget-mb", + "512", + "--", + "--anything", + ])); + assert_eq!(code, 0, "{out}{err}"); + assert!( + out.lines().any(|l| l.ends_with( + "--no-http-qos-header --speculate auto --prefix-blob-budget-mb 512 --anything" + )), + "{out}" + ); + + // --print-command shows the start and runs nothing, installed or not. + let (code, out, err) = + run(f + .basert() + .args(["serve", f.model(), "--print-command", "--port", "9001"])); + assert_eq!(code, 0, "{out}{err}"); + let line = out + .lines() + .find(|l| l.starts_with("SUPERFLUID_HOME=")) + .unwrap_or_else(|| panic!("{out}")); + assert!(line.contains("SUPERFLUID_NO_UPDATE_CHECK=1"), "{line}"); + assert!( + line.contains(&format!( + "{} serve --model {}", + f.install.join("bin").join("superfluid").display(), + f.model() + )), + "{line}" + ); + assert!(line.contains("--port 9001"), "{line}"); + assert!(!out.contains("ARGS:"), "{out}"); + let g = fixture(); + let (code, out, _) = run(g.basert().args(["serve", g.model(), "--print-command"])); + assert_eq!(code, 0, "{out}"); + assert!(out.contains("superfluid (not installed;"), "{out}"); + assert!(!g.install.exists(), "print-command installs nothing"); +} + +#[test] +fn serve_hands_superfluid_the_engine_library_beside_the_launcher() { + let f = fixture(); + // A bundle: basert and libbaseRT in one directory, superfluid installed + // beside them by default. + let bundle = f.install.parent().unwrap().join("bundle"); + fs::create_dir(&bundle).unwrap(); + fs::copy(env!("CARGO_BIN_EXE_basert"), bundle.join("basert")).unwrap(); + for lib in ["libbaseRT.dylib", "libbaseRT.so"] { + fs::write(bundle.join(lib), b"").unwrap(); + } + let basert = || { + let mut c = Command::new(bundle.join("basert")); + c.env("BASERT_SUPERFLUID_SOURCE", &f.assets) + .env("BASERT_STATE_DIR", &f.state) + .env("FAKE_SUPERFLUID_PROBE_LOG", f.probe_log()) + .env("HOME", f.tmp.path()) + .env("XDG_DATA_HOME", f.tmp.path().join("xdg")) + .env_remove("BASERT_SUPERFLUID_DIR") + .env_remove("BASERT_LIB") + .env_remove("BASERT_SUPERFLUID_AUTO_INSTALL") + .stdin(Stdio::null()); + c + }; + let (code, out, err) = run(basert().args(["superfluid", "install"])); + assert_eq!(code, 0, "{out}{err}"); + assert!(bundle + .join("superfluid") + .join("bin") + .join("superfluid") + .is_file()); + + let (code, out, err) = run(basert().args(["serve", f.model()])); + assert_eq!(code, 0, "{out}{err}"); + let canonical = fs::canonicalize(&bundle).unwrap(); + assert!( + out.contains(&format!("--basert-lib {}", canonical.display())), + "{out}" + ); + assert!(out.contains("--port 8453 --max-batch 1"), "{out}"); + // The probe was told the same engine directory. + let probes = f.probes(); + assert!( + probes.contains(&format!("PROBE: BASERT_LIB={}", canonical.display())), + "{probes}" + ); + // The user's own --basert-lib is superfluid's flag, given once, and the + // probe's engine. + let (code, out, err) = + run(basert().args(["serve", f.model(), "--basert-lib", f.lib.to_str().unwrap()])); + assert_eq!(code, 0, "{out}{err}"); + assert_eq!(out.matches("--basert-lib").count(), 1, "{out}"); + assert!( + out.contains(&format!("--basert-lib {}", f.lib.display())), + "{out}" + ); + let probes = f.probes(); + assert!( + probes.contains(&format!("PROBE: BASERT_LIB={}", f.lib.display())), + "{probes}" + ); +} + +#[test] +fn a_bundle_that_cannot_be_written_gets_superfluid_in_the_users_data_directory() { + use std::os::unix::fs::PermissionsExt; + let f = fixture(); + let bundle = f.tmp.path().join("app"); + fs::create_dir(&bundle).unwrap(); + fs::copy(env!("CARGO_BIN_EXE_basert"), bundle.join("basert")).unwrap(); + fs::write(bundle.join("libbaseRT.dylib"), b"").unwrap(); + fs::write(bundle.join("libbaseRT.so"), b"").unwrap(); + let home = f.tmp.path().join("home"); + fs::create_dir(&home).unwrap(); + fs::set_permissions(&bundle, fs::Permissions::from_mode(0o555)).unwrap(); + let basert = || { + let mut c = Command::new(bundle.join("basert")); + c.env("BASERT_SUPERFLUID_SOURCE", &f.assets) + .env("BASERT_STATE_DIR", &f.state) + .env("HOME", &home) + .env("XDG_DATA_HOME", home.join(".local").join("share")) + .env_remove("BASERT_SUPERFLUID_DIR") + .env_remove("BASERT_LIB") + .env_remove("BASERT_SUPERFLUID_AUTO_INSTALL") + .stdin(Stdio::null()); + c + }; + let (code, out, err) = run(basert().args(["serve", f.model()])); + fs::set_permissions(&bundle, fs::Permissions::from_mode(0o755)).unwrap(); + assert_eq!(code, 0, "{out}{err}"); + assert!( + !bundle.join("superfluid").exists(), + "nothing written into the bundle" + ); + // Under the user's data directory, wherever the platform keeps it. + let installed = err + .lines() + .find_map(|l| l.strip_prefix("basert serve: installed superfluid ")) + .unwrap_or_else(|| panic!("{err}")); + let (_, at) = installed.split_once(" at ").unwrap(); + assert!(at.starts_with(home.to_str().unwrap()), "{at}"); + assert!(at.ends_with("baseRT/superfluid"), "{at}"); + assert!(Path::new(at).join("bin").join("superfluid").is_file()); + assert!(out.contains("ARGS: serve"), "{out}"); + // The next start finds it there and installs nothing. + let (code, out, err) = run(basert().args(["serve", f.model()])); + assert_eq!(code, 0, "{out}{err}"); + assert!(!err.contains("installing superfluid"), "{err}"); + assert!(out.contains("ARGS: serve"), "{out}"); +} + +#[test] +fn an_installed_copy_that_cannot_serve_the_engine_is_brought_to_the_pin() { + let f = fixture(); + f.install_old("0.0.1"); + let pin = pinned(&f); + let unready = |c: &mut Command| { + c.env( + "FAKE_SUPERFLUID_UNREADY", + "libbaseRT 0.4.0 states no ABI number and is not a 0.3 library", + ) + .env("FAKE_SUPERFLUID_UNREADY_VERSION", "0.0.1"); + }; + let mut status = f.basert(); + unready(&mut status); + let (code, out, _) = run(status.args(["superfluid", "status"])); + assert_eq!(code, 2, "{out}"); + assert!(out.contains("installed: v0.0.1 ("), "{out}"); + + let mut serve = f.basert(); + unready(&mut serve); + let (code, out, err) = run(serve.args(["serve", f.model()])); + assert_eq!(code, 0, "{out}{err}"); + assert!( + err.contains(&format!( + "superfluid v0.0.1 at {} cannot serve this engine (libbaseRT 0.4.0 states no ABI number and is not a 0.3 library); {pin} can", + f.install.display() + )), + "{err}" + ); + assert!( + err.contains(&format!("installing superfluid {pin} into")), + "{err}" + ); + assert_eq!(f.stamp(), pin); + assert!(out.contains("ARGS: serve"), "{out}"); + + // A release the probe accepts is kept as it is, pinned or not. + let g = fixture(); + g.install_old("0.0.1"); + let (code, out, err) = run(g.basert().args(["serve", g.model()])); + assert_eq!(code, 0, "{out}{err}"); + assert!(!err.contains("installing superfluid"), "{err}"); + assert_eq!(g.stamp(), "v0.0.1"); + assert!(out.contains("ARGS: serve"), "{out}"); +} + +#[test] +fn the_pin_that_cannot_serve_the_engine_is_an_error_and_no_download() { + let f = fixture(); + assert_eq!(run(f.basert().args(["superfluid", "install"])).0, 0); + let pin = pinned(&f); + let (code, out, err) = run(f + .basert() + .env( + "FAKE_SUPERFLUID_UNREADY", + "libbaseRT states ABI 2, this build serves ABI 1", + ) + .env("FAKE_SUPERFLUID_UNREADY_VERSION", "9.9.9") + .args(["serve", f.model()])); + assert_eq!(code, 1, "{out}{err}"); + assert!( + err.contains(&format!( + "superfluid {pin} at {} cannot serve this engine: libbaseRT states ABI 2, this build serves ABI 1", + f.install.display() + )), + "{err}" + ); + assert!(!err.contains("downloading"), "{err}"); + assert!(!out.contains("ARGS:"), "{out}"); +} + +#[test] +fn a_failed_install_of_the_pin_is_an_error_that_names_the_cause() { + let f = fixture(); + f.install_old("0.0.1"); + let nowhere = f.state.join("no-release"); + fs::create_dir_all(&nowhere).unwrap(); + let (code, out, err) = run(f + .basert() + .env("FAKE_SUPERFLUID_UNREADY", "an engine of another minor") + .env("FAKE_SUPERFLUID_UNREADY_VERSION", "0.0.1") + .env("BASERT_SUPERFLUID_SOURCE", &nowhere) + .args(["serve", f.model()])); + assert_eq!(code, 1, "{out}{err}"); + assert!( + err.contains("cannot serve this engine (an engine of another minor)"), + "{err}" + ); + assert!(err.contains("copying") && err.contains(".tar.gz"), "{err}"); + assert!(!out.contains("ARGS:"), "{out}"); + assert_eq!(f.stamp(), "v0.0.1", "the copy that was there is kept"); +} + +#[test] +fn serve_help_version_and_a_missing_model() { + let f = fixture(); + let (code, out, _) = run(f.basert().args(["serve", "--help"])); + assert_eq!(code, 0); + assert!(out.starts_with("Usage: basert serve"), "{out}"); + assert!(out.contains("basert superfluid install"), "{out}"); + assert!(out.contains("BASERT_SUPERFLUID_AUTO_INSTALL"), "{out}"); + + let (code, out, _) = run(f.basert().arg("serve")); + assert_eq!(code, 1); + assert!(out.contains("Usage: basert serve"), "{out}"); + + let (code, out, _) = run(f.basert().args(["serve", "--version"])); + assert_eq!(code, 0); + assert!(out.starts_with("basert serve "), "{out}"); + assert!(out.contains("superfluid"), "{out}"); + + let (code, _, err) = run(f + .basert() + .args(["serve", "m.base", "--runtime", "llamacpp"])); + assert_eq!(code, 1); + assert!(err.contains("--runtime is not accepted"), "{err}"); + + assert_eq!(run(f.basert().args(["superfluid", "install"])).0, 0); + let missing = f.state.join("nowhere.base"); + let (code, _, err) = run(f.basert().args(["serve", missing.to_str().unwrap()])); + assert_eq!(code, 1); + assert!(err.contains("Failed to load model"), "{err}"); +} diff --git a/base-convert/crates/base-quant/src/profile.rs b/base-convert/crates/base-quant/src/profile.rs index 3046766..c1287b4 100644 --- a/base-convert/crates/base-quant/src/profile.rs +++ b/base-convert/crates/base-quant/src/profile.rs @@ -506,6 +506,45 @@ mod tests { assert!(matches!(router.dtype, TensorDtype::F16 | TensorDtype::Bf16)); } + /// Gemma-4 q4mix keeps the attention key projection at q8 (q / v / o + /// stay q4). Gemma-4 attends with scale 1.0 over RMS-normed q and k, + /// and every later query reads a position's cached key, so 4-bit keys + /// blunt the heads that point back at an earlier occurrence of a token: + /// the 26B-A4B garbles identifiers it is copying (`y_position_m` in a + /// script it is writing). 8-bit keys alone restore it. + #[test] + fn gemma4_moe_q4mix_keeps_k_proj_q8() { + let manifest = std::env::var("CARGO_MANIFEST_DIR").unwrap(); + let path = std::path::Path::new(&manifest) + .parent() + .unwrap() + .parent() + .unwrap() + .join("profiles") + .join("gemma4-moe-q4mix.json"); + let p = QuantProfile::from_path(&path).unwrap(); + for name in [ + "model.language_model.layers.0.self_attn.k_proj.weight", + "layers.5.self_attn.k_proj.weight", + "blk.3.attn_k.weight", + ] { + let r = p.resolve(name).unwrap(); + assert_eq!(r.dtype, TensorDtype::BaseQ8, "{name}"); + assert_eq!(r.group_size, 64, "{name}"); + assert_eq!(r.scale_dtype, ScaleDtype::Bf16, "{name}"); + } + for name in [ + "model.language_model.layers.0.self_attn.q_proj.weight", + "model.language_model.layers.0.self_attn.v_proj.weight", + "model.language_model.layers.0.self_attn.o_proj.weight", + "model.vision_tower.encoder.layers.0.self_attn.k_proj.linear.weight", + ] { + let r = p.resolve(name).unwrap(); + assert_eq!(r.dtype, TensorDtype::BaseQ4, "{name}"); + assert_eq!(r.group_size, 64, "{name}"); + } + } + /// q3-aggressive profile routes MLP / experts to q3 / gs=32 per spec. #[test] fn q3_aggressive_profile_uses_q3_gs32() { diff --git a/base-convert/crates/base-readers/src/gguf/dequant.rs b/base-convert/crates/base-readers/src/gguf/dequant.rs index b2814c1..2a5c6ba 100644 --- a/base-convert/crates/base-readers/src/gguf/dequant.rs +++ b/base-convert/crates/base-readers/src/gguf/dequant.rs @@ -101,6 +101,11 @@ impl GgmlType { Q5K => (256, 176), Q6K => (256, 210), Q8K => (256, 292), + Iq4Xs => (256, 136), // fp16 d + 2-byte scales_h + 4-byte scales_l + 128 bytes + Iq2Xs => (256, 74), // fp16 d + 32 x u16 grid/sign indices + 8 scale bytes + Iq2S => (256, 82), // fp16 d + 32 grid + 32 sign + 8 qh + 8 scale bytes + Iq3Xxs => (256, 98), // fp16 d + 64 grid indices + 32 scale/sign bytes + Iq3S => (256, 110), // fp16 d + 64 grid + 8 qh + 32 sign + 4 scale bytes other => panic!("block_geometry not defined for {other:?}"), } } @@ -159,6 +164,7 @@ pub fn dequant_to_f32(info: &TensorInfo, bytes: &[u8]) -> Result> { GgmlType::Q4K => dequant_q4_k(bytes, n), GgmlType::Q5K => dequant_q5_k(bytes, n), GgmlType::Q6K => dequant_q6_k(bytes, n), + GgmlType::Iq4Xs => dequant_iq4_xs(bytes, n), other => bail!( "dequant_to_f32: {:?} not yet implemented (tensor {:?})", other, @@ -640,6 +646,49 @@ fn dequant_q5_k(bytes: &[u8], n: usize) -> Result> { Ok(out) } +const IQ4_XS_BLOCK: usize = 136; // 2 (d) + 2 (scales_h) + 4 (scales_l) + 128 (qs) +const KVALUES_IQ4NL: [i8; 16] = [ + -127, -104, -83, -65, -49, -35, -22, -10, 1, 13, 25, 38, 53, 69, 89, 113, +]; + +/// ggml dequantize_row_iq4_xs: value = d * (ls - 32) * kvalues_iq4nl[q], ls the +/// sub-block's 6-bit scale, q the low nibble of qs[16*ib + j] for j < 16 and +/// the high nibble of qs[16*ib + j - 16] for j >= 16. +fn dequant_iq4_xs(bytes: &[u8], n: usize) -> Result> { + if n % QK_K != 0 { + bail!("IQ4_XS element count {n} not a multiple of {QK_K}"); + } + let n_blocks = n / QK_K; + if bytes.len() != n_blocks * IQ4_XS_BLOCK { + bail!( + "IQ4_XS byte length mismatch: got {}, expected {}", + bytes.len(), + n_blocks * IQ4_XS_BLOCK + ); + } + let mut out = Vec::with_capacity(n); + for b in 0..n_blocks { + let base = b * IQ4_XS_BLOCK; + let d = f16::from_le_bytes([bytes[base], bytes[base + 1]]).to_f32(); + let scales_h = u16::from_le_bytes([bytes[base + 2], bytes[base + 3]]); + let scales_l = &bytes[base + 4..base + 8]; + let qs = &bytes[base + 8..base + IQ4_XS_BLOCK]; + for ib in 0..8usize { + let ls = ((scales_l[ib / 2] >> (4 * (ib % 2))) & 0xF) as i32 + | ((((scales_h >> (2 * ib)) & 3) as i32) << 4); + let dl = d * (ls - 32) as f32; + let q = &qs[16 * ib..16 * ib + 16]; + for j in 0..16 { + out.push(dl * KVALUES_IQ4NL[(q[j] & 0xF) as usize] as f32); + } + for j in 0..16 { + out.push(dl * KVALUES_IQ4NL[(q[j] >> 4) as usize] as f32); + } + } + } + Ok(out) +} + const Q6_K_BLOCK: usize = 210; // 128 (ql) + 64 (qh) + 16 (scales) + 2 (d) fn dequant_q6_k(bytes: &[u8], n: usize) -> Result> { @@ -748,4 +797,35 @@ mod tests { assert_eq!(out[16], 0.0); assert_eq!(out[17], -8.0); } + + /// The grid i-quants are copied verbatim, never dequantized here, but the + /// parser sizes their tensors from these block geometries (ggml-common.h). + #[test] + fn grid_iquant_block_geometry_matches_ggml() { + assert_eq!(GgmlType::Iq2Xs.block_geometry(), (256, 74)); + assert_eq!(GgmlType::Iq2S.block_geometry(), (256, 82)); + assert_eq!(GgmlType::Iq3Xxs.block_geometry(), (256, 98)); + assert_eq!(GgmlType::Iq3S.block_geometry(), (256, 110)); + } + + /// IQ4_XS: d * (ls - 32) * kvalue, low nibbles first then high nibbles of + /// each sub-block's 16 bytes, ls split across scales_l and scales_h. + #[test] + fn iq4_xs_dequant_matches_ggml_layout() { + let mut block = vec![0u8; 136]; + block[0..2].copy_from_slice(&half::f16::from_f32(0.5).to_le_bytes()); + // sub-block 0: ls = 34 (low nibble 2, high bits 2); sub-block 1: ls = 31 (15, 1). + block[4] = 2 | (15 << 4); + block[2] = 2 | (1 << 2); + for j in 0..16 { + block[8 + j] = 8 | (15 << 4); // values 1 (low) and 113 (high) + block[8 + 16 + j] = 7 << 4; // values -127 (low) and -10 (high) + } + let out = dequant_iq4_xs(&block, 256).unwrap(); + assert_eq!(out[0], 0.5 * 2.0 * 1.0); + assert_eq!(out[16], 0.5 * 2.0 * 113.0); + assert_eq!(out[32], -0.5 * -127.0); // d * (31 - 32) * kvalue + assert_eq!(out[48], -0.5 * -10.0); + assert_eq!(out[64], 0.5 * -32.0 * -127.0); // ls = 0 for the zeroed sub-blocks + } } diff --git a/bindings/node/package.json b/bindings/node/package.json index 45f4f1c..17b523d 100644 --- a/bindings/node/package.json +++ b/bindings/node/package.json @@ -1,6 +1,6 @@ { "name": "@baseRT/node", - "version": "0.3.0", + "version": "0.4.0", "private": true, "description": "Node.js bindings for BaseRT — LLM inference engine for Apple Silicon (Metal)", "main": "dist/index.js", diff --git a/bindings/python/baseRT/__init__.py b/bindings/python/baseRT/__init__.py index 0825091..ec0d049 100644 --- a/bindings/python/baseRT/__init__.py +++ b/bindings/python/baseRT/__init__.py @@ -21,7 +21,7 @@ from pathlib import Path from typing import Any, Callable, Dict, Iterator, List, Optional, Tuple, Union -__version__ = "0.3.0" +__version__ = "0.4.0" # --------------------------------------------------------------------------- # Library loading diff --git a/bindings/python/setup.py b/bindings/python/setup.py index dceb288..1e26078 100644 --- a/bindings/python/setup.py +++ b/bindings/python/setup.py @@ -6,7 +6,7 @@ setup( name="baseRT", - version="0.3.0", + version="0.4.0", description="Python bindings for the BaseRT LLM inference engine (Apple Silicon / Metal)", long_description=long_description, long_description_content_type="text/markdown", diff --git a/bindings/rust/baseRT-sys/Cargo.toml b/bindings/rust/baseRT-sys/Cargo.toml index 72e0944..47c1fde 100644 --- a/bindings/rust/baseRT-sys/Cargo.toml +++ b/bindings/rust/baseRT-sys/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "baseRT-sys" -version = "0.3.0" +version = "0.4.0" edition = "2021" description = "Raw FFI bindings for the BaseRT LLM inference engine" license = "Apache-2.0" diff --git a/bindings/rust/baseRT-sys/src/lib.rs b/bindings/rust/baseRT-sys/src/lib.rs index 7bd6f1b..b082f71 100644 --- a/bindings/rust/baseRT-sys/src/lib.rs +++ b/bindings/rust/baseRT-sys/src/lib.rs @@ -315,6 +315,11 @@ extern "C" { pub fn baseRT_get_config(model: baseRT_model_t) -> BaseRTModelConfig; pub fn baseRT_model_config_sizeof() -> usize; + /// The library's BASERT_ABI_VERSION (since 0.3.1). + pub fn baseRT_abi_version() -> std::os::raw::c_uint; + /// The configuration copied into a caller-sized buffer, zero-filled past + /// the library's struct; returns the library's sizeof (since 0.4.0). + pub fn baseRT_get_config_into(model: baseRT_model_t, out: *mut c_void, out_size: usize) -> usize; pub fn baseRT_model_memory(model: baseRT_model_t) -> usize; /// Device memory budget in bytes (Metal: recommended working set; CUDA: /// total device memory). 0 when no supported device is present. diff --git a/bindings/rust/baseRT/Cargo.toml b/bindings/rust/baseRT/Cargo.toml index 5121901..1b240f3 100644 --- a/bindings/rust/baseRT/Cargo.toml +++ b/bindings/rust/baseRT/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "baseRT" -version = "0.3.0" +version = "0.4.0" edition = "2021" description = "Safe Rust bindings for the BaseRT LLM inference engine (Apple Silicon)" license = "Apache-2.0" diff --git a/bindings/swift/Sources/CBaseRT/include/baseRT.h b/bindings/swift/Sources/CBaseRT/include/baseRT.h index 04823d8..9768a55 100644 --- a/bindings/swift/Sources/CBaseRT/include/baseRT.h +++ b/bindings/swift/Sources/CBaseRT/include/baseRT.h @@ -63,7 +63,7 @@ extern "C" { // === Versioning === #define BASERT_VERSION_MAJOR 0 -#define BASERT_VERSION_MINOR 3 +#define BASERT_VERSION_MINOR 4 #define BASERT_VERSION_PATCH 0 /// Compile-time version, packed as `(MAJOR<<16) | (MINOR<<8) | PATCH`. @@ -75,6 +75,22 @@ extern "C" { /// `.dylib` than it was compiled against. const char *baseRT_version_string(void); +/// The engine's C ABI number. Two libraries with the same number can be +/// driven through the same bindings: the same functions with the same +/// signatures, struct layouts and enum values. It changes only when one of +/// those changes incompatibly, whatever the version number does; a patch +/// release never changes it, and a release may add functions, and append +/// fields to BaseRTModelConfig, without changing it (a loader reads that +/// struct with baseRT_get_config_into(), by size). A binding that opens +/// libbaseRT at run time compares this with `baseRT_abi_version()` rather +/// than with the version string. +#define BASERT_ABI_VERSION 1 + +/// Runtime-resolved ABI number of the loaded library (its +/// BASERT_ABI_VERSION). Since 0.3.1: a library without this function is +/// 0.3.0 or older, and its ABI is told by its major.minor alone. +unsigned int baseRT_abi_version(void); + /// Opaque model handle. typedef void *baseRT_model_t; @@ -275,9 +291,23 @@ void baseRT_free_model(baseRT_model_t model); // === Model info === -/// Get model configuration. +/// Get model configuration, by value. A caller that opened the library at +/// run time should prefer baseRT_get_config_into(): the struct grows by +/// appended fields across releases, and a by-value return into a caller +/// built for another size is undefined. BaseRTModelConfig baseRT_get_config(baseRT_model_t model); +/// Copy the model configuration into a caller-sized buffer: the first +/// `out_size` bytes of the library's BaseRTModelConfig, the whole struct +/// followed by zeros when `out_size` is larger. Returns the library's +/// sizeof(BaseRTModelConfig), so the caller can tell what it was given. +/// Fields are only ever appended to the struct, so a caller built against +/// an older or newer header reads the fields it knows and sees zeros for +/// the rest, which every field treats as "not applicable". This is how a +/// loader stays on one ABI number while the struct grows. A null model +/// fills `out` with zeros. Since 0.4.0. +size_t baseRT_get_config_into(baseRT_model_t model, void *out, size_t out_size); + /// sizeof(BaseRTModelConfig) as compiled into the library. Language /// bindings that mirror the struct by hand (Python/Rust/Node) compare /// this against their mirror's size at load time so layout drift fails diff --git a/include/baseRT/baseRT.h b/include/baseRT/baseRT.h index 04823d8..9768a55 100644 --- a/include/baseRT/baseRT.h +++ b/include/baseRT/baseRT.h @@ -63,7 +63,7 @@ extern "C" { // === Versioning === #define BASERT_VERSION_MAJOR 0 -#define BASERT_VERSION_MINOR 3 +#define BASERT_VERSION_MINOR 4 #define BASERT_VERSION_PATCH 0 /// Compile-time version, packed as `(MAJOR<<16) | (MINOR<<8) | PATCH`. @@ -75,6 +75,22 @@ extern "C" { /// `.dylib` than it was compiled against. const char *baseRT_version_string(void); +/// The engine's C ABI number. Two libraries with the same number can be +/// driven through the same bindings: the same functions with the same +/// signatures, struct layouts and enum values. It changes only when one of +/// those changes incompatibly, whatever the version number does; a patch +/// release never changes it, and a release may add functions, and append +/// fields to BaseRTModelConfig, without changing it (a loader reads that +/// struct with baseRT_get_config_into(), by size). A binding that opens +/// libbaseRT at run time compares this with `baseRT_abi_version()` rather +/// than with the version string. +#define BASERT_ABI_VERSION 1 + +/// Runtime-resolved ABI number of the loaded library (its +/// BASERT_ABI_VERSION). Since 0.3.1: a library without this function is +/// 0.3.0 or older, and its ABI is told by its major.minor alone. +unsigned int baseRT_abi_version(void); + /// Opaque model handle. typedef void *baseRT_model_t; @@ -275,9 +291,23 @@ void baseRT_free_model(baseRT_model_t model); // === Model info === -/// Get model configuration. +/// Get model configuration, by value. A caller that opened the library at +/// run time should prefer baseRT_get_config_into(): the struct grows by +/// appended fields across releases, and a by-value return into a caller +/// built for another size is undefined. BaseRTModelConfig baseRT_get_config(baseRT_model_t model); +/// Copy the model configuration into a caller-sized buffer: the first +/// `out_size` bytes of the library's BaseRTModelConfig, the whole struct +/// followed by zeros when `out_size` is larger. Returns the library's +/// sizeof(BaseRTModelConfig), so the caller can tell what it was given. +/// Fields are only ever appended to the struct, so a caller built against +/// an older or newer header reads the fields it knows and sees zeros for +/// the rest, which every field treats as "not applicable". This is how a +/// loader stays on one ABI number while the struct grows. A null model +/// fills `out` with zeros. Since 0.4.0. +size_t baseRT_get_config_into(baseRT_model_t model, void *out, size_t out_size); + /// sizeof(BaseRTModelConfig) as compiled into the library. Language /// bindings that mirror the struct by hand (Python/Rust/Node) compare /// this against their mirror's size at load time so layout drift fails From 90946776b4d78edd9b3f9e31d50c12d8888d7358 Mon Sep 17 00:00:00 2001 From: prabod Date: Thu, 22 Oct 2026 18:40:31 +1100 Subject: [PATCH 2/2] Gemma 4 q4mix profile keeps the attention key projection at 8 bits --- base-convert/profiles/gemma4-moe-q4mix.json | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/base-convert/profiles/gemma4-moe-q4mix.json b/base-convert/profiles/gemma4-moe-q4mix.json index 9ee7633..763da7d 100644 --- a/base-convert/profiles/gemma4-moe-q4mix.json +++ b/base-convert/profiles/gemma4-moe-q4mix.json @@ -50,10 +50,11 @@ {"pattern": "**.ffn_gate_exps.scale", "dtype": "f16"}, {"pattern": "**.ffn_gate_up_exps.scale", "dtype": "f16"}, + {"pattern": "**.self_attn.k_proj.weight", "dtype": "base_q8", "scale_dtype": "bf16", "group_size": 64}, {"pattern": "**.{q,k,v,o}_proj.weight", "dtype": "base_q4", "scale_dtype": "bf16", "group_size": 64}, {"pattern": "**.{q,k,v,o}_proj.linear.weight", "dtype": "base_q4", "scale_dtype": "bf16", "group_size": 64}, {"pattern": "**.attn_q.weight", "dtype": "base_q4", "scale_dtype": "bf16", "group_size": 64}, - {"pattern": "**.attn_k.weight", "dtype": "base_q4", "scale_dtype": "bf16", "group_size": 64}, + {"pattern": "**.attn_k.weight", "dtype": "base_q8", "scale_dtype": "bf16", "group_size": 64}, {"pattern": "**.attn_v.weight", "dtype": "base_q4", "scale_dtype": "bf16", "group_size": 64}, {"pattern": "**.attn_output.weight", "dtype": "base_q4", "scale_dtype": "bf16", "group_size": 64},