From 4290abf51df7244b57ca93404c2487b7f7e0d988 Mon Sep 17 00:00:00 2001 From: baseRT sync Date: Fri, 23 Oct 2026 18:57:14 +1100 Subject: [PATCH] sync: open-core source for 0.4.1 Generated by tools/sync_public.py from the internal tree at 2ed117b13. Only allow-listed paths are included; tuned quant profiles and internal tooling are excluded by rule. --- base-convert/Cargo.lock | 16 +- base-convert/Cargo.toml | 2 +- base-convert/crates/base-convert/src/hub.rs | 1005 ++++++++++++++++- base-convert/crates/base-convert/src/main.rs | 7 + base-convert/crates/base-convert/src/serve.rs | 24 +- .../crates/base-convert/src/superfluid.rs | 446 ++++++-- .../base-convert/tests/serve_launcher.rs | 134 ++- base-convert/crates/base-hub/src/cache.rs | 5 + base-convert/crates/base-hub/src/registry.rs | 55 +- bindings/node/package.json | 2 +- bindings/python/baseRT/__init__.py | 2 +- bindings/python/setup.py | 2 +- bindings/rust/baseRT-sys/Cargo.toml | 2 +- bindings/rust/baseRT/Cargo.toml | 2 +- .../swift/Sources/CBaseRT/include/baseRT.h | 2 +- include/baseRT/baseRT.h | 2 +- 16 files changed, 1571 insertions(+), 137 deletions(-) diff --git a/base-convert/Cargo.lock b/base-convert/Cargo.lock index 812f821..d7a4d57 100644 --- a/base-convert/Cargo.lock +++ b/base-convert/Cargo.lock @@ -151,7 +151,7 @@ dependencies = [ [[package]] name = "base-arch" -version = "0.4.0" +version = "0.4.1" dependencies = [ "anyhow", "base-format", @@ -162,7 +162,7 @@ dependencies = [ [[package]] name = "base-awq" -version = "0.4.0" +version = "0.4.1" dependencies = [ "anyhow", "base-format", @@ -176,7 +176,7 @@ dependencies = [ [[package]] name = "base-convert" -version = "0.4.0" +version = "0.4.1" dependencies = [ "anyhow", "base-arch", @@ -201,7 +201,7 @@ dependencies = [ [[package]] name = "base-format" -version = "0.4.0" +version = "0.4.1" dependencies = [ "anyhow", "bitflags", @@ -217,7 +217,7 @@ dependencies = [ [[package]] name = "base-hub" -version = "0.4.0" +version = "0.4.1" dependencies = [ "anyhow", "base-format", @@ -236,7 +236,7 @@ dependencies = [ [[package]] name = "base-quant" -version = "0.4.0" +version = "0.4.1" dependencies = [ "anyhow", "base-format", @@ -248,7 +248,7 @@ dependencies = [ [[package]] name = "base-readers" -version = "0.4.0" +version = "0.4.1" dependencies = [ "anyhow", "base-format", @@ -262,7 +262,7 @@ dependencies = [ [[package]] name = "base-sign" -version = "0.4.0" +version = "0.4.1" dependencies = [ "anyhow", "base-format", diff --git a/base-convert/Cargo.toml b/base-convert/Cargo.toml index 3d75485..87f7ffc 100644 --- a/base-convert/Cargo.toml +++ b/base-convert/Cargo.toml @@ -12,7 +12,7 @@ members = [ ] [workspace.package] -version = "0.4.0" +version = "0.4.1" edition = "2021" license = "Apache-2.0" repository = "https://github.com/basecompute/baseRT" diff --git a/base-convert/crates/base-convert/src/hub.rs b/base-convert/crates/base-convert/src/hub.rs index f94e52e..999d2ff 100644 --- a/base-convert/crates/base-convert/src/hub.rs +++ b/base-convert/crates/base-convert/src/hub.rs @@ -9,9 +9,12 @@ use crate::{AwqMode, ConvertArgs, ListArgs, PullArgs, TargetScheme}; use anyhow::{bail, Context, Result}; use base_hub::cache::{self, HubSidecar}; +use base_hub::catalog::CatalogEntry; use base_hub::fetch::{self, Fetcher, HfFetcher}; use base_hub::parts::{self, Artifact, Grouped}; -use base_hub::registry::{MergedRegistry, ModelEntry, ModelRef, Registry, SourceKind}; +use base_hub::registry::{ + CatalogRegistry, MergedRegistry, ModelEntry, ModelRef, Registry, SourceKind, +}; use std::path::{Path, PathBuf}; use std::process::Command; @@ -142,11 +145,111 @@ fn select_base_file<'a>(files: &'a Grouped, want: &str) -> Result<&'a Artifact> ) .collect(); bail!( - "no pre-converted .base for quant {want:?} in this repo; it offers: {}", - avail.join(", ") + "no pre-converted .base for quant {want:?} in this repo; it offers: {} \ + (or take one by name with --file: {})", + avail.join(", "), + file_names(files) ) } +/// Whether `name`, as given to `--file`, names the repo file `f`: by its +/// whole path when `name` has a directory, else by its file name; the +/// `.base` extension is optional either way. +fn names_file(name: &str, f: &str) -> bool { + let bare = |s: &str| s.strip_suffix(".base").unwrap_or(s).to_string(); + if name.contains('/') { + bare(name) == bare(f) + } else { + base_file_stem(f) == bare(name) + } +} + +/// Pick the `.base` that `name` names (see [`names_file`]). A file name +/// shared by several directories must be given with its path; a malformed +/// part set by that name is an error saying why; no match lists the names +/// there are. +fn select_named_base_file<'a>(files: &'a Grouped, name: &str) -> Result<&'a Artifact> { + let hits: Vec<&Artifact> = files + .artifacts + .iter() + .filter(|f| names_file(name, &f.name)) + .collect(); + let broken: Vec<&(String, String)> = files + .malformed + .iter() + .filter(|(n, _)| names_file(name, n)) + .collect(); + match (hits.as_slice(), broken.as_slice()) { + ([one], []) => Ok(one), + ([], [(n, why)]) => bail!("{n}: this publication is incomplete ({why})"), + ([], []) => bail!( + "no .base named {name:?} in this repo; it has: {}", + file_names(files) + ), + _ => bail!( + "{name:?} names several files in this repo ({}); give its path", + hits.iter() + .map(|f| f.name.clone()) + .chain(broken.iter().map(|(n, _)| format!("{n} (incomplete)"))) + .collect::>() + .join(", ") + ), + } +} + +fn file_names(files: &Grouped) -> String { + files + .artifacts + .iter() + .map(|f| f.name.clone()) + .chain( + files + .malformed + .iter() + .map(|(n, _)| format!("{n} (incomplete)")), + ) + .collect::>() + .join(", ") +} + +/// The variant a `.base` picked by quant installs under: the artifact's own +/// quant when it carries one (so a `-Q8.base` never lands in a `default-q4` +/// dir); otherwise what was requested. +fn base_variant(file: &str, want: &str) -> String { + let file_tag = quant_tag(&base_file_stem(file)); + if is_quant_tag(&file_tag) { + format!("default-{file_tag}") + } else { + format!("default-{want}") + } +} + +/// Variants of files taken by `--file` that no catalog row publishes start +/// with this, so none is ever a catalog row's or a quant's: the resolver +/// reaches one only by its exact name. +pub(crate) const NAMED_VARIANT_PREFIX: &str = "file-"; + +/// The variant a `--file` pull installs a file the catalog does not publish +/// under: the prefix, its path in the repo made readable (`Foo-Medium.base` +/// → `file-foo-medium-…`) and cut to 64 bytes so the directory name stays +/// well inside a filesystem's 255-byte limit, and the first 8 hex digits of the +/// path's sha256, so paths the readable part folds together (`a/model.base`, +/// `a-model.base`, `A-Model.base`, two long paths sharing a prefix) still get +/// variants of their own. +fn named_variant(file: &str) -> String { + use sha2::{Digest, Sha256}; + let path = file.strip_suffix(".base").unwrap_or(file); + let mut readable = path.replace('/', "-").to_ascii_lowercase(); + let mut end = readable.len().min(64); + while !readable.is_char_boundary(end) { + end -= 1; + } + readable.truncate(end); + let digest = Sha256::digest(file.as_bytes()); + let short: String = digest[..4].iter().map(|b| format!("{b:02x}")).collect(); + format!("{NAMED_VARIANT_PREFIX}{readable}-{short}") +} + /// Decide whether `token` should be treated as a hub model id rather than a /// filesystem path. Hub ids are HuggingFace-style `namespace/name` (optionally /// `:variant`); anything that already exists on disk, names a `.base` file, or @@ -280,6 +383,8 @@ fn installed_best_variant(reg: &MergedRegistry, id: &str, want: &str) -> Option< .filter(|r| { r.id == id && runnable(&r.variant) + // a file taken by name answers only to its exact variant + && !r.variant.starts_with(NAMED_VARIANT_PREFIX) && base_hub::registry::quant_bits(&r.variant).unwrap_or(r.variant.as_str()) == want_bits }) @@ -316,6 +421,7 @@ fn auto_pull_and_resolve( profile: None, target, revision: "main".to_string(), + file: None, force: false, dry_run: false, }) @@ -569,6 +675,10 @@ pub fn cmd_pull(args: PullArgs) -> Result<()> { let reg = MergedRegistry::load()?; let root = reg.root.clone(); let want = quant_token(&args); + if let Some(file) = &args.file { + let fetcher = HfFetcher::new(cache::hf_staging_dir(&root))?; + return pull_named_file(®, &args, file, &fetcher); + } let r = reg.resolve(&args.id, &args.revision, Some(&want), args.force)?; if args.dry_run { @@ -608,7 +718,17 @@ pub fn cmd_pull(args: PullArgs) -> Result<()> { } else { let fetcher = HfFetcher::new(cache::hf_staging_dir(&root))?; let base_files = list_base_files(&fetcher, hf_repo, revision)?; - pull_base_direct(&root, &args, id, hf_repo, revision, &fetcher, &base_files) + pull_base_direct( + &root, + &args, + id, + hf_repo, + revision, + &fetcher, + &base_files, + None, + None, + ) } } // A raw HF repo: if it already hosts pre-converted `.base` artifacts @@ -623,12 +743,205 @@ pub fn cmd_pull(args: PullArgs) -> Result<()> { if base_files.artifacts.is_empty() && base_files.malformed.is_empty() { pull_and_convert(&root, &args, id, repo, revision) } else { - pull_base_direct(&root, &args, id, repo, revision, &fetcher, &base_files) + pull_base_direct( + &root, + &args, + id, + repo, + revision, + &fetcher, + &base_files, + None, + None, + ) } } } } +/// `pull --file`: download the named `.base` from the id's repo (the catalog's +/// repo for a catalog id, else the id itself) into its own variant. +fn pull_named_file( + reg: &MergedRegistry, + args: &PullArgs, + file: &str, + fetcher: &dyn Fetcher, +) -> Result<()> { + // A file a catalog row of the id publishes is that row's: taken from its + // repo and revision, refused when it is for another backend, and + // installed under its variant (`cuda-q4mix`), as a catalog pull of it + // would be. Any other file comes from the id's repo (its catalog repo, + // else the id itself), whatever quant or backend the catalog lists. + let rows: Vec<&CatalogEntry> = reg + .catalog + .rows(&args.id) + .filter(|e| names_file(file, &e.file)) + .collect(); + let listed = match rows.as_slice() { + [] => None, + [one] => Some(*one), + many => bail!( + "{file:?} names several files the catalog lists for {} ({}); give its path", + args.id, + many.iter() + .map(|e| format!("{} in {}", e.file, e.hf_repo)) + .collect::>() + .join(", ") + ), + }; + if let Some(e) = listed.filter(|e| !CatalogRegistry::backend_ok(e)) { + bail!( + "{} is the {} bundle of {}; this client runs {}", + e.file, + e.backend.as_deref().unwrap_or_default(), + e.id, + CatalogRegistry::client_backend() + ); + } + // Where to look: the listing row's repo; else every repo the id's rows + // keep files in (a name found in more than one must be given its row); + // else the id itself as a Hub repo. + let id = listed + .or_else(|| reg.catalog.entry(&args.id)) + .map_or_else(|| args.id.clone(), |e| e.id.clone()); + let mut sources: Vec<(String, String)> = Vec::new(); + let rows = listed + .into_iter() + .chain(reg.catalog.rows(&args.id).filter(|_| listed.is_none())); + for e in rows { + let at = (e.hf_repo.clone(), e.revision.clone()); + if !sources.contains(&at) { + sources.push(at); + } + } + if sources.is_empty() { + if args.id.split('/').filter(|s| !s.is_empty()).count() < 2 { + bail!( + "unknown model id {:?}: not in the catalog and not an `org/model` HF repo", + args.id + ); + } + sources.push((args.id.clone(), args.revision.clone())); + } + // The name is resolved against the repo every time, cached or not, so a + // cache hit answers exactly what a fresh pull would take. + let mut found = Vec::new(); + for (repo, revision) in &sources { + let base_files = list_base_files(fetcher, repo, revision)?; + let names = |n: &str| names_file(file, n); + if base_files.artifacts.iter().any(|f| names(&f.name)) + || base_files.malformed.iter().any(|(n, _)| names(n)) + { + found.push((repo.clone(), revision.clone(), base_files)); + } else if sources.len() == 1 { + // one place to look: its own error lists what is there + select_named_base_file(&base_files, file)?; + } + } + let (repo, revision, base_files) = match found.len() { + 1 => found.remove(0), + 0 => bail!( + "no .base named {file:?} in {}", + sources + .iter() + .map(|(r, v)| format!("{r}@{v}")) + .collect::>() + .join(", ") + ), + _ => bail!( + "{file:?} is in several repos of {id} ({}); pull it from its repo's id", + found + .iter() + .map(|(r, v, _)| format!("{r}@{v}")) + .collect::>() + .join(", ") + ), + }; + let artifact = select_named_base_file(&base_files, file)?; + if let Some(e) = listed.filter(|e| e.file != artifact.name) { + bail!( + "{file:?} names {} in the repo, but the catalog of {id} lists {}", + artifact.name, + e.file + ); + } + let variant = listed.map_or_else(|| named_variant(&artifact.name), |e| e.quant.clone()); + + if let Some(path) = reg + .local + .installed_path(&id, &variant) + .filter(|_| !args.force) + { + // Only this file's own install answers for it: one that records it, + // or (for the catalog's file) a catalog pull of it from before files + // were recorded. Anything else there is refused, not handed back. + let vdir = cache::variant_dir(®.root, &id, &variant)?; + let sidecar = cache::read_sidecar(&vdir)?; + let held = sidecar.as_ref().and_then(|s| { + s.file.clone().or_else(|| { + // a catalog pull from before files were recorded is the row's + // file only if its bytes are the row's + listed + .filter(|e| { + s.source_kind == "catalog" + && matches!((&s.base_sha256, &e.sha256), (Some(got), Some(want)) if got.eq_ignore_ascii_case(want)) + }) + .map(|e| e.file.clone()) + }) + }); + match held { + Some(held) if held == artifact.name => {} + Some(held) => bail!( + "{id} [{variant}] holds {held}, not {} (--force replaces it)", + artifact.name + ), + None => bail!( + "{id} [{variant}] holds {}, not {} (--force replaces it)", + match sidecar.and_then(|s| s.profile) { + Some(_) => "a conversion", + None => "an install from another source", + }, + artifact.name + ), + } + if args.dry_run { + println!( + "plan: {id} [{variant}] already installed at {}", + path.display() + ); + return Ok(()); + } + // a catalog pull from before files were recorded is now known as this one + if let Some(mut s) = cache::read_sidecar(&vdir)?.filter(|s| s.file.is_none()) { + s.file = Some(artifact.name.clone()); + cache::write_sidecar(&vdir, &s)?; + } + eprintln!( + "{id} [{variant}] already installed: {}\n (use --force to re-pull)", + path.display() + ); + return Ok(()); + } + if args.dry_run { + println!( + "plan: pull {} from {repo}@{revision} → {id} [{variant}] (no conversion)", + artifact.name + ); + return Ok(()); + } + pull_base_direct( + ®.root, + args, + &id, + &repo, + &revision, + fetcher, + &base_files, + listed, + Some(&variant), + ) +} + fn print_plan(r: &ModelRef, want: &str) { match r { ModelRef::Local { id, variant, path } => { @@ -723,7 +1036,9 @@ fn pull_catalog(root: &Path, r: &ModelRef) -> Result<()> { variant, None, Some(got_sha), + Some(file.as_str()), )?; + count_download(&fetcher, hf_repo, revision); fetch::cleanup_staging(&fetcher, hf_repo); eprintln!("installed {id} [{variant}] → {}", out.display()); Ok(()) @@ -743,23 +1058,25 @@ fn pull_base_direct( revision: &str, fetcher: &dyn Fetcher, base_files: &Grouped, + listed: Option<&CatalogEntry>, + variant: Option<&str>, ) -> Result<()> { eprintln!("basert pull v{}", env!("CARGO_PKG_VERSION")); eprintln!(" source: {repo}@{revision} (HuggingFace, pre-converted .base)"); let want = quant_token(args); - let artifact = select_base_file(base_files, &want)?; + let artifact = match &args.file { + Some(name) => select_named_base_file(base_files, name)?, + None => select_base_file(base_files, &want)?, + }; let file = &artifact.name; - // Label the on-disk variant by the artifact's own quant when it carries - // one (so a `-Q8.base` never lands in a `default-q4` dir); otherwise fall - // back to what was requested. - let file_tag = quant_tag(&base_file_stem(file)); - let variant_tag = if is_quant_tag(&file_tag) { - file_tag - } else { - want + // The catalog's own file keeps the catalog's variant and checksums, + // however it was asked for. + let listed = listed.filter(|e| e.file == *file); + let variant = match variant { + Some(v) => v.to_string(), + None => listed.map_or_else(|| base_variant(file, &want), |e| e.quant.clone()), }; - let variant = format!("default-{variant_tag}"); eprintln!(" variant: {variant}"); eprintln!(" file: {file}"); @@ -771,25 +1088,54 @@ fn pull_base_direct( // Moves the staged download into place (same filesystem), so the pulled // artifact exists exactly once on disk. A part set is reassembled beside // `out` and renamed into place once complete. - parts::install(fetcher, repo, revision, artifact, &out, None)?; + let parts_sha256 = listed.and_then(|e| e.parts_sha256.as_deref()); + parts::install(fetcher, repo, revision, artifact, &out, parts_sha256)?; let sha = crate::compute_sha256_streaming(&out).ok(); + if let Some(expected) = listed.and_then(|e| e.sha256.as_deref()) { + let got = sha.as_deref().unwrap_or("nothing readable"); + if !got.eq_ignore_ascii_case(expected) { + std::fs::remove_file(&out).ok(); + fetch::cleanup_staging(fetcher, repo); + bail!("sha256 mismatch for {id} ({file}): expected {expected}, got {got}"); + } + eprintln!(" sha256: verified"); + } write_sidecar_for( &vdir, id, - "huggingface", + if listed.is_some() { + "catalog" + } else { + "huggingface" + }, repo, None, revision, &variant, None, sha, + Some(file.as_str()), )?; + count_download(fetcher, repo, revision); fetch::cleanup_staging(fetcher, repo); eprintln!("installed {id} [{variant}] → {}", out.display()); Ok(()) } +/// Hugging Face counts a repo's downloads by requests for its `config.json`. +/// Fetched for basecompute's own repos when one is published; a failure does +/// not fail the pull. +fn count_download(fetcher: &dyn Fetcher, repo: &str, revision: &str) { + if repo.starts_with("basecompute/") + && fetcher + .list_files(repo, revision) + .is_ok_and(|files| files.iter().any(|f| f == "config.json")) + { + let _ = fetcher.get_file(repo, revision, "config.json"); + } +} + /// One line about a split bundle, so the user knows why the install step /// runs on after the download bar finishes and what the disk needs. fn announce_parts(artifact: &Artifact, out: &Path) { @@ -878,6 +1224,7 @@ fn pull_and_convert( &variant, Some(&variant), sha, + None, )?; // The staged source snapshot is now a redundant multi-GB copy of a model // we keep in converted form (provenance lives in hub.json; a re-convert @@ -1064,6 +1411,7 @@ fn write_sidecar_for( variant: &str, profile: Option<&str>, base_sha256: Option, + file: Option<&str>, ) -> Result<()> { cache::write_sidecar( vdir, @@ -1077,6 +1425,7 @@ fn write_sidecar_for( profile: profile.map(|s| s.to_string()), pulled_at: crate::chrono_now(), base_sha256, + file: file.map(|s| s.to_string()), }, ) } @@ -1444,6 +1793,7 @@ mod tests { profile: None, target: TargetScheme::BaseQ4, revision: "main".into(), + file: None, force: false, dry_run: false, }; @@ -1455,6 +1805,8 @@ mod tests { "main", &fetcher, &base_files, + None, + None, ) .unwrap(); @@ -1471,6 +1823,29 @@ mod tests { assert!(stray.is_empty(), "leftovers: {stray:?}"); } + #[test] + fn config_json_is_requested_only_from_a_basecompute_repo_that_has_one() { + struct Recording(std::cell::RefCell>, Vec<&'static str>); + impl Fetcher for Recording { + fn get_file(&self, _repo: &str, _revision: &str, filename: &str) -> Result { + self.0.borrow_mut().push(filename.to_string()); + Ok(PathBuf::from(filename)) + } + fn list_files(&self, _repo: &str, _revision: &str) -> Result> { + Ok(self.1.iter().map(|f| f.to_string()).collect()) + } + } + let with = Recording(Default::default(), vec!["m-Q4.base", "config.json"]); + count_download(&with, "basecompute/m", "main"); + assert_eq!(*with.0.borrow(), ["config.json"]); + let without = Recording(Default::default(), vec!["m-Q4.base"]); + count_download(&without, "basecompute/m", "main"); + assert!(without.0.borrow().is_empty()); + let elsewhere = Recording(Default::default(), vec!["m-Q4.base", "config.json"]); + count_download(&elsewhere, "someone/m", "main"); + assert!(elsewhere.0.borrow().is_empty()); + } + #[test] fn pull_base_direct_installs_requested_quant() { let tmp = tempfile::tempdir().unwrap(); @@ -1488,6 +1863,7 @@ mod tests { profile: None, target: TargetScheme::BaseQ8, revision: "main".into(), + file: None, force: false, dry_run: false, }; @@ -1499,6 +1875,8 @@ mod tests { "main", &fetcher, &base_files, + None, + None, ) .unwrap(); @@ -1512,6 +1890,595 @@ mod tests { assert!(repo_dir.join("m-Q8.base").exists(), "fixture must survive"); } + #[test] + fn select_named_base_file_matches_by_name_and_lists_the_rest() { + let files = Grouped { + artifacts: vec![ + Artifact::whole("m-Small.base"), + Artifact::whole("sub/m-Large.base"), + ], + malformed: vec![( + "m-Medium.base".into(), + "part set is missing part 001".into(), + )], + }; + assert_eq!( + select_named_base_file(&files, "m-Small.base").unwrap().name, + "m-Small.base" + ); + assert_eq!( + select_named_base_file(&files, "m-Large").unwrap().name, + "sub/m-Large.base" + ); + let err = select_named_base_file(&files, "m-Medium.base") + .unwrap_err() + .to_string(); + assert!( + err.contains("incomplete") && err.contains("missing part 001"), + "{err}" + ); + let err = select_named_base_file(&files, "m-Huge.base") + .unwrap_err() + .to_string(); + assert!( + err.contains("m-Small.base") && err.contains("sub/m-Large.base"), + "{err}" + ); + } + + #[test] + fn variants_of_quant_picked_and_named_files() { + assert_eq!(base_variant("m-Q8.base", "q4"), "default-q8"); + assert_eq!(base_variant("model.base", "q4"), "default-q4"); + let medium = named_variant("m-Medium.base"); + assert!( + medium.starts_with("file-m-medium-") && medium.len() == "file-m-medium-".len() + 8, + "{medium}" + ); + // paths the readable part folds together keep variants of their own + let folded = ["a/model.base", "a-model.base", "A-Model.base"].map(named_variant); + assert!( + folded.iter().all(|v| v.starts_with("file-a-model-")), + "{folded:?}" + ); + for wide in ["é", "𝔸"] { + let long = format!("{}/model.base", wide.repeat(200)); + let v = named_variant(&long); + assert!(v.len() <= "file-".len() + 64 + 9, "{} bytes", v.len()); + assert_ne!( + v, + named_variant(&format!("{}/other.base", wide.repeat(200))) + ); + } + assert!( + folded[0] != folded[1] && folded[1] != folded[2] && folded[0] != folded[2], + "{folded:?}" + ); + } + + #[test] + fn a_path_picks_among_files_that_share_a_name() { + let files = Grouped { + artifacts: vec![ + Artifact::whole("a/model.base"), + Artifact::whole("b/model.base"), + ], + malformed: vec![], + }; + assert_eq!( + select_named_base_file(&files, "b/model.base").unwrap().name, + "b/model.base" + ); + assert_eq!( + select_named_base_file(&files, "a/model").unwrap().name, + "a/model.base" + ); + let err = select_named_base_file(&files, "model.base") + .unwrap_err() + .to_string(); + assert!( + err.contains("names several files") && err.contains("b/model.base"), + "{err}" + ); + assert!(select_named_base_file(&files, "c/model.base").is_err()); + + // An incomplete upload by that name counts toward the ambiguity. + let files = Grouped { + artifacts: vec![Artifact::whole("a/model.base")], + malformed: vec![("b/model.base".into(), "part set is missing part 001".into())], + }; + let err = select_named_base_file(&files, "model.base") + .unwrap_err() + .to_string(); + assert!( + err.contains("names several files") && err.contains("b/model.base (incomplete)"), + "{err}" + ); + let err = select_named_base_file(&files, "b/model.base") + .unwrap_err() + .to_string(); + assert!( + err.contains("incomplete") && err.contains("missing part 001"), + "{err}" + ); + assert_eq!( + select_named_base_file(&files, "a/model.base").unwrap().name, + "a/model.base" + ); + } + + fn named(file: &str, dry_run: bool, id: &str) -> PullArgs { + PullArgs { + id: id.into(), + profile: None, + target: TargetScheme::BaseQ4, + revision: "main".into(), + file: Some(file.into()), + force: false, + dry_run, + } + } + + fn pull_named(reg: &MergedRegistry, fetcher: &dyn Fetcher, args: &PullArgs) -> Result<()> { + pull_named_file(reg, args, args.file.as_deref().unwrap(), fetcher) + } + + fn catalog(json_models: &str) -> base_hub::registry::CatalogRegistry { + base_hub::registry::CatalogRegistry::from_catalog( + base_hub::catalog::Catalog::from_json(&format!( + r#"{{"schema":1,"models":[{json_models}]}}"# + )) + .unwrap(), + ) + } + + /// A flat fixture repo `MockFetcher` serves from `root`. + fn fixture_repo(root: &Path, repo: &str, files: &[(&str, &[u8])]) { + let dir = root.join(repo); + std::fs::create_dir_all(&dir).unwrap(); + for (f, bytes) in files { + std::fs::write(dir.join(f), bytes).unwrap(); + } + } + + /// A repo listing and nothing to download: what the cache checks see. + struct Listing(Vec<&'static str>); + + impl Fetcher for Listing { + fn get_file(&self, _: &str, _: &str, f: &str) -> Result { + bail!("no download expected ({f})") + } + fn list_files(&self, _: &str, _: &str) -> Result> { + Ok(self.0.iter().map(|f| f.to_string()).collect()) + } + } + + /// An installed variant as an earlier pull (of any kind) would leave it. + fn install( + reg: &MergedRegistry, + id: &str, + variant: &str, + file: Option<&str>, + kind: &str, + profile: Option<&str>, + ) { + let vdir = cache::variant_dir(®.root, id, variant).unwrap(); + std::fs::create_dir_all(&vdir).unwrap(); + std::fs::write(cache::base_artifact_path(&vdir), b"x").unwrap(); + write_sidecar_for( + &vdir, id, kind, id, None, "main", variant, profile, None, file, + ) + .unwrap(); + } + + fn held(reg: &MergedRegistry, id: &str, variant: &str) -> Option { + let vdir = cache::variant_dir(®.root, id, variant).unwrap(); + cache::read_sidecar(&vdir).unwrap().unwrap().file + } + + /// A repo whose `.base` files are named by size, not quant (the BaseQuant + /// releases): no quant picks one, `--file` does, each into a variant of + /// its own that records the file, which `basert list` reports. + #[test] + fn a_size_named_repo_installs_each_file_under_its_own_variant() { + let tmp = tempfile::tempdir().unwrap(); + fixture_repo( + tmp.path(), + "basecompute/m", + &[ + ("m-Small.base", b"small"), + ("m-Medium.base", b"medium"), + ("m-Medium.gguf", b"gguf"), + ], + ); + let fetcher = MockFetcher::new(tmp.path()); + let base_files = list_base_files(&fetcher, "basecompute/m", "main").unwrap(); + let err = select_base_file(&base_files, "q4").unwrap_err().to_string(); + assert!( + err.contains("--file") && err.contains("m-Medium.base"), + "{err}" + ); + + let reg = MergedRegistry::new(tmp.path().join("cache"), catalog("")); + pull_named( + ®, + &fetcher, + &named("m-Medium.base", false, "basecompute/m"), + ) + .unwrap(); + pull_named(®, &fetcher, &named("m-Small", false, "basecompute/m")).unwrap(); + let m = reg.root.join("basecompute/m"); + assert_eq!( + std::fs::read(m.join(named_variant("m-Medium.base")).join("model.base")).unwrap(), + b"medium" + ); + assert_eq!( + std::fs::read(m.join(named_variant("m-Small.base")).join("model.base")).unwrap(), + b"small" + ); + assert_eq!( + held(®, "basecompute/m", &named_variant("m-Medium.base")).as_deref(), + Some("m-Medium.base") + ); + let rows = reg.local.list().unwrap(); + let files: Vec<_> = rows.iter().filter_map(|r| r.file.as_deref()).collect(); + assert_eq!(files, ["m-Medium.base", "m-Small.base"]); + // and a quant never reaches one by its bits + assert!(installed_best_variant(®, "basecompute/m", "default-q4").is_none()); + } + + /// A cached file answers only for itself: the name is resolved against + /// the repo first, and whatever else is in the variant is refused. + #[test] + fn a_cached_file_answers_only_for_itself() { + let tmp = tempfile::tempdir().unwrap(); + fixture_repo(tmp.path(), "basecompute/m", &[("m-Medium.base", b"medium")]); + let fetcher = MockFetcher::new(tmp.path()); + let reg = MergedRegistry::new(tmp.path().join("cache"), catalog("")); + let medium = reg + .root + .join("basecompute/m") + .join(named_variant("m-Medium.base")) + .join("model.base"); + + pull_named( + ®, + &fetcher, + &named("m-Medium.base", true, "basecompute/m"), + ) + .unwrap(); + assert!(!medium.exists(), "a dry run installs nothing"); + pull_named( + ®, + &fetcher, + &named("m-Medium.base", false, "basecompute/m"), + ) + .unwrap(); + std::fs::write(&medium, b"cached").unwrap(); + for dry in [false, true] { + pull_named(®, &fetcher, &named("m-Medium", dry, "basecompute/m")).unwrap(); + } + assert_eq!( + std::fs::read(&medium).unwrap(), + b"cached", + "a cache hit downloads nothing" + ); + let err = pull_named( + ®, + &fetcher, + &named("Typo-Medium.base", true, "basecompute/m"), + ) + .unwrap_err(); + assert!(err.to_string().contains("no .base named"), "{err}"); + + for (file, profile, says) in [ + (Some("other.base"), None, "holds other.base"), + (None, Some("default-q4"), "holds a conversion"), + (None, None, "an install from another source"), + ] { + install( + ®, + "basecompute/m", + &named_variant("m-Medium.base"), + file, + "huggingface", + profile, + ); + for dry in [false, true] { + let err = pull_named( + ®, + &fetcher, + &named("m-Medium.base", dry, "basecompute/m"), + ) + .unwrap_err(); + assert!( + err.to_string().contains(says) && err.to_string().contains("--force"), + "{err}" + ); + } + } + let mut force = named("m-Medium.base", false, "basecompute/m"); + force.force = true; + pull_named(®, &fetcher, &force).unwrap(); + assert_eq!(std::fs::read(&medium).unwrap(), b"medium"); + } + + /// A bare name is resolved against the repo listing whether or not its + /// file is cached, so it is ambiguous exactly when a fresh pull says so; + /// a path picks among same-named files, which get variants of their own. + #[test] + fn a_bare_name_is_resolved_against_the_repo_even_when_cached() { + let tmp = tempfile::tempdir().unwrap(); + let reg = MergedRegistry::new(tmp.path(), catalog("")); + install( + ®, + "basecompute/m", + &named_variant("model.base"), + Some("model.base"), + "huggingface", + None, + ); + install( + ®, + "basecompute/m", + &named_variant("a/model.base"), + Some("a/model.base"), + "huggingface", + None, + ); + pull_named( + ®, + &Listing(vec!["model.base"]), + &named("model.base", false, "basecompute/m"), + ) + .unwrap(); + let grown = Listing(vec!["model.base", "a/model.base", "b/model.base"]); + for bare in ["model.base", "model"] { + let err = pull_named(®, &grown, &named(bare, false, "basecompute/m")).unwrap_err(); + assert!(err.to_string().contains("names several files"), "{err}"); + } + pull_named(®, &grown, &named("a/model.base", true, "basecompute/m")).unwrap(); + let err = + pull_named(®, &grown, &named("b/model.base", false, "basecompute/m")).unwrap_err(); + assert!( + err.to_string().contains("no download expected"), + "b/model is not cached: {err}" + ); + // an incomplete upload by that name counts toward the ambiguity too + let files = Grouped { + artifacts: vec![Artifact::whole("a/model.base")], + malformed: vec![("b/model.base".into(), "part set is missing part 001".into())], + }; + let err = select_named_base_file(&files, "model.base") + .unwrap_err() + .to_string(); + assert!( + err.contains("names several files") && err.contains("b/model.base (incomplete)"), + "{err}" + ); + } + + /// A file a catalog row publishes is that row's: from its repo, under its + /// variant, refused for another backend, answered by an older catalog + /// pull of it; a bare name two rows publish must be given with its path. + #[test] + fn a_catalog_file_is_its_rows() { + let here = CatalogRegistry::client_backend(); + let there = if here == "cuda" { "metal" } else { "cuda" }; + let tmp = tempfile::tempdir().unwrap(); + let mine = format!("cat-{here}-q4mix.base"); + let theirs = format!("cat-{there}-q4mix.base"); + fixture_repo( + tmp.path(), + "basecompute/cat-a", + &[ + ("cat-Q4.base", b"q4"), + (&mine, b"mine"), + (&theirs, b"theirs"), + ("Experimental-Q4.base", b"exp"), + ("Both.base", b"a"), + ], + ); + fixture_repo( + tmp.path(), + "basecompute/cat-b", + &[ + ("cat-Q8.base", b"q8"), + ("Only-B.base", b"b"), + ("Both.base", b"b"), + ], + ); + let row = |repo: &str, quant: &str, file: &str, backend: &str| { + format!( + r#"{{"id":"basecompute/cat","hf_repo":"basecompute/{repo}","quant":"{quant}","file":"{file}"{backend}}}"# + ) + }; + let reg = MergedRegistry::new( + tmp.path().join("cache"), + catalog( + &[ + row("cat-a", "default-q4", "cat-Q4.base", r#","sha256":"112f2dfa31205df3f5f9db109460c8b85067d7dc27f70781c2e4ef903ee9f26a""#), + row("cat-b", "default-q8", "cat-Q8.base", ""), + row( + "cat-a", + &format!("{here}-q4mix"), + &mine, + &format!(r#","backend":"{here}""#), + ), + row( + "cat-a", + &format!("{there}-q4mix"), + &theirs, + &format!(r#","backend":"{there}""#), + ), + ] + .join(","), + ), + ); + let fetcher = MockFetcher::new(tmp.path()); + let cat = reg.root.join("basecompute/cat"); + + pull_named( + ®, + &fetcher, + &named("cat-Q8.base", false, "basecompute/cat"), + ) + .unwrap(); + assert_eq!( + std::fs::read(cat.join("default-q8/model.base")).unwrap(), + b"q8", + "from its own row's repo" + ); + pull_named(®, &fetcher, &named(&mine, false, "basecompute/cat")).unwrap(); + assert_eq!( + std::fs::read(cat.join(format!("{here}-q4mix/model.base"))).unwrap(), + b"mine" + ); + let err = + pull_named(®, &fetcher, &named(&theirs, false, "basecompute/cat")).unwrap_err(); + assert!( + err.to_string().contains(&format!("the {there} bundle")), + "{err}" + ); + assert!(!cat.join(format!("{there}-q4mix")).exists()); + + // an unlisted file never takes a catalog variant, now or later + pull_named( + ®, + &fetcher, + &named("Experimental-Q4.base", false, "basecompute/cat"), + ) + .unwrap(); + assert!( + cat.join(named_variant("Experimental-Q4.base")) + .join("model.base") + .exists() + && !cat.join("default-q4").exists() + ); + + // an unlisted file is looked for in every repo the id's rows use + pull_named( + ®, + &fetcher, + &named("Only-B.base", false, "basecompute/cat"), + ) + .unwrap(); + let only_b = cat.join(named_variant("Only-B.base")).join("model.base"); + assert_eq!(std::fs::read(only_b).unwrap(), b"b"); + let err = + pull_named(®, &fetcher, &named("Both.base", true, "basecompute/cat")).unwrap_err(); + assert!( + err.to_string().contains("several repos") && err.to_string().contains("cat-b"), + "{err}" + ); + + // a catalog pull from before files were recorded is its row's file + // only when its bytes are the row's + let legacy = |sha: &str| { + let vdir = cache::variant_dir(®.root, "basecompute/cat", "default-q4").unwrap(); + std::fs::create_dir_all(&vdir).unwrap(); + std::fs::write(cache::base_artifact_path(&vdir), b"q4").unwrap(); + let id = "basecompute/cat"; + write_sidecar_for( + &vdir, + id, + "catalog", + id, + None, + "main", + "default-q4", + None, + Some(sha.into()), + None, + ) + .unwrap(); + }; + legacy(&"0".repeat(64)); + let err = pull_named( + ®, + &fetcher, + &named("cat-Q4.base", false, "basecompute/cat"), + ) + .unwrap_err(); + assert!(err.to_string().contains("--force"), "{err}"); + legacy("112f2dfa31205df3f5f9db109460c8b85067d7dc27f70781c2e4ef903ee9f26a"); + pull_named( + ®, + &fetcher, + &named("cat-Q4.base", false, "basecompute/cat"), + ) + .unwrap(); + assert_eq!( + held(®, "basecompute/cat", "default-q4").as_deref(), + Some("cat-Q4.base") + ); + + let two = MergedRegistry::new( + tmp.path().join("two"), + catalog( + &[ + row("cat-a", "default-q4", "a/model.base", ""), + row("cat-b", "default-q8", "b/model.base", ""), + ] + .join(","), + ), + ); + let err = pull_named( + &two, + &Listing(vec!["a/model.base"]), + &named("model.base", true, "basecompute/cat"), + ) + .unwrap_err(); + assert!( + err.to_string() + .contains("names several files the catalog lists"), + "{err}" + ); + } + + /// The catalog's own file is held to the catalog's sha256 when named. + #[test] + fn a_named_catalog_file_is_checked_against_the_catalog() { + let tmp = tempfile::tempdir().unwrap(); + fixture_repo( + tmp.path(), + "basecompute/cat-files", + &[("cat-Medium.base", b"medium")], + ); + let fetcher = MockFetcher::new(tmp.path()); + let row = |sha: &str| { + catalog(&format!( + r#"{{"id":"basecompute/cat","hf_repo":"basecompute/cat-files","quant":"default-medium","file":"cat-Medium.base","sha256":"{sha}"}}"# + )) + }; + let out = |reg: &MergedRegistry| reg.root.join("basecompute/cat/default-medium/model.base"); + + let bad = MergedRegistry::new(tmp.path().join("bad"), row(&"0".repeat(64))); + let err = pull_named( + &bad, + &fetcher, + &named("cat-Medium.base", false, "basecompute/cat"), + ) + .unwrap_err(); + assert!(err.to_string().contains("sha256 mismatch"), "{err}"); + assert!( + !out(&bad).exists(), + "a mismatched download is not installed" + ); + + let good = MergedRegistry::new( + tmp.path().join("good"), + row("c082456a7766e23a18db084cd34b6ff510baef506548b897cc80e9b7d3e121c8"), + ); + pull_named( + &good, + &fetcher, + &named("cat-Medium.base", false, "basecompute/cat"), + ) + .unwrap(); + assert_eq!(std::fs::read(out(&good)).unwrap(), b"medium"); + } + /// Fetcher owning an hf-hub-style staging tree (blob + snapshot symlink), /// mirroring what `HfFetcher` leaves on disk mid-pull. struct StagedFetcher { @@ -1576,6 +2543,7 @@ mod tests { profile: None, target: TargetScheme::BaseQ4, revision: "main".into(), + file: None, force: false, dry_run: false, }; @@ -1587,6 +2555,8 @@ mod tests { "main", &fetcher, &base_files, + None, + None, ) .unwrap(); @@ -1942,6 +2912,7 @@ mod tests { profile: None, target: t, revision: "main".into(), + file: None, force: false, dry_run: false, }; diff --git a/base-convert/crates/base-convert/src/main.rs b/base-convert/crates/base-convert/src/main.rs index d506915..bd61773 100644 --- a/base-convert/crates/base-convert/src/main.rs +++ b/base-convert/crates/base-convert/src/main.rs @@ -382,6 +382,13 @@ struct PullArgs { /// HF revision / branch / tag. #[arg(long, default_value = "main")] revision: String, + /// Take this `.base` file from the repo (e.g. `Foo-Medium.base`) rather + /// than picking one by quant — for repos whose `.base` files are not + /// named by quant. A file the catalog lists installs under its catalog + /// variant; any other under one of its own (`file-foo-medium-`), + /// which `basert list --json` shows beside the file. + #[arg(long)] + file: Option, /// Re-download / re-convert even if already cached. #[arg(long)] force: bool, diff --git a/base-convert/crates/base-convert/src/serve.rs b/base-convert/crates/base-convert/src/serve.rs index f93342c..deadf53 100644 --- a/base-convert/crates/base-convert/src/serve.rs +++ b/base-convert/crates/base-convert/src/serve.rs @@ -1,7 +1,7 @@ //! `basert serve`: the command line basert-serve has always had (the C++ //! server's, flag for flag), run on superfluid. The launcher reads the -//! command line, finds the engine library and the pinned superfluid release -//! beside itself (installing it if asked, see [`crate::superfluid`]), and +//! command line, finds the engine library and a superfluid beside itself +//! (installing the newest release if asked, see [`crate::superfluid`]), and //! execs `superfluid serve` with baseRT's models only, nothing fetched, and a //! request log that starts fresh with each start: the server basert-serve has //! always been. @@ -22,9 +22,10 @@ Usage: basert serve [--model ...] [options] The server is superfluid (https://github.com/basecompute/superfluid). basert runs the one installed beside it or in its own data directory, installing - the release it was tested with when there is none, or when the one found - cannot serve this engine: asked first on a terminal, without asking - otherwise, never with BASERT_SUPERFLUID_AUTO_INSTALL=0. + the newest release that serves this engine when there is none, or when the + one found cannot serve it (the release basert was tested with when GitHub + cannot be read): asked first on a terminal, without asking otherwise, never + with BASERT_SUPERFLUID_AUTO_INSTALL=0. `basert superfluid install` does it ahead of time, and `basert superfluid status` shows what is installed and whether it can serve. @@ -94,8 +95,10 @@ State (the request log, uploaded files) is kept under $BASERT_STATE_DIR, by default the serve/ directory beside the model cache, one directory per port. The engine library is found beside basert (or in ../lib); BASERT_LIB names another directory. BASERT_SUPERFLUID_DIR names where superfluid is installed; -BASERT_SUPERFLUID_SOURCE a directory holding its release tarball, read instead -of GitHub; BASERT_SUPERFLUID_AUTO_INSTALL=0 forbids a download at a start. +BASERT_SUPERFLUID_RELEASES another release listing (a URL or a file); +BASERT_SUPERFLUID_SOURCE a directory holding the tested release's tarball, +read instead of GitHub; BASERT_SUPERFLUID_AUTO_INSTALL=0 forbids a download +at a start. "; /// Flags every start hands superfluid; one given on the command line too is @@ -432,7 +435,7 @@ pub fn run(rest: &[String]) -> Result<()> { i.binary().display() ), None => println!( - "superfluid {} pinned, not installed (`basert superfluid install`)", + "superfluid not installed (`basert superfluid install` installs the newest release; tested with {})", superfluid::RELEASE ), } @@ -504,10 +507,7 @@ pub fn run(rest: &[String]) -> Result<()> { if o.print_command { let binary = match superfluid::Install::locate()? { Some(i) => i.binary().display().to_string(), - None => format!( - "superfluid (not installed; {} would be)", - superfluid::RELEASE - ), + None => "superfluid (not installed; the newest release would be)".to_string(), }; let mut words = vec![binary, "serve".to_string()]; words.extend(superfluid_args(&o, &sessions)); diff --git a/base-convert/crates/base-convert/src/superfluid.rs b/base-convert/crates/base-convert/src/superfluid.rs index c1eecdb..e3ca0ad 100644 --- a/base-convert/crates/base-convert/src/superfluid.rs +++ b/base-convert/crates/base-convert/src/superfluid.rs @@ -5,10 +5,12 @@ //! installs for itself at a start. Which superfluid may serve this engine is //! superfluid's own call: before a start the launcher asks each install it //! finds (`superfluid runtimes --json`, the engine library named) whether its -//! basert runtime is ready, and runs the first that is. One that is not, and -//! is not the pinned release, is brought to the pin. The pin ([`RELEASE`]) is -//! the release this basert was tested with; a newer one the probe accepts -//! runs as it is, and nothing is ever downgraded. +//! basert runtime is ready, and runs the first that is, whatever its release: +//! nothing is ever downgraded. When none is installed, or none can serve, the +//! newest release that states this engine's ABI number beside its tarballs +//! (`superfluid-basert-abi-.txt`) is installed, or the release this basert +//! was tested with ([`RELEASE`]) when the release listing cannot be read or +//! names none. //! //! Where superfluid is looked for, in order: `$BASERT_SUPERFLUID_DIR`; //! `superfluid/` beside basert, the one-shot installer's layout; a directory @@ -22,10 +24,10 @@ //! otherwise, since an older Local calls `basert serve` without knowing //! superfluid exists; `$BASERT_SUPERFLUID_AUTO_INSTALL=0` forbids it, and the //! command to run is named instead. `basert superfluid install` installs -//! ahead of time; `$BASERT_SUPERFLUID_SOURCE` names a directory holding the -//! release's tarball and checksum, read instead of GitHub. +//! ahead of time; `$BASERT_SUPERFLUID_RELEASES` names another release listing +//! (a mirror's, a file), and `$BASERT_SUPERFLUID_SOURCE` a directory holding +//! the tested release's tarball and checksum, read instead of GitHub. -use std::fmt; use std::fs; use std::io::{BufRead, IsTerminal, Write}; use std::path::{Path, PathBuf}; @@ -37,11 +39,20 @@ use clap::{Args, Subcommand}; use sha2::{Digest, Sha256}; /// The superfluid release this basert was tested with, a tag of -/// : installed when -/// none is found, or when the one found cannot serve this engine. +/// : installed instead +/// of the newest when the release listing cannot be read or names no release +/// for this engine, and the release a `$BASERT_SUPERFLUID_SOURCE` holds. pub const RELEASE: &str = "v0.1.3"; +/// `BASERT_ABI_VERSION` of the engine shipped beside this basert, which a +/// superfluid release names when it serves it (`superfluid-basert-abi-.txt`). +pub const ENGINE_ABI: u32 = 1; + const REPO: &str = "basecompute/superfluid"; +/// The releases the newest superfluid is chosen from, newest first. +const RELEASES: &str = "https://api.github.com/repos/basecompute/superfluid/releases?per_page=30"; +/// Another release listing in GitHub's form, a URL or a file: a mirror's, or the tests'. +pub const RELEASES_ENV: &str = "BASERT_SUPERFLUID_RELEASES"; /// Where superfluid is installed, instead of the places looked in by default. pub const DIR_ENV: &str = "BASERT_SUPERFLUID_DIR"; /// A directory holding the release's tarball and its `.sha256`, read instead @@ -64,44 +75,53 @@ pub struct SuperfluidArgs { #[derive(Subcommand, Debug)] pub enum Action { - /// Install the pinned superfluid release where `basert serve` looks for - /// it (nothing to do when it is already there). + /// Install the newest superfluid release that serves this engine where + /// `basert serve` looks for it (nothing to do when it is already there). Install { - /// Install again even when the pinned release is already there. + /// Install again even when that release, or a newer one, is there. #[arg(long)] force: bool, }, - /// Show the pinned release, what is installed where, and whether it can - /// serve the engine beside basert. Exits 0 when an installed superfluid - /// can, 1 when none is installed, 2 when the installed one cannot. + /// Show the release this basert was tested with, what is installed where, + /// and whether it can serve the engine beside basert. Exits 0 when an + /// installed superfluid can, 1 when none is installed, 2 when the + /// installed one cannot. Status, } pub fn cmd(args: SuperfluidArgs) -> Result<()> { match args.action { Action::Install { force } => { + let source = Source::locate(); + let release = wanted(&source, &mut |line| eprintln!("{line}")); + let tag = &release.tag; let dest = match Install::locate()? { - Some(i) if !force && i.installed_release().as_deref() == Some(RELEASE) => { - println!("superfluid {RELEASE} is installed at {}", i.root.display()); + Some(i) if !force && i.installed_release().as_deref() == Some(tag.as_str()) => { + println!("superfluid {tag} is installed at {}", i.root.display()); + return Ok(()); + } + Some(i) if !force && newer(i.installed_release().as_deref(), tag) => { + println!( + "superfluid {} is installed at {}, newer than {tag}: nothing is downgraded (--force installs {tag})", + i.installed_release().unwrap_or_default(), + i.root.display() + ); return Ok(()); } Some(i) if i.writable() => i, _ => Install::for_install()?, }; - dest.install(&Source::locate(), &mut |line| eprintln!("{line}"))?; - println!( - "superfluid {RELEASE} is installed at {}", - dest.root.display() - ); + dest.install(&source, &release, &mut |line| eprintln!("{line}"))?; + println!("superfluid {tag} is installed at {}", dest.root.display()); Ok(()) } Action::Status => { - println!("pinned: {RELEASE}"); + println!("tested: {RELEASE}"); let code = match Install::locate()? { None => { let dest = Install::for_install()?; println!( - "installed: nothing (`basert superfluid install` puts {RELEASE} at {})", + "installed: nothing (`basert superfluid install` puts the newest release at {})", dest.root.display() ); 1 @@ -302,11 +322,17 @@ impl Install { } } - /// Download the pinned release, check its SHA-256 against the one the - /// release publishes, unpack it beside the install and swap it in. An - /// install that is there stays until the new one is complete, and comes - /// back if the swap fails. - pub fn install(&self, source: &Source, say: &mut dyn FnMut(&str)) -> Result<()> { + /// Download `release`, check its SHA-256 against the one the release + /// publishes, unpack it beside the install and swap it in. An install + /// that is there stays until the new one is complete, and comes back if + /// the swap fails. + pub fn install( + &self, + source: &Source, + release: &Release, + say: &mut dyn FnMut(&str), + ) -> Result<()> { + let tag = &release.tag; let name = format!("superfluid-{}", target()?); let parent = self.parent(); fs::create_dir_all(&parent).with_context(|| format!("creating {}", parent.display()))?; @@ -319,10 +345,11 @@ impl Install { let checksum = stage.path().join(format!("{name}.tar.gz.sha256")); say(&format!( - "downloading superfluid {RELEASE} ({name}.tar.gz) from {source}" + "downloading superfluid {tag} ({name}.tar.gz) from {}", + source.describe(release, &format!("{name}.tar.gz")) )); - source.fetch(&format!("{name}.tar.gz"), &tarball)?; - source.fetch(&format!("{name}.tar.gz.sha256"), &checksum)?; + source.fetch(release, &format!("{name}.tar.gz"), &tarball)?; + source.fetch(release, &format!("{name}.tar.gz.sha256"), &checksum)?; let want = fs::read_to_string(&checksum) .with_context(|| format!("reading {}", checksum.display()))? .split_whitespace() @@ -353,7 +380,7 @@ impl Install { if !is_executable(&fresh.join("bin").join("superfluid")) { bail!("{name}.tar.gz holds no {name}/bin/superfluid"); } - fs::write(fresh.join(STAMP), format!("{RELEASE}\n"))?; + fs::write(fresh.join(STAMP), format!("{tag}\n"))?; let old = parent.join(format!(".superfluid-old-{}", std::process::id())); let had_old = self.root.exists(); @@ -376,7 +403,7 @@ impl Install { let _ = fs::remove_dir_all(&old); } say(&format!( - "installed superfluid {RELEASE} at {}", + "installed superfluid {tag} at {}", self.root.display() )); Ok(()) @@ -439,12 +466,44 @@ impl Probe { /// Where a release's files come from. pub enum Source { - /// The release's assets on GitHub. + /// The release's assets on GitHub, or where the listing that named it + /// says they are. GitHub, /// A directory holding the tarball and its `.sha256` ([`SOURCE_ENV`]). Dir(PathBuf), } +/// A superfluid release to install: its tag, and the URL of each asset its +/// listing gave, by name. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Release { + pub tag: String, + urls: Vec<(String, String)>, +} + +impl Release { + /// [`RELEASE`], on GitHub. + pub fn tested() -> Release { + Release { + tag: RELEASE.to_string(), + urls: Vec::new(), + } + } + + /// Where the asset `file` is: the URL the listing gave for it, else + /// GitHub's download URL for this tag. A mirror may keep a checksum + /// elsewhere than its tarball. + fn url(&self, file: &str) -> String { + match self.urls.iter().find(|(name, _)| name == file) { + Some((_, url)) => url.clone(), + None => format!( + "https://github.com/{REPO}/releases/download/{}/{file}", + self.tag + ), + } + } +} + impl Source { pub fn locate() -> Source { match std::env::var_os(SOURCE_ENV).filter(|d| !d.is_empty()) { @@ -453,7 +512,7 @@ impl Source { } } - fn fetch(&self, file: &str, dest: &Path) -> Result<()> { + fn fetch(&self, release: &Release, file: &str, dest: &Path) -> Result<()> { match self { Source::Dir(dir) => { let from = dir.join(file); @@ -462,17 +521,10 @@ impl Source { .map(|_| ()) } Source::GitHub => { - let url = format!("https://github.com/{REPO}/releases/download/{RELEASE}/{file}"); + let url = release.url(file); // GitHub answers an asset with a redirect to its storage, which - // the agent follows. A server that goes quiet fails the install - // rather than hanging a `basert serve` start. - let agent: ureq::Agent = ureq::Agent::config_builder() - .timeout_connect(Some(Duration::from_secs(30))) - .timeout_recv_response(Some(Duration::from_secs(60))) - .timeout_recv_body(Some(Duration::from_secs(600))) - .build() - .into(); - let response = agent + // the agent follows. + let response = agent(Duration::from_secs(600)) .get(&url) .call() .with_context(|| format!("downloading {url}"))?; @@ -486,15 +538,112 @@ impl Source { } } -impl fmt::Display for Source { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { +impl Source { + /// Where `file` of `release` comes from, for a message. + fn describe(&self, release: &Release, file: &str) -> String { match self { - Source::GitHub => write!(f, "https://github.com/{REPO}/releases/tag/{RELEASE}"), - Source::Dir(dir) => write!(f, "{}", dir.display()), + Source::GitHub => release.url(file), + Source::Dir(dir) => dir.display().to_string(), } } } +/// An HTTP agent whose server, gone quiet, fails the request rather than +/// hanging a `basert serve` start; a body may take up to `body`. +fn agent(body: Duration) -> ureq::Agent { + ureq::Agent::config_builder() + .timeout_connect(Some(Duration::from_secs(30))) + .timeout_recv_response(Some(Duration::from_secs(60))) + .timeout_recv_body(Some(body)) + .build() + .into() +} + +/// The release a start, or `basert superfluid install`, installs from +/// `source`: from GitHub the newest that serves this engine, else the one +/// this basert was tested with, saying why; from a directory, the one it holds. +pub fn wanted(source: &Source, say: &mut dyn FnMut(&str)) -> Release { + if let Source::Dir(_) = source { + return Release::tested(); + } + let found = target().and_then(|target| { + let listing = releases()?; + newest_in(&listing, target, ENGINE_ABI).with_context(|| { + format!("no superfluid release there serves engine ABI {ENGINE_ABI} on {target}") + }) + }); + found.unwrap_or_else(|why| { + say(&format!( + "the newest superfluid release could not be chosen ({why:#}); {RELEASE}, the release this basert was tested with, is installed" + )); + Release::tested() + }) +} + +/// superfluid's release listing, or the one `$BASERT_SUPERFLUID_RELEASES` names. +fn releases() -> Result { + let named = std::env::var(RELEASES_ENV).ok().filter(|v| !v.is_empty()); + let at = named.as_deref().unwrap_or(RELEASES); + let text = if at.starts_with("https://") || at.starts_with("http://") { + agent(Duration::from_secs(60)) + .get(at) + .header("Accept", "application/vnd.github+json") + .call() + .and_then(|r| r.into_body().read_to_string()) + .with_context(|| format!("reading the release listing {at}"))? + } else { + fs::read_to_string(at).with_context(|| format!("reading the release listing {at}"))? + }; + serde_json::from_str(&text).with_context(|| format!("the release listing {at}")) +} + +/// The newest release in a listing of GitHub's form that is neither a draft +/// nor a pre-release and publishes, for `target`, the tarball, its checksum +/// and the marker of engine ABI `abi`. +pub fn newest_in(listing: &serde_json::Value, target: &str, abi: u32) -> Option { + let tarball = format!("superfluid-{target}.tar.gz"); + let checksum = format!("{tarball}.sha256"); + let marker = format!("superfluid-basert-abi-{abi}.txt"); + listing + .as_array()? + .iter() + .filter(|r| !r["draft"].as_bool().unwrap_or(false)) + .filter(|r| !r["prerelease"].as_bool().unwrap_or(false)) + .filter_map(|r| { + let tag = r["tag_name"].as_str()?; + let assets = r["assets"].as_array()?; + let named = |name: &str| assets.iter().find(|a| a["name"] == name); + let url = |name: &str| -> Option<(String, String)> { + let at = named(name)?["browser_download_url"].as_str()?; + Some((name.to_string(), at.to_string())) + }; + named(&marker)?; + Some(( + release_key(tag)?, + Release { + tag: tag.to_string(), + urls: vec![url(&tarball)?, url(&checksum)?], + }, + )) + }) + .max_by(|a, b| a.0.cmp(&b.0)) + .map(|(_, release)| release) +} + +/// `v0.1.3` as `[0, 1, 3]`; `None` for a tag that is not a release number. +fn release_key(tag: &str) -> Option> { + let digits = tag.strip_prefix('v').unwrap_or(tag); + digits.split('.').map(|part| part.parse().ok()).collect() +} + +/// Whether the release `have` is newer than `than`. +fn newer(have: Option<&str>, than: &str) -> bool { + match (have.and_then(release_key), release_key(than)) { + (Some(have), Some(than)) => have > than, + _ => false, + } +} + /// Whether a start may download superfluid. #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum Download { @@ -539,69 +688,91 @@ pub fn ensure_with( home: &Path, say: &mut dyn FnMut(&str), ) -> Result { - // Every install found is asked; the first that can serve runs. An install - // that cannot and is the pin itself makes the engine the problem, which no - // download mends. - let mut unready: Vec = Vec::new(); + // Every install found is asked; the first that can serve runs. + let mut unready: Vec<(Install, Option, String)> = Vec::new(); for install in Install::candidates()? .into_iter() .filter(Install::is_installed) { - let why = match install.probe(engine, Some(home)).ready { + match install.probe(engine, Some(home)).ready { Ok(()) => return Ok(install.binary()), - Err(why) => why, - }; - let have = install.installed_release(); - if have.as_deref() == Some(RELEASE) { + Err(why) => { + let have = install.installed_release(); + unready.push((install, have, why)); + } + } + } + // Looked up only when it may be downloaded: a start forbidden to download + // does not wait on GitHub. + let release = match policy { + Download::Never => None, + Download::Ask | Download::Auto => Some(wanted(source, say)), + }; + let tag = release + .as_ref() + .map_or(RELEASE, |r| r.tag.as_str()) + .to_string(); + for (install, have, why) in &unready { + // An install that cannot serve and is the release itself makes the + // engine the problem, which no download mends. + if have.as_deref() == Some(tag.as_str()) { bail!( - "superfluid {RELEASE} at {} cannot serve this engine: {why}", + "superfluid {tag} at {} cannot serve this engine: {why}", install.root.display() ); } say(&format!( - "superfluid {} at {} cannot serve this engine ({why}); {RELEASE} can", + "superfluid {} at {} cannot serve this engine ({why}){}", have.as_deref().unwrap_or("of a release it does not say"), - install.root.display() + install.root.display(), + if release.is_some() { + format!("; {tag} can") + } else { + String::new() + } )); - unready.push(install); } - let dest = match unready.iter().find(|i| i.writable()) { + let dest = match unready.iter().map(|(i, _, _)| i).find(|i| i.writable()) { Some(found) => found.clone(), None => Install::for_install()?, }; let hint = format!( - "install it with\n basert superfluid install\n(superfluid {RELEASE} from \ - https://github.com/{REPO}/releases, into {})", + "install it with\n basert superfluid install\n(the newest superfluid release that serves \ + this engine, from https://github.com/{REPO}/releases, into {})", dest.root.display() ); - match policy { - Download::Never => { + let release = match (policy, release) { + (Download::Never, _) | (_, None) => { if unready.is_empty() { bail!("superfluid is not installed, and {AUTO_INSTALL_ENV}=0 forbids a download at a start; {hint}"); } bail!("{AUTO_INSTALL_ENV}=0 forbids a download at a start; {hint}"); } - Download::Ask => { + (Download::Ask, Some(release)) => { if unready.is_empty() { eprintln!("basert serve runs superfluid, which is not installed."); } if !confirm(&format!( - "Download and install superfluid {RELEASE} into {} now? [Y/n] ", + "Download and install superfluid {tag} into {} now? [Y/n] ", dest.root.display() ))? { - bail!("superfluid {RELEASE} is not installed; {hint}"); + bail!("superfluid {tag} is not installed; {hint}"); } + release } - Download::Auto => say(&format!( - "installing superfluid {RELEASE} into {} ({AUTO_INSTALL_ENV}=0 forbids this)", - dest.root.display() - )), - } - dest.install(source, say)?; + (Download::Auto, Some(release)) => { + say(&format!( + "installing superfluid {tag} into {} ({AUTO_INSTALL_ENV}=0 forbids this)", + dest.root.display() + )); + release + } + }; + dest.install(source, &release, say)?; match dest.probe(engine, Some(home)).ready { Ok(()) => Ok(dest.binary()), Err(why) => bail!( - "superfluid {RELEASE}, installed at {}, cannot serve this engine: {why}", + "superfluid {tag}, installed at {}, cannot serve this engine: {why}", dest.root.display() ), } @@ -713,7 +884,9 @@ mod tests { assert_eq!(install.installed_release(), None); let mut said = Vec::new(); install - .install(&Source::Dir(assets), &mut |l| said.push(l.to_string())) + .install(&Source::Dir(assets), &Release::tested(), &mut |l| { + said.push(l.to_string()) + }) .unwrap(); assert!(install.is_installed()); assert!(install @@ -739,10 +912,12 @@ mod tests { let good = release(&tmp.path().join("good"), "0.1.3", true); let bad = release(&tmp.path().join("bad"), "0.1.3", false); let install = install_at(tmp.path().join("superfluid")); - install.install(&Source::Dir(good), &mut |_| {}).unwrap(); + install + .install(&Source::Dir(good), &Release::tested(), &mut |_| {}) + .unwrap(); fs::write(install.root.join("marker"), "kept").unwrap(); let err = install - .install(&Source::Dir(bad), &mut |_| {}) + .install(&Source::Dir(bad), &Release::tested(), &mut |_| {}) .unwrap_err() .to_string(); assert!( @@ -761,7 +936,11 @@ mod tests { let tmp = tempfile::tempdir().unwrap(); let install = install_at(tmp.path().join("superfluid")); let err = install - .install(&Source::Dir(tmp.path().join("nowhere")), &mut |_| {}) + .install( + &Source::Dir(tmp.path().join("nowhere")), + &Release::tested(), + &mut |_| {}, + ) .unwrap_err() .to_string(); assert!(err.contains("copying") && err.contains(".tar.gz"), "{err}"); @@ -773,14 +952,18 @@ mod tests { let tmp = tempfile::tempdir().unwrap(); let assets = release(tmp.path(), "0.0.9", true); let install = install_at(tmp.path().join("superfluid")); - install.install(&Source::Dir(assets), &mut |_| {}).unwrap(); + install + .install(&Source::Dir(assets), &Release::tested(), &mut |_| {}) + .unwrap(); assert_eq!(install.installed_release().as_deref(), Some(RELEASE)); // A copy put here by hand has no stamp: the binary is asked. fs::remove_file(install.root.join(STAMP)).unwrap(); assert_eq!(install.installed_release().as_deref(), Some("v0.0.9")); // A second install replaces it, stamp included. let newer = release(&tmp.path().join("again"), "0.1.3", true); - install.install(&Source::Dir(newer), &mut |_| {}).unwrap(); + install + .install(&Source::Dir(newer), &Release::tested(), &mut |_| {}) + .unwrap(); assert_eq!(install.installed_release().as_deref(), Some(RELEASE)); assert!(!tmp .path() @@ -864,7 +1047,9 @@ mod tests { let tmp = tempfile::tempdir().unwrap(); let assets = release(tmp.path(), "0.1.3", true); let install = install_at(tmp.path().join("superfluid")); - install.install(&Source::Dir(assets), &mut |_| {}).unwrap(); + install + .install(&Source::Dir(assets), &Release::tested(), &mut |_| {}) + .unwrap(); let p = install.probe(Some(Path::new("/lib")), Some(Path::new("/home"))); assert_eq!( p, @@ -877,6 +1062,97 @@ mod tests { assert!(p.ready.unwrap_err().contains("could not be run")); } + /// A release as GitHub's listing has it, with assets of these names. + fn listed(tag: &str, names: &[String]) -> serde_json::Value { + let assets: Vec = names + .iter() + .map(|name| { + serde_json::json!({ + "name": name, + "browser_download_url": format!("https://github.com/{REPO}/releases/download/{tag}/{name}"), + }) + }) + .collect(); + serde_json::json!({"tag_name": tag, "draft": false, "prerelease": false, "assets": assets}) + } + + #[test] + fn the_newest_release_is_the_highest_that_serves_the_engine_here() { + let t = "aarch64-apple-darwin"; + let tarball = format!("superfluid-{t}.tar.gz"); + let checksum = format!("{tarball}.sha256"); + let marker = |abi: u32| format!("superfluid-basert-abi-{abi}.txt"); + let whole = + |tag: &str, abi: u32| listed(tag, &[tarball.clone(), checksum.clone(), marker(abi)]); + let mut draft = whole("v0.9.0", 1); + draft["draft"] = true.into(); + let mut pre = whole("v0.8.0", 1); + pre["prerelease"] = true.into(); + let listing = serde_json::json!([ + draft, + pre, + whole("v0.7.0", 2), + listed("v0.6.0", &[tarball.clone(), marker(1)]), + listed("v0.5.0", &[tarball.clone(), checksum.clone()]), + whole("nightly", 1), + whole("v0.1.9", 1), + whole("v0.1.10", 1), + ]); + let newest = newest_in(&listing, t, 1).unwrap(); + assert_eq!(newest.tag, "v0.1.10", "by number, not as text"); + assert_eq!( + newest.url(&tarball), + format!("https://github.com/{REPO}/releases/download/v0.1.10/{tarball}") + ); + // Each asset where the listing says it is, not beside the tarball. + let mut mirrored = whole("v0.2.0", 1); + mirrored["assets"][1]["browser_download_url"] = + format!("https://mirror/checksums/{checksum}").into(); + let release = newest_in(&serde_json::json!([mirrored]), t, 1).unwrap(); + assert_eq!( + release.url(&checksum), + format!("https://mirror/checksums/{checksum}") + ); + assert_eq!( + Release::tested().url(&checksum), + format!("https://github.com/{REPO}/releases/download/{RELEASE}/{checksum}") + ); + assert_eq!(newest_in(&listing, t, 2).unwrap().tag, "v0.7.0"); + assert_eq!(newest_in(&listing, "x86_64-unknown-linux-gnu", 1), None); + let refused = serde_json::json!({"message": "API rate limit exceeded"}); + assert_eq!(newest_in(&refused, t, 1), None); + } + + #[test] + fn nothing_installed_is_downgraded() { + assert!(newer(Some("v0.1.10"), "v0.1.9")); + assert!(!newer(Some("v0.1.3"), "v0.1.3")); + assert!(!newer(Some("v0.1.2"), "v0.1.3")); + assert!(!newer(None, "v0.1.3")); + assert!(!newer(Some("a release that does not say"), "v0.1.3")); + } + + #[test] + fn the_engine_abi_is_the_one_the_header_states() { + // The headers are at the repository's root: four directories up here, + // three in the public repository, where this crate is `base-convert/`. + let crate_dir = Path::new(env!("CARGO_MANIFEST_DIR")); + let header = [ + "../../../../include/baseRT/baseRT.h", + "../../../include/baseRT/baseRT.h", + ] + .iter() + .map(|rel| crate_dir.join(rel)) + .find(|p| p.is_file()) + .expect("include/baseRT/baseRT.h at the repository's root"); + let text = fs::read_to_string(&header).unwrap(); + let stated = text + .lines() + .find_map(|l| l.strip_prefix("#define BASERT_ABI_VERSION ")) + .map(|n| n.trim().parse::().unwrap()); + assert_eq!(stated, Some(ENGINE_ABI), "{}", header.display()); + } + /// `ensure_with` against installs named by `BASERT_SUPERFLUID_DIR` is /// covered end to end in tests/serve_launcher.rs, where the environment /// is the launched process's own; here the pieces that need none. diff --git a/base-convert/crates/base-convert/tests/serve_launcher.rs b/base-convert/crates/base-convert/tests/serve_launcher.rs index 8b3a3ba..de45d61 100644 --- a/base-convert/crates/base-convert/tests/serve_launcher.rs +++ b/base-convert/crates/base-convert/tests/serve_launcher.rs @@ -1,7 +1,8 @@ //! `basert serve` and `basert superfluid`, end to end against a fake superfluid -//! release: the launcher installs it from a directory of release assets, asks -//! it whether it can serve the engine, brings one that cannot to the pin, and -//! execs it with the C++ server's command line translated to superfluid's. +//! release: the launcher installs it from a directory of release assets, or +//! the newest a release listing names, asks it whether it can serve the +//! engine, replaces one that cannot, and execs it with the C++ server's +//! command line translated to superfluid's. use std::fs; use std::path::{Path, PathBuf}; @@ -173,13 +174,134 @@ fn run(command: &mut Command) -> (i32, String, String) { ) } -/// The pinned release, as `basert superfluid status` prints it. +/// The release basert was tested with, as `basert superfluid status` prints +/// it: what a directory of release assets holds. fn pinned(f: &Fixture) -> String { let (_, out, _) = run(f.basert().args(["superfluid", "status"])); out.lines() - .find_map(|l| l.strip_prefix("pinned:")) + .find_map(|l| l.strip_prefix("tested:")) .map(|v| v.trim().to_string()) - .unwrap_or_else(|| panic!("no pinned line in {out:?}")) + .unwrap_or_else(|| panic!("no tested line in {out:?}")) +} + +/// Serves the files under `root` over HTTP on 127.0.0.1 while the test runs, +/// as GitHub serves a release's assets; returns the base URL. +fn serve_files(root: PathBuf) -> String { + use std::io::{BufRead, BufReader, Write}; + let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap(); + let base = format!("http://{}", listener.local_addr().unwrap()); + std::thread::spawn(move || { + for stream in listener.incoming() { + let Ok(mut stream) = stream else { continue }; + let mut reader = BufReader::new(stream.try_clone().unwrap()); + let mut request = String::new(); + if reader.read_line(&mut request).is_err() { + continue; + } + let mut line = String::new(); + while reader.read_line(&mut line).is_ok_and(|n| n > 0) && line != "\r\n" { + line.clear(); + } + let path = request.split_whitespace().nth(1).unwrap_or("/"); + let _ = match fs::read(root.join(path.trim_start_matches('/'))) { + Ok(body) => write!( + stream, + "HTTP/1.1 200 OK\r\nContent-Length: {}\r\nConnection: close\r\n\r\n", + body.len() + ) + .and_then(|_| stream.write_all(&body)), + Err(_) => write!( + stream, + "HTTP/1.1 404 Not Found\r\nContent-Length: 0\r\nConnection: close\r\n\r\n" + ), + }; + } + }); + base +} + +#[test] +fn without_superfluid_the_newest_release_that_serves_the_engine_is_installed() { + let f = fixture(); + // A mirror's layout: each checksum in another directory than its tarball. + let www = f.tmp.path().join("www"); + for version in ["1.2.0", "2.0.0"] { + let assets = release_assets(&f.tmp.path().join(format!("release-{version}")), version); + for entry in fs::read_dir(&assets).unwrap() { + let entry = entry.unwrap(); + let kind = match entry.path().extension().and_then(|e| e.to_str()) { + Some("sha256") => "checksums", + _ => "files", + }; + let dir = www.join(kind).join(format!("v{version}")); + fs::create_dir_all(&dir).unwrap(); + fs::copy(entry.path(), dir.join(entry.file_name())).unwrap(); + } + } + let base = serve_files(www); + let name = format!("superfluid-{}", target()); + let release = |tag: &str, abi: u32| { + serde_json::json!({"tag_name": tag, "draft": false, "prerelease": false, "assets": [ + {"name": format!("{name}.tar.gz"), "browser_download_url": format!("{base}/files/{tag}/{name}.tar.gz")}, + {"name": format!("{name}.tar.gz.sha256"), "browser_download_url": format!("{base}/checksums/{tag}/{name}.tar.gz.sha256")}, + {"name": format!("superfluid-basert-abi-{abi}.txt"), "browser_download_url": format!("{base}/{tag}/abi")}, + ]}) + }; + // 2.0.0 serves another engine ABI: 1.2.0 is the newest that serves this one. + let listing = f.tmp.path().join("releases.json"); + let list = |releases: Vec| { + fs::write(&listing, serde_json::Value::from(releases).to_string()).unwrap() + }; + list(vec![ + release("v2.0.0", 2), + release("v1.2.0", 1), + release("v1.1.0", 1), + ]); + let basert = || { + let mut c = f.basert(); + c.env_remove("BASERT_SUPERFLUID_SOURCE") + .env("BASERT_SUPERFLUID_RELEASES", &listing); + c + }; + + let (code, out, err) = run(basert() + .env("BASERT_SUPERFLUID_AUTO_INSTALL", "0") + .args(["serve", f.model()])); + assert_eq!(code, 1, "{out}{err}"); + assert!(err.contains("superfluid is not installed"), "{err}"); + assert!( + !err.contains("v1.2.0"), + "a start forbidden to download looks nothing up: {err}" + ); + + let (code, out, err) = run(basert().args(["serve", f.model()])); + assert_eq!(code, 0, "{out}{err}"); + assert!( + err.contains("basert serve: installing superfluid v1.2.0 into"), + "{err}" + ); + assert!( + err.contains(&format!( + "downloading superfluid v1.2.0 ({name}.tar.gz) from {base}/files/v1.2.0/{name}.tar.gz" + )), + "{err}" + ); + assert_eq!(f.stamp(), "v1.2.0"); + assert!(out.contains("ARGS: serve"), "{out}"); + + let (code, out, _) = run(basert().args(["superfluid", "install"])); + assert_eq!(code, 0, "{out}"); + assert!(out.contains("superfluid v1.2.0 is installed at"), "{out}"); + + // A listing whose newest is older than the install downgrades nothing. + list(vec![release("v1.1.0", 1)]); + let (code, out, err) = run(basert().args(["superfluid", "install"])); + assert_eq!(code, 0, "{out}{err}"); + assert!( + out.contains("newer than v1.1.0: nothing is downgraded"), + "{out}" + ); + assert_eq!(f.stamp(), "v1.2.0"); } #[test] diff --git a/base-convert/crates/base-hub/src/cache.rs b/base-convert/crates/base-hub/src/cache.rs index 43e2544..7124e5a 100644 --- a/base-convert/crates/base-hub/src/cache.rs +++ b/base-convert/crates/base-hub/src/cache.rs @@ -137,6 +137,10 @@ pub struct HubSidecar { /// sha256 of the installed `model.base`. #[serde(default, skip_serializing_if = "Option::is_none")] pub base_sha256: Option, + /// The repo file a pre-converted `.base` was taken from, so a later + /// `pull --file` can tell its own file from another with the same tag. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub file: Option, } /// Write the provenance sidecar into a variant directory (creates the dir). @@ -244,6 +248,7 @@ mod tests { profile: Some("default-q4".into()), pulled_at: "2026-06-24T00:00:00Z".into(), base_sha256: None, + file: None, }; write_sidecar(&vdir, &s).unwrap(); let back = read_sidecar(&vdir).unwrap().unwrap(); diff --git a/base-convert/crates/base-hub/src/registry.rs b/base-convert/crates/base-hub/src/registry.rs index 1d8ebcb..5e8ad48 100644 --- a/base-convert/crates/base-hub/src/registry.rs +++ b/base-convert/crates/base-hub/src/registry.rs @@ -102,6 +102,10 @@ pub struct ModelEntry { pub source_kind: SourceKind, #[serde(skip_serializing_if = "Option::is_none")] pub path: Option, + /// The repo file the `.base` was pulled from (`hub.json`), or that a + /// catalog row publishes. + #[serde(skip_serializing_if = "Option::is_none")] + pub file: Option, } /// Common interface implemented by each individual source. @@ -200,6 +204,10 @@ impl LocalRegistry { installed: true, source_kind: SourceKind::Local, path: Some(artifact.to_path_buf()), + file: cache::read_sidecar(variant_dir) + .ok() + .flatten() + .and_then(|s| s.file), }) } @@ -257,7 +265,8 @@ impl CatalogRegistry { } } - fn backend_ok(e: &crate::catalog::CatalogEntry) -> bool { + /// Whether this client can run `e`: a universal row, or one for its backend. + pub fn backend_ok(e: &crate::catalog::CatalogEntry) -> bool { e.backend .as_deref() .map(|b| b == Self::client_backend()) @@ -291,6 +300,34 @@ impl CatalogRegistry { } } + /// The catalog row for `id` (exact id first, then case-insensitive), + /// whatever its quant or backend: where `pull --file` finds the repo a + /// cataloged id's files live in. + pub fn entry(&self, id: &str) -> Option<&crate::catalog::CatalogEntry> { + let rows = &self.catalog.models; + rows.iter() + .find(|e| e.id == id) + .or_else(|| rows.iter().find(|e| e.id.eq_ignore_ascii_case(id))) + } + + /// Every catalog row of `id`, whatever its quant or backend: where a + /// `pull --file` finds the variant and checksums of a file the catalog + /// publishes. Rows of the exact id when there are any, else its + /// case-insensitive aliases, as resolution prefers. + pub fn rows<'a>( + &'a self, + id: &'a str, + ) -> impl Iterator { + let exact = self.catalog.models.iter().any(|e| e.id == id); + self.catalog.models.iter().filter(move |e| { + if exact { + e.id == id + } else { + e.id.eq_ignore_ascii_case(id) + } + }) + } + /// Quant-agnostic, backend-aware resolve. pub fn resolve(&self, id: &str) -> Option { self.resolve_variant(id, None) @@ -411,6 +448,7 @@ impl Registry for CatalogRegistry { installed: false, source_kind: SourceKind::Catalog, path: None, + file: Some(e.file.clone()).filter(|f| !f.is_empty()), }) .collect()) } @@ -810,6 +848,21 @@ mod tests { } } + #[test] + fn rows_prefer_the_exact_id_over_case_aliases() { + let reg = catalog_json( + r#"{"id":"basecompute/CamelModel","hf_repo":"basecompute/alias","file":"alias-Q4.base","quant":"default-q4"}, + {"id":"basecompute/camelmodel","hf_repo":"basecompute/exact","file":"exact-Q4.base","quant":"default-q4"}"#, + ); + let repos = |id| reg.rows(id).map(|e| e.hf_repo.as_str()).collect::>(); + assert_eq!(repos("basecompute/camelmodel"), ["basecompute/exact"]); + assert_eq!(repos("basecompute/CamelModel"), ["basecompute/alias"]); + assert_eq!( + repos("basecompute/CAMELMODEL"), + ["basecompute/alias", "basecompute/exact"] + ); + } + #[test] fn local_scan_walks_tree_and_skips_staging() { let tmp = tempfile::tempdir().unwrap(); diff --git a/bindings/node/package.json b/bindings/node/package.json index 17b523d..b1967b1 100644 --- a/bindings/node/package.json +++ b/bindings/node/package.json @@ -1,6 +1,6 @@ { "name": "@baseRT/node", - "version": "0.4.0", + "version": "0.4.1", "private": true, "description": "Node.js bindings for BaseRT — LLM inference engine for Apple Silicon (Metal)", "main": "dist/index.js", diff --git a/bindings/python/baseRT/__init__.py b/bindings/python/baseRT/__init__.py index ec0d049..f919f27 100644 --- a/bindings/python/baseRT/__init__.py +++ b/bindings/python/baseRT/__init__.py @@ -21,7 +21,7 @@ from pathlib import Path from typing import Any, Callable, Dict, Iterator, List, Optional, Tuple, Union -__version__ = "0.4.0" +__version__ = "0.4.1" # --------------------------------------------------------------------------- # Library loading diff --git a/bindings/python/setup.py b/bindings/python/setup.py index 1e26078..b4e8f53 100644 --- a/bindings/python/setup.py +++ b/bindings/python/setup.py @@ -6,7 +6,7 @@ setup( name="baseRT", - version="0.4.0", + version="0.4.1", description="Python bindings for the BaseRT LLM inference engine (Apple Silicon / Metal)", long_description=long_description, long_description_content_type="text/markdown", diff --git a/bindings/rust/baseRT-sys/Cargo.toml b/bindings/rust/baseRT-sys/Cargo.toml index 47c1fde..d27e4ef 100644 --- a/bindings/rust/baseRT-sys/Cargo.toml +++ b/bindings/rust/baseRT-sys/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "baseRT-sys" -version = "0.4.0" +version = "0.4.1" edition = "2021" description = "Raw FFI bindings for the BaseRT LLM inference engine" license = "Apache-2.0" diff --git a/bindings/rust/baseRT/Cargo.toml b/bindings/rust/baseRT/Cargo.toml index 1b240f3..24b16fb 100644 --- a/bindings/rust/baseRT/Cargo.toml +++ b/bindings/rust/baseRT/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "baseRT" -version = "0.4.0" +version = "0.4.1" edition = "2021" description = "Safe Rust bindings for the BaseRT LLM inference engine (Apple Silicon)" license = "Apache-2.0" diff --git a/bindings/swift/Sources/CBaseRT/include/baseRT.h b/bindings/swift/Sources/CBaseRT/include/baseRT.h index 9768a55..2774b34 100644 --- a/bindings/swift/Sources/CBaseRT/include/baseRT.h +++ b/bindings/swift/Sources/CBaseRT/include/baseRT.h @@ -64,7 +64,7 @@ extern "C" { #define BASERT_VERSION_MAJOR 0 #define BASERT_VERSION_MINOR 4 -#define BASERT_VERSION_PATCH 0 +#define BASERT_VERSION_PATCH 1 /// Compile-time version, packed as `(MAJOR<<16) | (MINOR<<8) | PATCH`. /// Useful for `#if BASERT_VERSION >= 0x000200` feature checks. diff --git a/include/baseRT/baseRT.h b/include/baseRT/baseRT.h index 9768a55..2774b34 100644 --- a/include/baseRT/baseRT.h +++ b/include/baseRT/baseRT.h @@ -64,7 +64,7 @@ extern "C" { #define BASERT_VERSION_MAJOR 0 #define BASERT_VERSION_MINOR 4 -#define BASERT_VERSION_PATCH 0 +#define BASERT_VERSION_PATCH 1 /// Compile-time version, packed as `(MAJOR<<16) | (MINOR<<8) | PATCH`. /// Useful for `#if BASERT_VERSION >= 0x000200` feature checks.