diff --git a/.gitignore b/.gitignore index 7b856ece..ea276a44 100644 --- a/.gitignore +++ b/.gitignore @@ -1,10 +1,12 @@ __pycache__/ *.pyc +.venv .venv/ *.egg-info/ build/ dist/ .DS_Store +node_modules node_modules/ src-tauri/target/ src-tauri/resources/embedded/ diff --git a/CHANGELOG.md b/CHANGELOG.md index a1951c17..2f93f23a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -19,6 +19,8 @@ First release since v0.9.4; versions 0.9.5 to 0.9.7 were internal. Full notes in - New models: LFM2.5 24B-A2B (MLX 4/8-bit and GGUF), Muse Glimmer 30B GGUF (vision, DFlash 2 sidecar), Kolibri 1 GGUF Q3_K_M, Qwen3.8 Flash Next GGUF, Qwen3.6 35B-A3B MTP through TensorFold, Qwen-Image-2.1 Turbo and FLUX 3 Image Klein through mflux. - Image input for MLX vision models loaded through mlx-vlm (Gemma 4 on Apple Silicon). - Memory and speed estimates rebuilt for chat, image and video: real KV-cache layout per architecture, memory pressure from available memory, decode speed from this machine's history or its memory bandwidth, an advisory check before a chat model loads, and engine-aware image and video estimates (FU-110). +- Dashboard process list: Stop per leftover model process and a Stop idle button; unloading cancels a load in flight and a newer load replaces the one running. +- Disk usage breaks the shared Hugging Face blob store down by repo, and a cached repo can be deleted there through the cache API. - `release-constraints.txt` pins every package in release builds to the tested versions. ### Fixed diff --git a/RELEASE_NOTES_v0.9.8.md b/RELEASE_NOTES_v0.9.8.md index fce3d6da..62461233 100644 --- a/RELEASE_NOTES_v0.9.8.md +++ b/RELEASE_NOTES_v0.9.8.md @@ -7,6 +7,7 @@ The first release since v0.9.4. Versions 0.9.5 to 0.9.7 were internal builds, so - **DFlash on Apple Silicon finally runs.** Since dflash-mlx 0.1.5, every DFlash reply had quietly fallen back to normal decoding while the load note still said DFlash was active. Fixed: Qwen3-4B now writes at 113 to 138 tokens per second against 53 without it, with identical output. - **Newer MLX underneath.** Releases now ship mlx 0.32, mlx-lm 0.32 and transformers 5.20. Setup installs resolve mlx-vlm 0.7.10 and diffusers 0.41. Current Gemma 4 conversions load (the old mlx-vlm could not read them), and every release build pins the exact versions it was tested with. - **Images in chat on Apple Silicon.** Vision models loaded through mlx-vlm (Gemma 4) now see attached images. Before, the image was always dropped. +- **Models stop loading when you ask.** Unload cancels a load in flight, and picking a different model replaces the one that is loading instead of queueing behind it. The Dashboard lists leftover model processes with a Stop button each and a Stop idle button, so a stuck worker can no longer hold 20 GB of memory with no way out. Send is disabled while a model loads, so a message can no longer go to the previous model. - **Honest memory and speed estimates.** The launch screen works out the real KV-cache size for each model's architecture and estimates speed from your own past runs or your machine's memory bandwidth. It warns before you load a chat model that may not fit. Image and video estimates know which engine they run on and count a chat model that is already loaded. ### New models @@ -31,7 +32,7 @@ The first release since v0.9.4. Versions 0.9.5 to 0.9.7 were internal builds, so ### Faster generation - **DFlash** (Apple Silicon) fixed, as above. -- **Thinking Off now works** for Qwen 3 / 3.5 / 3.6 / 3.8 on Apple Silicon. Before, they reasoned anyway and spent the reply budget on it. +- **Thinking Off now works** for Qwen 3 / 3.5 / 3.6 / 3.8 on Apple Silicon, and the switch is sent to llama.cpp models too. Before, they reasoned anyway and spent the reply budget on it. - **TensorFold 0.6.6** (Apple Silicon): exact speculative decoding with no change in output. Install it from the launch settings. - **llama.cpp b11810** bundled: DFlash 2, EAGLE-3 and draft-free `ngram-mod`, picked automatically from the drafter files next to a GGUF. - **MTPLX 2.12**: reasoning now streams correctly. @@ -54,7 +55,9 @@ A Voice group in the sidebar: speech-to-text and text-to-speech with Studio, Mod - **Chat > My Models lists chat models only.** 3D tool parts, text encoders, voice models and bare weight files no longer show up as chat models. - **Settings > Diagnostics is split into three tabs**: Diagnostics, Accelerators and Disk usage. -- **Disk usage rows expand and can be deleted.** The shared Hugging Face `blobs` store is protected. +- **Disk usage rows expand and can be deleted.** The shared Hugging Face `blobs` store is now broken down by repo, and any cached repo can be deleted from there; blobs another repo still uses are kept. +- **Streaming follows the bottom.** In Chat, Compare and the HTML challenge the view now stays with the newest text until you scroll up. Before, one long chunk made it stop following, and Latest had to be pressed again and again. +- **Multi-quant GGUF repos show the size of the one file that loads**, not the whole folder, in the library and the memory bar. A model that fits no longer looks too big to run. ### Fixes @@ -68,6 +71,10 @@ A Voice group in the sidebar: speech-to-text and text-to-speech with Studio, Mod - **Dead catalog links** repaired; an audit script checks every row. - **Release builds** include the `dflash` package, the Setup installers and a pinned llama.cpp build, so GGUF models work out of the box. - **Tool use on Apple Silicon** no longer loops. +- **Loading** a model no longer hangs forever when a second load is requested: repeated requests for the same model share one load. +- **My Models and the chat picker** no longer come up empty while the first library scan is still running. +- **MTPLX** is no longer chosen for models without an MTP head, and the TensorFold toggle names DFlash 2 and MTP drafting. A cold TensorFold start says it can take a few minutes. +- **Extras overlay**: an older huggingface-hub or transformers left by a Setup install can no longer shadow the ones the app ships. - **MLX probe**: one stray kill of the startup check at launch no longer turns MLX off until you restart; it is retried once. The launch screen's context setting is capped at the model's own window, so a stored 262K no longer triggers a false memory warning on a 32K model. - **Hub listings** work on Python installs without a certificate bundle (Discover came back empty there). - **Downloads** skip weight shards that no index file uses, and the Gemma 4 entries are merged into one family. diff --git a/backend_service/helpers/discovery.py b/backend_service/helpers/discovery.py index 233ba7a6..a413c217 100644 --- a/backend_service/helpers/discovery.py +++ b/backend_service/helpers/discovery.py @@ -197,6 +197,14 @@ def _model_has_files(path: Path, pattern: str) -> bool: +def _gguf_load_size_gb(directory: Path) -> float | None: + """Size of the one GGUF that loads from a multi-quant repo folder.""" + from backend_service.helpers.text_gguf import gguf_load_bytes + + size = gguf_load_bytes(directory) + return round(size / 1_000_000_000, 2) if size else None + + def _detect_storage_format(path: Path, *, name_hint: str = "") -> str: lowered_hint = f"{name_hint} {path}".lower() if path.is_file() and path.suffix.lower() == ".gguf": @@ -483,6 +491,7 @@ def _discover_local_models(model_directories: list[dict[str, Any]], limit: int = "backend": backend, "modelType": model_type, "sizeGb": _du_size_gb(child), + "loadSizeGb": _gguf_load_size_gb(child) if file_format == "GGUF" and child.is_dir() else None, "lastModified": time.strftime("%Y-%m-%d %H:%M", time.localtime(child.stat().st_mtime)), "actions": ["Run Chat", "Run Server", "Cache Preview", "Delete"], "directoryId": directory_id, diff --git a/backend_service/helpers/persistence.py b/backend_service/helpers/persistence.py index 303689dc..b4280dde 100644 --- a/backend_service/helpers/persistence.py +++ b/backend_service/helpers/persistence.py @@ -11,7 +11,7 @@ LEGACY_SEEDED_CHAT_IDS = {"ui-direction", "model-shortlist"} LEGACY_SEEDED_BENCHMARK_IDS = {"baseline", "native-34", "native-36", "native-44"} -LIBRARY_CACHE_VERSION = 4 # 4: modelType "other" (non-chat models) added +LIBRARY_CACHE_VERSION = 5 # 5: loadSizeGb (size of the one GGUF that loads) def _default_chat_variant() -> dict[str, Any]: diff --git a/backend_service/helpers/quantization.py b/backend_service/helpers/quantization.py index 78fa63a2..bf655ca6 100644 --- a/backend_service/helpers/quantization.py +++ b/backend_service/helpers/quantization.py @@ -14,6 +14,7 @@ from __future__ import annotations import re +import sys from typing import Any @@ -51,7 +52,32 @@ def _quantization_algo_label(config: dict[str, Any] | None) -> str | None: return None +def _is_fp8_compressed_tensors(config: dict[str, Any] | None) -> bool: + """A compressed-tensors checkpoint whose weights are FP8 ("float-quantized").""" + if not isinstance(config, dict): + return False + payload = config.get("quantization_config") + if not isinstance(payload, dict) or payload.get("quant_method") != "compressed-tensors": + return False + if payload.get("format") == "float-quantized": + return True + groups = payload.get("config_groups") + if isinstance(groups, dict): + for group in groups.values(): + weights = group.get("weights") if isinstance(group, dict) else None + if isinstance(weights, dict) and weights.get("type") == "float": + return True + return False + + def _unsupported_mlx_quantization_reason(config: dict[str, Any] | None) -> str | None: + # On a Mac MLX is the only runtime, and it cannot dequantize FP8 safetensors + # (mlx-lm convert stops with "unsupported compressed-tensors format"). + if sys.platform == "darwin" and _is_fp8_compressed_tensors(config): + return ( + "This FP8 checkpoint (compressed-tensors) needs an NVIDIA GPU runtime such as vLLM; " + "MLX cannot read it. Use an MLX or GGUF build of this model on a Mac." + ) algo = _quantization_algo_label(config) if not algo or algo not in _UNSUPPORTED_MLX_QUANT_ALGOS: return None diff --git a/backend_service/helpers/text_gguf.py b/backend_service/helpers/text_gguf.py index f9f5d353..df77f132 100644 --- a/backend_service/helpers/text_gguf.py +++ b/backend_service/helpers/text_gguf.py @@ -192,3 +192,27 @@ def pick_model_gguf(directory: Path, model_files: list[Path]) -> Path: if first.exists(): return first return chosen + + +def gguf_load_bytes(directory: Path) -> int | None: + """Bytes of the GGUF llama.cpp would load from ``directory`` (all shards), or None. + + A repo folder can hold many quants (a 24B model's BF16, F16, Q8_0 ... is 150 GB + on disk), but one file loads, so memory estimates must use that file. + """ + try: + files = [f for f in directory.rglob("*.gguf") if not is_aux_gguf(f.name)] + if not files: + return None + files.sort(key=lambda f: f.stat().st_size, reverse=True) + chosen = pick_model_gguf(directory, files) + total = chosen.stat().st_size + match = _SHARD_RE.search(chosen.name) + if match: + stem = chosen.name[: match.start()] + for shard in chosen.parent.glob(f"{stem}-*-of-{match.group(2)}.gguf"): + if shard != chosen: + total += shard.stat().st_size + return total + except OSError: + return None diff --git a/backend_service/inference/_mtp.py b/backend_service/inference/_mtp.py index 5777218e..38be16d6 100644 --- a/backend_service/inference/_mtp.py +++ b/backend_service/inference/_mtp.py @@ -183,6 +183,37 @@ def _read_safetensors_index(model_dir: _Path) -> dict[str, str] | None: return weight_map if isinstance(weight_map, dict) else None +def _snapshot_dir(directory: _Path) -> _Path: + """The newest snapshot of a Hugging Face cache repo folder, else ``directory``. + + The library hands the engine ``models--owner--name`` (no weights in it), so + without this the probe finds no index and callers fall back to a name match, + which a mis-matched catalog name can get wrong. + """ + snapshots = directory / "snapshots" + if not snapshots.is_dir(): + return directory + try: + found = [d for d in snapshots.iterdir() if d.is_dir()] + except OSError: + return directory + return max(found, key=lambda d: d.stat().st_mtime) if found else directory + + +def _read_single_safetensors_keys(model_dir: _Path) -> dict[str, str] | None: + """Tensor names of an unsharded ``model.safetensors`` (its JSON header), or None.""" + weights = model_dir / "model.safetensors" + try: + with weights.open("rb") as fh: + header_len = int.from_bytes(fh.read(8), "little") + if not 0 < header_len < 100 * 1024 * 1024: + return None + header = _json.loads(fh.read(header_len)) + except (OSError, ValueError): + return None + return {name: "model.safetensors" for name in header if name != "__metadata__"} + + def model_has_mtp_tensors(path: str | None) -> bool | None: """Authoritative MTP-head probe via local model files. @@ -225,10 +256,12 @@ def model_has_mtp_tensors(path: str | None) -> bool | None: # MLX / safetensors case: look for the dedicated shard or any # ``mtp_*`` key in the index. - model_dir = p if p.is_dir() else p.parent + model_dir = _snapshot_dir(p if p.is_dir() else p.parent) if (model_dir / "mtp.safetensors").exists(): return True weight_map = _read_safetensors_index(model_dir) + if weight_map is None: + weight_map = _read_single_safetensors_keys(model_dir) if weight_map is None: return None for tensor_name in weight_map.keys(): diff --git a/backend_service/inference/_tensorfold.py b/backend_service/inference/_tensorfold.py index 12e2fb12..79874152 100644 --- a/backend_service/inference/_tensorfold.py +++ b/backend_service/inference/_tensorfold.py @@ -290,12 +290,27 @@ def match_model( return TensorFoldMatch(family=family, repo=repo, matched_by="repo") directory = model_dir or _directory_candidate(path, runtime_target) - family = _EXCLUSIVE_MODEL_TYPES.get(read_model_type(directory) or "") + model_type = read_model_type(directory) or "" + family = _EXCLUSIVE_MODEL_TYPES.get(model_type) + # "Exclusive" means stock MLX cannot load the architecture. mlx-lm 0.32 ships + # prism_hadamard_qwen35, so a small Ternary Bonsai 2 loads there, and + # TensorFold itself refuses every checkpoint but the 27B. + if family is not None and _stock_mlx_loads(model_type): + family = None if family is not None: return TensorFoldMatch(family=family, repo=None, matched_by="config") return None +def _stock_mlx_loads(model_type: str) -> bool: + import importlib.util + + try: + return importlib.util.find_spec(f"mlx_lm.models.{model_type}") is not None + except (ImportError, ValueError): + return False + + def _directory_candidate(*candidates: str | None) -> Path | None: for candidate in candidates: if not candidate: diff --git a/backend_service/inference/controller.py b/backend_service/inference/controller.py index 49cb13ba..2f4a84f1 100644 --- a/backend_service/inference/controller.py +++ b/backend_service/inference/controller.py @@ -13,7 +13,7 @@ from __future__ import annotations -from functools import lru_cache +from functools import lru_cache, wraps import json import os @@ -21,7 +21,7 @@ import time from dataclasses import dataclass from pathlib import Path -from threading import Lock, Thread +from threading import Event, Lock, RLock, Thread from collections.abc import Callable, Iterator from typing import Any @@ -112,11 +112,40 @@ from backend_service.inference.tensorfold_engine import TensorFoldEngine +def _model_weights_bytes(path: str) -> int: + """Bytes a model occupies when loaded, as opposed to bytes in its folder. + + A GGUF repo folder usually holds several quants (a 24B model's BF16, F16 and + Q8_0 are 148 GB together) but one file loads, so count that file. Folders + that also carry safetensors are sized whole: their loader reads those. + """ + try: + folder = Path(path) + if folder.is_dir() and not any(folder.rglob("*.safetensors")): + from backend_service.helpers.text_gguf import gguf_load_bytes # noqa: PLC0415 + + loaded = gguf_load_bytes(folder) + if loaded: + return loaded + except OSError: + pass + return _path_size_bytes(path) + + @lru_cache(maxsize=32) def _cached_path_size_bytes(path: str) -> int: """Size of a loaded model's files. Cached: the runtime status (polled every few seconds) reports it, and a loaded model's files do not change.""" - return _path_size_bytes(path) + return _model_weights_bytes(path) + + +def _one_load_at_a_time(method): + @wraps(method) + def wrapper(self, *args, **kwargs): + with self._load_lock: + return method(self, *args, **kwargs) + + return wrapper class RuntimeController: @@ -136,6 +165,16 @@ def __init__(self, *, background_probe: bool = False) -> None: # Warm pool: keeps previously loaded engines alive for instant switch-back self._warm_pool: dict[str, tuple[BaseInferenceEngine, LoadedModelInfo]] = {} self._pool_lock = Lock() + # One load or unload at a time: a second request used to start while the + # first was still loading, replace ``self.engine`` and orphan the first + # engine's process (a 21 GB TensorFold worker kept running, starving the next model). + self._load_lock = RLock() + # Newest request wins: a load for another model aborts the one in flight, + # a repeat of the same request waits for it, and unload cancels any load. + self._latest_load_key: str | None = None + self._inflight_load_key: str | None = None + self._loaded_key: str | None = None + self._load_aborted = Event() self._loading_progress: dict[str, Any] | None = None self._loading_log_tail: list[str] = [] self._recent_orphaned_workers: list[dict[str, Any]] = [] @@ -321,7 +360,8 @@ def _tracked_process_pids(self) -> set[int]: # pure noise. ORPHAN_DETECTION_GRACE_SECONDS = 3.0 - def prune_stale_backend_children(self) -> None: + def prune_stale_backend_children(self, grace_seconds: float | None = None) -> None: + grace = self.ORPHAN_DETECTION_GRACE_SECONDS if grace_seconds is None else grace_seconds tracked = self._tracked_process_pids() try: import psutil @@ -356,7 +396,7 @@ def prune_stale_backend_children(self) -> None: # every time an engine was swapped in or out, so a normal # model-change session produced 5-10 "orphans" purely from # timing races. - if now_wall - create_time < self.ORPHAN_DETECTION_GRACE_SECONDS: + if now_wall - create_time < grace: continue if is_mlx_worker: @@ -392,6 +432,62 @@ def prune_stale_backend_children(self) -> None: self._expire_orphan_records() + _OWNED_PROCESS_MARKERS = ("mlx_worker", "llama-server", "/mtplx-venv/", "/tensorfold-venv/") + + def stop_process(self, pid: int) -> dict[str, Any]: + """Stop one process this app started that is not serving the loaded model. + + The loaded model's process is refused (unload the model instead), a parked + warm model is evicted properly, and anything that is not a model process + under this backend is refused, so the button cannot touch other programs. + """ + active = self.engine.process_pid() if self.engine else None + if active and int(active) == pid: + raise ValueError("That process serves the loaded model. Unload the model instead.") + with self._pool_lock: + warm_key = next( + (key for key, (engine, _info) in self._warm_pool.items() if engine.process_pid() == pid), + None, + ) + warm = self._warm_pool.pop(warm_key) if warm_key is not None else None + if warm is not None: + try: + warm[0].unload_model() + except Exception: + pass + self.prune_stale_backend_children(grace_seconds=0.0) + return {"pid": pid, "action": "unloaded", "label": warm[1].name} + try: + import psutil + + process = psutil.Process(pid) + owned = process.parent() is not None and any( + parent.pid == os.getpid() for parent in [process.parent(), *process.parents()] + ) + cmdline = " ".join(process.cmdline()).lower() + except Exception as exc: # noqa: BLE001 - gone, or not ours to read + raise ValueError("That process is no longer running.") from exc + if not owned or not any(marker in cmdline for marker in self._OWNED_PROCESS_MARKERS): + raise ValueError("That is not a model process started by ChaosEngineAI.") + try: + process.terminate() + process.wait(timeout=3) + action = "terminated" + except Exception: # noqa: BLE001 + try: + process.kill() + action = "killed" + except Exception: # noqa: BLE001 + action = "kill_failed" + return {"pid": pid, "action": action} + + def stop_idle(self) -> dict[str, Any]: + """Evict every parked warm model and stop every model process nothing is using.""" + before = len(self._recent_orphaned_workers) + evicted = self.clear_warm_pool() + self.prune_stale_backend_children(grace_seconds=0.0) + return {"warmEvicted": evicted, "orphansStopped": max(0, len(self._recent_orphaned_workers) - before)} + def _expire_orphan_records(self) -> None: if not self._recent_orphaned_workers: return @@ -730,7 +826,7 @@ def _target_resident_bytes(*, path: str | None, runtime_target: str | None) -> i for candidate in (path, runtime_target): if not candidate: continue - size = _path_size_bytes(candidate) + size = _model_weights_bytes(candidate) if size > 0: return size return 0 @@ -814,7 +910,63 @@ def _make_room_keeping_active(self, required_free_bytes: int) -> bool: free += self._pop_oldest_warm_entry() return free >= required_free_bytes - def load_model( + def load_model(self, **kwargs: Any) -> LoadedModelInfo: + key = self._warm_pool_key( + model_ref=kwargs["model_ref"], + runtime_target=kwargs.get("runtime_target"), + path=kwargs.get("path"), + cache_strategy=kwargs.get("cache_strategy", "native"), + cache_bits=kwargs.get("cache_bits", 0), + fp16_layers=kwargs.get("fp16_layers", 0), + fused_attention=kwargs.get("fused_attention", False), + fit_model_in_memory=kwargs.get("fit_model_in_memory", True), + context_tokens=kwargs.get("context_tokens", 8192), + speculative_decoding=kwargs.get("speculative_decoding", False), + ) + self._latest_load_key = key + waited = False + if not self._load_lock.acquire(timeout=0.2): + waited = True + if self._inflight_load_key not in (None, key): + self._abort_inflight_load() + self._load_lock.acquire() + try: + if self._latest_load_key != key: + raise RuntimeError("A newer load request replaced this one.") + # A repeat of the request that just finished loading is already served; + # a request that did not have to wait is a deliberate reload. + if waited and self.loaded_model is not None and self._loaded_key == key: + return self.loaded_model + self._load_aborted.clear() + self._inflight_load_key = key + try: + loaded = self._load_model_locked(**kwargs) + finally: + self._inflight_load_key = None + self._loaded_key = key + return loaded + finally: + self._load_lock.release() + + def _abort_inflight_load(self) -> None: + """Make the load in flight fail now by ending its engine's process.""" + self._load_aborted.set() + engine = self.engine + pid = engine.process_pid() if engine is not None else None + if pid: + try: + import psutil + + psutil.Process(int(pid)).kill() + return + except Exception: # noqa: BLE001 - already gone + pass + try: + engine.unload_model() + except Exception: # noqa: BLE001 + pass + + def _load_model_locked( self, *, model_ref: str, @@ -941,6 +1093,13 @@ def _internal_progress(progress: dict[str, Any]) -> None: try: loaded = self.engine.load_model(**_load_kwargs) except RuntimeError as _startup_exc: + if self._load_aborted.is_set(): + self.loaded_model = None + self.runtime_note = None + self._loading_progress = None + self._loading_log_tail = [] + self.prune_stale_backend_children() + raise RuntimeError("Model load cancelled.") from _startup_exc # An accelerator engine's startup failure → fall back to the # standard MLX worker (see ``_startup_fallback_label``). fallback_from = self._startup_fallback_label( @@ -1020,10 +1179,18 @@ def loading_progress(self) -> tuple[dict[str, Any] | None, list[str]]: return self._loading_progress, list(self._loading_log_tail) def unload_model(self) -> None: - self.engine.unload_model() - self.loaded_model = None - self.runtime_note = None - self.prune_stale_backend_children() + self._latest_load_key = None # queued loads give up instead of reloading + if not self._load_lock.acquire(timeout=0.2): + self._abort_inflight_load() + self._load_lock.acquire() + try: + self.engine.unload_model() + self.loaded_model = None + self._loaded_key = None + self.runtime_note = None + self.prune_stale_backend_children() + finally: + self._load_lock.release() def generate( self, diff --git a/backend_service/inference/llama_cpp_engine.py b/backend_service/inference/llama_cpp_engine.py index 33b505ec..42e4dfea 100644 --- a/backend_service/inference/llama_cpp_engine.py +++ b/backend_service/inference/llama_cpp_engine.py @@ -156,6 +156,7 @@ def _apply_sampler_kwargs( samplers: dict[str, Any] | None, reasoning_effort: str | None, json_schema: dict[str, Any] | None, + thinking_mode: str | None = None, ) -> None: """Merge Phase 2.2 sampler overrides into a chat-completions payload. @@ -172,6 +173,11 @@ def _apply_sampler_kwargs( payload[key] = value if reasoning_effort: payload["reasoning_effort"] = reasoning_effort + if thinking_mode is not None: + # Templates that gate reasoning on ``enable_thinking`` (Qwen3 family, + # LFM2.5, ...) reason by default; the server-wide ``--reasoning off`` + # does not reach every one, so say it per request. + payload["chat_template_kwargs"] = {"enable_thinking": thinking_mode != "off"} if json_schema: payload["response_format"] = { "type": "json_schema", @@ -974,6 +980,7 @@ def stream_generate( samplers=samplers, reasoning_effort=reasoning_effort, json_schema=json_schema, + thinking_mode=thinking_mode, ) url = self._server_url("/v1/chat/completions") data = json.dumps(payload).encode("utf-8") diff --git a/backend_service/inference/tensorfold_engine.py b/backend_service/inference/tensorfold_engine.py index 19e219cb..bcfb8c58 100644 --- a/backend_service/inference/tensorfold_engine.py +++ b/backend_service/inference/tensorfold_engine.py @@ -167,6 +167,8 @@ def _latest_server_line(log_path: Path | None) -> str | None: return None +_COLD_START_NOTE = "A cold start can take a few minutes." + class TensorFoldEngine(BaseInferenceEngine): engine_name = "tensorfold" engine_label = "TensorFold (exact speculative decoding)" @@ -257,6 +259,10 @@ def _wait_for_server( """Poll ``/health`` until the model is loaded and the port answers.""" deadline = time.monotonic() + MLX_LOAD_TIMEOUT_SECONDS last_line: str | None = None + # A first start reads every weight, loads the draft model and times the + # decoder lanes: about two to three minutes for a 27B model. Say so, or a + # silent badge invites a second click that restarts the load. + _emit_progress(progress_callback, "loading", None, f"Starting TensorFold. {_COLD_START_NOTE}") while time.monotonic() < deadline: if self.process is not None and self.process.poll() is not None: raise self._startup_failure() @@ -269,7 +275,7 @@ def _wait_for_server( line = _latest_server_line(self.log_path) if line and line != last_line: last_line = line - _emit_progress(progress_callback, "loading", None, line) + _emit_progress(progress_callback, "loading", None, f"{line[:140]} · {_COLD_START_NOTE}") time.sleep(_HEALTH_POLL_SECONDS) raise self._startup_failure( f"TensorFold did not finish loading {model_name} within " diff --git a/backend_service/mlx_worker_diagnostics.py b/backend_service/mlx_worker_diagnostics.py index f9c39362..463ece3d 100644 --- a/backend_service/mlx_worker_diagnostics.py +++ b/backend_service/mlx_worker_diagnostics.py @@ -32,6 +32,30 @@ _UNSUPPORTED_QUANT_ALGOS = {"NVFP4", "NVINT4"} +_FLINT_USE_GGUF = ( + "Use the Flint INT4 GGUF (RecursiveSI/flint-int4-gguf) with the llama.cpp backend instead." +) + + +def _reject_unloadable_flint(model_dir: Path) -> None: + """Say why a Flint checkpoint cannot load on MLX, instead of 'Model type flint not supported'.""" + if (model_dir / "flint-int4.json").exists(): + raise RuntimeError( + "Recursive's MLX INT4 build of Flint stores its embeddings in a layout only their own " + "rec engine reads, so MLX cannot load it. " + _FLINT_USE_GGUF + ) + shipped = [ + name + for name in ("mlx_lm", "mlx_vlm") + if importlib.util.find_spec(f"{name}.models.flint") is not None + ] + if not shipped: + raise RuntimeError( + "The installed MLX packages do not include the Flint architecture " + "(mlx-vlm 0.7.10 or newer does). " + _FLINT_USE_GGUF + ) + + def _reject_unsupported_quant(model_path: str) -> None: """Raise early if the model uses a quantisation format MLX cannot handle.""" cfg_path = Path(model_path) / "config.json" @@ -40,6 +64,8 @@ def _reject_unsupported_quant(model_path: str) -> None: try: with open(cfg_path) as f: cfg = json.load(f) + if cfg.get("model_type") == "flint": + _reject_unloadable_flint(Path(model_path)) qcfg = cfg.get("quantization_config") or {} algo = qcfg.get("quant_algo", "") method = qcfg.get("quant_method", "") diff --git a/backend_service/routes/diagnostics.py b/backend_service/routes/diagnostics.py index 2cd4655b..902beb00 100644 --- a/backend_service/routes/diagnostics.py +++ b/backend_service/routes/diagnostics.py @@ -31,6 +31,7 @@ from typing import Any from fastapi import APIRouter, HTTPException, Request +from pydantic import BaseModel, Field router = APIRouter() @@ -198,8 +199,10 @@ def _storage_entry(child: Path, source_label: str, *, dirs_only: bool) -> dict[s if repo_label.startswith("models--"): # HF cache layout: ``models--{org}--{repo}`` -> ``{org}/{repo}`` repo_label = repo_label[len("models--"):].replace("--", "/", 1) + hf_repo = repo_label if child.name.startswith("models--") and "/" in repo_label else None return { "path": str(child), + "hfRepo": hf_repo, "repoLabel": repo_label, "sizeBytes": size_bytes, "sizeGb": _bytes_to_gb(size_bytes), @@ -265,6 +268,97 @@ def diagnostics_storage_top(request: Request, limit: int = 20) -> dict[str, Any] } +def _shared_blobs_by_repo(blobs_root: Path, source_label: str) -> list[dict[str, Any]]: + """Group the shared Hugging Face blob store by the repo each blob belongs to. + + huggingface_hub 1.x keeps one ```` file plus a ``.refs`` note + per blob, sharded under two-letter folders; the note names the repo + (``models--org--name/blobs/``) that fetched it. A blob several repos + share is counted once, under the first repo listed. + """ + groups: dict[str, dict[str, Any]] = {} + try: + shards = [d for d in blobs_root.iterdir() if d.is_dir()] + except OSError: + return [] + for shard in shards: + try: + files = list(shard.iterdir()) + except OSError: + continue + for blob in files: + if blob.suffix == ".refs" or not blob.is_file(): + continue + try: + size = blob.stat().st_size + refs = [ + line.split("/", 1)[0] + for line in blob.with_name(blob.name + ".refs").read_text().splitlines() + if line.strip() + ] + except OSError: + continue + repo_dir = refs[0] if refs else "unreferenced" + kind, _, rest = repo_dir.partition("--") + label = rest.replace("--", "/", 1) if kind in {"models", "datasets", "spaces"} and rest else repo_dir + group = groups.setdefault(label, {"size": 0, "mtime": 0.0, "files": 0, "shared": 0, "kind": kind}) + group["size"] += size + group["files"] += 1 + group["shared"] += 1 if len(set(refs)) > 1 else 0 + group["mtime"] = max(group["mtime"], blob.stat().st_mtime) + return [ + { + "path": str(blobs_root), + "repoLabel": f"{label} ({g['files']} file{'s' if g['files'] != 1 else ''})", + "sizeBytes": g["size"], + "sizeGb": _bytes_to_gb(g["size"]), + "sourceKind": source_label, + "lastModified": g["mtime"], + "expandable": False, + "deletable": g["kind"] == "models" and "/" in label, + "hfRepo": label if g["kind"] == "models" and "/" in label else None, + } + for label, g in groups.items() + ] + + +class DeleteHfRepoRequest(BaseModel): + repo: str = Field(min_length=3, max_length=256, pattern=r"^[\w.\-]+/[\w.\-]+$") + + +@router.post("/api/diagnostics/hf-repo/delete") +def diagnostics_delete_hf_repo(request: Request, body: DeleteHfRepoRequest) -> dict[str, Any]: + """Remove one cached Hugging Face model repo, including blobs in the shared store. + + Goes through huggingface_hub's cache API so a blob another repo links to is + kept and only this repo's own files are freed. Unloads the model first when + it is the one loaded. + """ + from backend_service.helpers.hf_cache_paths import _hf_hub_cache_root + + state: ChaosEngineState = request.app.state.chaosengine + try: + from huggingface_hub import scan_cache_dir + + info = scan_cache_dir(_hf_hub_cache_root()) + except Exception as exc: # noqa: BLE001 - cache layout errors come in many types + raise HTTPException(status_code=500, detail=f"Cannot read the Hugging Face cache: {exc}") from exc + repo = next((r for r in info.repos if r.repo_type == "model" and r.repo_id == body.repo), None) + if repo is None: + raise HTTPException(status_code=404, detail=f"{body.repo} is not in the Hugging Face cache.") + loaded = getattr(state.runtime, "loaded_model", None) + if loaded and body.repo in {getattr(loaded, "ref", None), getattr(loaded, "canonicalRepo", None)}: + state.runtime.unload_model() + plan = info.delete_revisions(*[rev.commit_hash for rev in repo.revisions]) + freed = int(plan.expected_freed_size) + try: + plan.execute() + except OSError as exc: + raise HTTPException(status_code=500, detail=f"Failed to delete {body.repo}: {exc}") from exc + state.add_log("library", "info", f"Deleted cached repo {body.repo} ({freed} bytes).") + return {"deleted": body.repo, "freedBytes": freed, "library": state._library(force=True)} + + @router.get("/api/diagnostics/storage-children") def diagnostics_storage_children(request: Request, path: str) -> dict[str, Any]: """Direct children of one folder from the Disk usage table, biggest first. @@ -284,6 +378,10 @@ def diagnostics_storage_children(request: Request, path: str) -> dict[str, Any]: break else: raise HTTPException(status_code=403, detail="Path is outside the configured model directories.") + if (target / ".huggingface-shared-blobs").exists(): + grouped = _shared_blobs_by_repo(target, source_label) + grouped.sort(key=lambda e: e["sizeBytes"], reverse=True) + return {"path": str(target), "entries": grouped[:200]} try: children = list(target.iterdir()) except OSError as exc: diff --git a/backend_service/routes/models.py b/backend_service/routes/models.py index 09345a1d..776f8b22 100644 --- a/backend_service/routes/models.py +++ b/backend_service/routes/models.py @@ -138,6 +138,30 @@ async def unload_model(request: Request) -> dict[str, Any]: return {"runtime": runtime} +@router.post("/api/models/processes/stop") +def stop_model_process(request: Request, body: dict[str, Any]) -> dict[str, Any]: + """Stop one leftover model process (not the loaded model's).""" + pid = body.get("pid") + if not isinstance(pid, int) or isinstance(pid, bool): + raise HTTPException(status_code=400, detail="pid must be an integer.") + state = request.app.state.chaosengine + try: + result = state.runtime.stop_process(pid) + except ValueError as exc: + raise HTTPException(status_code=409, detail=str(exc)) from exc + state.add_log("runtime", "info", f"Stopped model process {pid} ({result['action']}).") + return result + + +@router.post("/api/models/processes/stop-idle") +def stop_idle_model_processes(request: Request) -> dict[str, Any]: + """Evict parked warm models and stop model processes nothing is using.""" + state = request.app.state.chaosengine + result = state.runtime.stop_idle() + state.add_log("runtime", "info", f"Stopped idle model processes: {result}.") + return result + + @router.post("/api/models/convert") def convert_model(request: Request, body: ConvertModelRequest) -> dict[str, Any]: state = request.app.state.chaosengine diff --git a/backend_service/runtime_paths.py b/backend_service/runtime_paths.py index 2e59099c..566bb23a 100644 --- a/backend_service/runtime_paths.py +++ b/backend_service/runtime_paths.py @@ -112,6 +112,13 @@ def ensure_extras_on_sys_path() -> list[Path]: # brings mlx without mlx-lm) would run the new mlx-lm on the old mlx. _BUNDLED_CORE = ("mlx", "mlx-metal", "mlx-lm") +# Packages transformers / mlx-lm import in lock-step with each other. Successive +# ``pip install --target`` runs into the overlay layer several versions of these +# on top of one another (huggingface-hub 1.12 files under a 2.2 dist-info), so +# the overlay is not just stale, it is a mix. Unless the overlay holds exactly +# the bundled version, every copy of the package goes and the bundled one runs. +_LOCKSTEP_CORE = ("huggingface-hub", "transformers", "tokenizers", "safetensors", "hf-xet") + def _normalise(name: str) -> str: return name.lower().replace("_", "-").replace(".", "-") @@ -161,12 +168,16 @@ def quarantine_stale_core_packages(extras_dir: Path, bundled_dirs: list[Path]) - root = extras_dir.resolve() moved: list[str] = [] emptied: set[Path] = set() - for name in _BUNDLED_CORE: + for name in _BUNDLED_CORE + _LOCKSTEP_CORE: want = bundled.get(name) if want is None: continue - for version, info in overlay.get(name, []): - if version == want: + copies = overlay.get(name, []) + # A lock-step package that is clean (one copy, the bundled version) stays; + # anything else is moved whole, including a copy that matches the bundle. + move_all = name in _LOCKSTEP_CORE and (len(copies) > 1 or any(v != want for v, _ in copies)) + for version, info in copies: + if version == want and not move_all: continue target = extras_dir.parent / "stale-core" / f"{name}-{version}" try: @@ -192,6 +203,12 @@ def quarantine_stale_core_packages(extras_dir: Path, bundled_dirs: list[Path]) - # the overlay root; rmdir refuses any folder that still holds files. for folder in sorted(emptied, key=lambda p: len(p.parts), reverse=True): while folder != root and root in folder.parents: + try: + leftovers = [child for child in folder.iterdir() if child.name != "__pycache__"] + if not leftovers: + shutil.rmtree(folder / "__pycache__", ignore_errors=True) + except OSError: + break try: folder.rmdir() except OSError: diff --git a/src/App.tsx b/src/App.tsx index 4c36d5c3..fbd8cc86 100644 --- a/src/App.tsx +++ b/src/App.tsx @@ -9,6 +9,8 @@ import { loadModel, getWorkspace, resolveApiToken, + stopIdleModelProcesses, + stopModelProcess, updateSession, updateSettings as updateSettingsApi, } from "./api"; @@ -118,6 +120,7 @@ import { import { useMtplxInstall } from "./hooks/useMtplxInstall"; import { useTensorfoldInstall } from "./hooks/useTensorfoldInstall"; import type { TensorfoldLaunchControls } from "./components/tensorfoldSupport"; +import { newStickyScroll, stickToBottom, updateStickyScroll, type StickyScroll } from "./utils/stickyScroll"; export default function App() { // FU-042: i18n hook — used for the workspace header tab label / @@ -407,7 +410,7 @@ export default function App() { return { key: `library:${item.path}`, label: item.name, - detail: `${displayFormat} / ${number(item.sizeGb)} GB`, + detail: `${displayFormat} / ${number(item.loadSizeGb ?? item.sizeGb)} GB`, group: "Local library", model: item.name, modelRef: item.name, @@ -416,7 +419,7 @@ export default function App() { path: item.path, backend: libraryItemBackend(item, matched), paramsB: matched?.paramsB, - sizeGb: item.sizeGb, + sizeGb: item.loadSizeGb ?? item.sizeGb, contextWindow: matched?.contextWindow, format: displayFormat, quantization: displayQuantization ?? undefined, @@ -890,20 +893,27 @@ export default function App() { }); }, [benchmarkOption?.key, setBenchmarkDraft]); - // Auto-scroll chat + // Auto-scroll chat: follow the stream until the user scrolls up (see + // utils/stickyScroll). A new chat or a new message turns the follow on. + const chatSticky = useRef(newStickyScroll()); const lastMessageLength = activeChat?.messages[activeChat.messages.length - 1]?.text?.length ?? 0; + useEffect(() => { + if (activeTab !== "chat") return; + const el = chatScrollRef.current; + if (!el) return; + const onScroll = () => { updateStickyScroll(el, chatSticky.current); }; + el.addEventListener("scroll", onScroll, { passive: true }); + return () => el.removeEventListener("scroll", onScroll); + }, [activeTab, activeChat?.id, chatScrollRef]); + useEffect(() => { + chatSticky.current.stuck = true; + }, [activeChat?.id, activeChat?.messages.length]); useEffect(() => { if (activeTab !== "chat") return; const handle = requestAnimationFrame(() => { const el = chatScrollRef.current; - if (!el) return; - // Only auto-scroll if the user is already near the bottom. - // This lets users scroll up to read earlier content during streaming - // without being yanked back to the bottom on every token. - const nearBottom = el.scrollHeight - el.scrollTop - el.clientHeight < 120; - if (nearBottom) { - el.scrollTop = el.scrollHeight; - } + if (!el || !chatSticky.current.stuck) return; + stickToBottom(el, chatSticky.current); }); return () => cancelAnimationFrame(handle); }, [activeTab, activeChat?.id, activeChat?.messages.length, lastMessageLength, chatScrollRef]); @@ -1215,6 +1225,22 @@ export default function App() { runtime={workspace.runtime} activity={workspace.activity} backendOnline={backendOnline} + onStopProcess={async (pid) => { + try { + await stopModelProcess(pid); + } catch (err) { + setError(err instanceof Error ? err.message : "Could not stop that process."); + } + await refreshWorkspace(); + }} + onStopIdleProcesses={async () => { + try { + await stopIdleModelProcesses(); + } catch (err) { + setError(err instanceof Error ? err.message : "Could not stop idle processes."); + } + await refreshWorkspace(); + }} />; if (activeTab === "online-models") { content = ( diff --git a/src/api/admin.ts b/src/api/admin.ts index eab230ed..03152ba4 100644 --- a/src/api/admin.ts +++ b/src/api/admin.ts @@ -127,6 +127,8 @@ export interface StorageTopEntry { lastModified: number; expandable?: boolean; deletable?: boolean; + /** Set for a cached Hugging Face repo: deleted through the cache API, not by path. */ + hfRepo?: string | null; } export interface StorageTopResponse { @@ -146,6 +148,10 @@ export async function fetchStorageTop(limit = 20): Promise { ); } +export async function deleteHfRepo(repo: string): Promise<{ deleted: string; freedBytes: number }> { + return await postJson<{ deleted: string; freedBytes: number }>("/api/diagnostics/hf-repo/delete", { repo }); +} + export async function fetchStorageChildren(path: string): Promise { const res = await fetchJson<{ entries: StorageTopEntry[] }>( `/api/diagnostics/storage-children?path=${encodeURIComponent(path)}`, diff --git a/src/api/index.ts b/src/api/index.ts index 083e17b1..19ac2550 100644 --- a/src/api/index.ts +++ b/src/api/index.ts @@ -637,6 +637,7 @@ export type { export { fetchDiagnosticsLogTail, fetchDiagnosticsSnapshot, + deleteHfRepo, fetchStorageChildren, fetchStorageTop, getModelMoveStatus, @@ -692,6 +693,8 @@ export { runBenchmark, searchHubModels, searchModels, + stopIdleModelProcesses, + stopModelProcess, unloadModel, } from "./models"; export type { diff --git a/src/api/models.ts b/src/api/models.ts index 9af969d0..e6ab6ce2 100644 --- a/src/api/models.ts +++ b/src/api/models.ts @@ -73,6 +73,14 @@ export async function unloadModel(ref?: string): Promise { return result.runtime; } +export async function stopModelProcess(pid: number): Promise<{ pid: number; action: string }> { + return await postJson<{ pid: number; action: string }>("/api/models/processes/stop", { pid }); +} + +export async function stopIdleModelProcesses(): Promise<{ warmEvicted: number; orphansStopped: number }> { + return await postJson<{ warmEvicted: number; orphansStopped: number }>("/api/models/processes/stop-idle"); +} + // --------------------------------------------------------------------------- // Downloads (text models — image / video have their own clusters) // --------------------------------------------------------------------------- diff --git a/src/components/RuntimeControls.tsx b/src/components/RuntimeControls.tsx index 1665c2ae..4a4aff1e 100644 --- a/src/components/RuntimeControls.tsx +++ b/src/components/RuntimeControls.tsx @@ -633,6 +633,11 @@ export function RuntimeControls({
+ {/* Only llama.cpp acts on these two. MLX and vLLM record Fit in memory + without using it, and Fused attention only reaches an MLX cache + strategy that is not Native, so showing them elsewhere offered + switches that did nothing. */} + {isGgufBackend ? (<>
) : null} + ) : null} + {isGgufBackend || settings.cacheStrategy !== "native" ? (<>
) : null} + ) : null} {/* FU-034 (2026-05-10): hide the DFlash toggle entirely when the selected model has no draft in DRAFT_MODEL_MAP, or when the engine is GGUF (DFlash requires MLX/vLLM). Both cases give diff --git a/src/components/TensorfoldLaunchControl.tsx b/src/components/TensorfoldLaunchControl.tsx index 03b72b98..6f9d53b7 100644 --- a/src/components/TensorfoldLaunchControl.tsx +++ b/src/components/TensorfoldLaunchControl.tsx @@ -78,7 +78,7 @@ export function TensorfoldLaunchControl({ disabled={!available} onChange={(event) => onSpeculativeChange(event.target.checked)} /> - {t("tensorfold.label", { defaultValue: "TensorFold" })} + {t("tensorfold.label", { defaultValue: "TensorFold (DFlash 2 / MTP drafting)" })} {exclusive ? ( diff --git a/src/features/chat/ChatComposer.tsx b/src/features/chat/ChatComposer.tsx index c065d011..977a882a 100644 --- a/src/features/chat/ChatComposer.tsx +++ b/src/features/chat/ChatComposer.tsx @@ -25,6 +25,8 @@ export interface ChatComposerProps { draftMessage: string; pendingImages: string[]; loadedModelRef: string | undefined; + /** A model is loading or reloading; sending would hit the previous model. */ + modelLoading?: boolean; loadedModelCapabilities?: ModelCapabilities | null; thinkingMode: ChatThinkingMode; reasoningEffort: ReasoningEffortLevel; @@ -78,6 +80,7 @@ export function ChatComposer({ draftMessage, pendingImages, loadedModelRef, + modelLoading = false, loadedModelCapabilities, thinkingMode, reasoningEffort, @@ -240,7 +243,7 @@ export function ChatComposer({ event.preventDefault(); // Mirror the Send button's disabled state — no-op when no // model is loaded so users don't trigger a confusing 500. - if (!loadedModelRef) return; + if (!loadedModelRef || modelLoading) return; void onSendMessage(); } }} @@ -412,9 +415,13 @@ export function ChatComposer({ className="primary-button" type="button" onClick={() => void onSendMessage()} - disabled={!loadedModelRef} + disabled={!loadedModelRef || modelLoading} title={ - !loadedModelRef + modelLoading + ? t("composer.sendLoadingTooltip", { + defaultValue: "Wait for the model to finish loading", + }) + : !loadedModelRef ? t("composer.sendDisabledTooltip", { defaultValue: "Load a model first to send messages", }) diff --git a/src/features/chat/ChatTab.tsx b/src/features/chat/ChatTab.tsx index f83f4769..e6a3105c 100644 --- a/src/features/chat/ChatTab.tsx +++ b/src/features/chat/ChatTab.tsx @@ -462,6 +462,7 @@ export function ChatTab({ draftMessage={draftMessage} pendingImages={pendingImages} loadedModelRef={loadedModelRef} + modelLoading={modelBusyLabel !== null} loadedModelCapabilities={loadedModelCapabilities ?? null} thinkingMode={thinkingMode} reasoningEffort={reasoningEffort} diff --git a/src/features/chat/CompareView.tsx b/src/features/chat/CompareView.tsx index 7df5110e..e21bcb50 100644 --- a/src/features/chat/CompareView.tsx +++ b/src/features/chat/CompareView.tsx @@ -28,6 +28,7 @@ import { resolvedSpeculativeMode, runtimeOutcomeWarning, } from "./runtimeDetails"; +import { newStickyScroll, stickToBottom, updateStickyScroll, type StickyScroll } from "../../utils/stickyScroll"; export const compareTargets = ["a", "b", "c", "d"] as const; export type CompareTarget = typeof compareTargets[number]; @@ -395,6 +396,7 @@ export function CompareView({ const [pickerDraftSettings, setPickerDraftSettings] = useState(() => cloneLaunchSettings(launchSettings)); const [resultAtBottom, setResultAtBottom] = useState>(emptyAtBottom); const abortRef = useRef(null); + const resultSticky = useRef>({}); const resultRefs = useRef>({ a: null, b: null, @@ -429,7 +431,7 @@ export function CompareView({ useEffect(() => { const handles = slots - .filter((slot) => resultAtBottom[slot.id]) + .filter((slot) => resultSticky.current[slot.id]?.stuck ?? true) .map((slot) => requestAnimationFrame(() => scrollResultToBottom(slot.id))); return () => handles.forEach((handle) => cancelAnimationFrame(handle)); // modelStates changes on each streamed token; that is intentional so @@ -458,14 +460,15 @@ export function CompareView({ function handleResultScroll(target: CompareTarget) { const element = resultRefs.current[target]; if (!element) return; - const atBottom = element.scrollHeight - element.scrollTop - element.clientHeight < 32; - setResultAtBottom((current) => ({ ...current, [target]: atBottom })); + const sticky = (resultSticky.current[target] ??= newStickyScroll()); + const atBottom = updateStickyScroll(element, sticky); + setResultAtBottom((current) => current[target] === atBottom ? current : { ...current, [target]: atBottom }); } function scrollResultToBottom(target: CompareTarget) { const element = resultRefs.current[target]; if (!element) return; - element.scrollTop = element.scrollHeight; + stickToBottom(element, (resultSticky.current[target] ??= newStickyScroll())); setResultAtBottom((current) => current[target] ? current : { ...current, [target]: true }); } diff --git a/src/features/chat/HtmlChallengeTab.tsx b/src/features/chat/HtmlChallengeTab.tsx index 733fe78f..4ac7c2ab 100644 --- a/src/features/chat/HtmlChallengeTab.tsx +++ b/src/features/chat/HtmlChallengeTab.tsx @@ -75,6 +75,7 @@ import { ChallengePickerModal } from "./html_challenge/ChallengePickerModal"; import { ChallengePromptLibraryModal } from "./html_challenge/ChallengePromptLibraryModal"; import { ChallengeSetupPanel } from "./html_challenge/ChallengeSetupPanel"; import { ChallengeSlotPanel } from "./html_challenge/ChallengeSlotPanel"; +import { newStickyScroll, stickToBottom, updateStickyScroll, type StickyScroll } from "../../utils/stickyScroll"; interface HtmlChallengeTabProps { modelOptions: ChatModelOption[]; @@ -149,6 +150,7 @@ export function HtmlChallengeTab({ const [pickerInitialKey, setPickerInitialKey] = useState(""); const [pickerInitialSettings, setPickerInitialSettings] = useState(() => cloneLaunchSettings(launchSettings)); const abortRef = useRef(null); + const streamSticky = useRef>({}); const streamRefs = useRef>({ a: null, b: null, @@ -168,13 +170,6 @@ export function HtmlChallengeTab({ d: null, }); const activePreviewSlotRef = useRef(null); - // FU-036: timestamp of the most recent programmatic scroll-to-bottom for - // each slot. ``handleStreamScroll`` ignores scroll events fired in the - // immediate window after one (the browser fires ``scroll`` for both - // user wheel input AND ``element.scrollTop = …`` writes; without the - // guard the post-write event re-flips ``streamAtBottom`` to true even - // when the user just scrolled away). - const lastProgrammaticScrollRef = useRef>({}); const textModelOptions = modelOptions.filter(isTextModelOption); const selectedBySlot = Object.fromEntries( @@ -282,30 +277,11 @@ export function HtmlChallengeTab({ }, []); useEffect(() => { - // FU-036 (2026-05-10): re-measure scroll position inside the rAF - // before yanking back to bottom. ``setStreamAtBottom`` from the - // ``onScroll`` handler is async, so a streaming chunk that arrives - // a few ms after the user wheel-scrolls would otherwise see the - // stale ``streamAtBottom[slot.id] === true`` from the previous - // render and snap the box back down — felt as a jerk that fought - // the user's scroll. Re-measuring against the live DOM closes the - // race; if the user has moved away in the gap, we drop tracking - // for that slot instead of stomping their scroll position. + // Follow state lives in a ref (see utils/stickyScroll): content growth + // never ends the follow, only the user scrolling up does. const handles = slots - .filter((slot) => streamAtBottom[slot.id]) - .map((slot) => requestAnimationFrame(() => { - const element = streamRefs.current[slot.id]; - if (!element) return; - const stillNearBottom = - element.scrollHeight - element.scrollTop - element.clientHeight < 32; - if (stillNearBottom) { - scrollStreamToBottom(slot.id); - } else { - setStreamAtBottom((current) => current[slot.id] - ? { ...current, [slot.id]: false } - : current); - } - })); + .filter((slot) => streamSticky.current[slot.id]?.stuck ?? true) + .map((slot) => requestAnimationFrame(() => scrollStreamToBottom(slot.id))); return () => handles.forEach((handle) => cancelAnimationFrame(handle)); }, [slots, slotStates, streamAtBottom]); @@ -349,24 +325,17 @@ export function HtmlChallengeTab({ } function handleStreamScroll(target: CompareTarget) { - // Ignore the scroll event the browser fires immediately after our - // own ``element.scrollTop = …`` write. Without this guard, the - // post-write event re-flipped ``streamAtBottom`` true and the next - // chunk would yank the box back even when the user had since - // scrolled away. - const lastProgrammatic = lastProgrammaticScrollRef.current[target] ?? 0; - if (performance.now() - lastProgrammatic < 80) return; const element = streamRefs.current[target]; if (!element) return; - const atBottom = element.scrollHeight - element.scrollTop - element.clientHeight < 32; - setStreamAtBottom((current) => ({ ...current, [target]: atBottom })); + const sticky = (streamSticky.current[target] ??= newStickyScroll()); + const atBottom = updateStickyScroll(element, sticky); + setStreamAtBottom((current) => current[target] === atBottom ? current : { ...current, [target]: atBottom }); } function scrollStreamToBottom(target: CompareTarget) { const element = streamRefs.current[target]; if (!element) return; - lastProgrammaticScrollRef.current[target] = performance.now(); - element.scrollTop = element.scrollHeight; + stickToBottom(element, (streamSticky.current[target] ??= newStickyScroll())); setStreamAtBottom((current) => current[target] ? current : { ...current, [target]: true }); } diff --git a/src/features/dashboard/DashboardTab.tsx b/src/features/dashboard/DashboardTab.tsx index ee77cd7a..e8465bf7 100644 --- a/src/features/dashboard/DashboardTab.tsx +++ b/src/features/dashboard/DashboardTab.tsx @@ -12,9 +12,20 @@ export interface DashboardTabProps { runtime: RuntimeStatus; activity: ActivityItem[]; backendOnline: boolean; + /** Stop one leftover model process (not the loaded model's). */ + onStopProcess?: (pid: number) => void | Promise; + /** Stop parked warm models and every model process nothing is using. */ + onStopIdleProcesses?: () => void | Promise; } -export function DashboardTab({ system, recommendation, runtime, activity, backendOnline }: DashboardTabProps) { +type RunningProcess = SystemStats["runningLlmProcesses"][number]; + +/** A model process this app started that serves nothing: stale, safe to stop. */ +function isStopCandidate(process: RunningProcess): boolean { + return process.owner === "ChaosEngineAI" && !process.modelStatus && process.kind !== "backend"; +} + +export function DashboardTab({ system, recommendation, runtime, activity, backendOnline, onStopProcess, onStopIdleProcesses }: DashboardTabProps) { const { t } = useTranslation("dashboard"); const warmModels = runtime.warmModels ?? []; const activeReq = runtime.activeRequests ?? 0; @@ -138,12 +149,26 @@ export function DashboardTab({ system, recommendation, runtime, activity, backen
) : null} -
+
{t("liveStats.processes.name")} {t("liveStats.processes.owner")} {t("liveStats.processes.memory")} {t("liveStats.processes.cpu")} + + {onStopIdleProcesses && (warmModels.length > 0 || system.runningLlmProcesses.some(isStopCandidate)) ? ( + + ) : null} +
{system.runningLlmProcesses.length ? ( @@ -169,6 +194,24 @@ export function DashboardTab({ system, recommendation, runtime, activity, backen {t("liveStats.processes.memoryGb", { value: number(process.memoryGb, 2) })} {t("liveStats.processes.cpuPercent", { value: number(process.cpuPercent, 0) })} + + {isStopCandidate(process) && onStopProcess ? ( + + ) : null} +
)) ) : ( diff --git a/src/features/settings/DiagnosticsPanel.tsx b/src/features/settings/DiagnosticsPanel.tsx index 1a26d5f1..cf032d87 100644 --- a/src/features/settings/DiagnosticsPanel.tsx +++ b/src/features/settings/DiagnosticsPanel.tsx @@ -3,6 +3,7 @@ import { useTranslation } from "react-i18next"; import { Panel } from "../../components/Panel"; import { fetchDiagnosticsSnapshot, + deleteHfRepo, deleteModelPath, fetchStorageChildren, fetchStorageTop, @@ -565,13 +566,14 @@ function StorageRow({ const remove = async () => { const size = entry.sizeGb !== null ? `${entry.sizeGb.toFixed(1)} GB` : "unknown size"; const confirmed = window.confirm( - `Permanently delete ${entry.path} (${size})?\n\nThis removes it from disk and cannot be undone.`, + `Permanently delete ${entry.hfRepo ?? entry.path} (${size})?\n\nThis removes it from disk and cannot be undone.`, ); if (!confirmed) return; setDeleting(true); setErr(null); try { - await deleteModelPath(entry.path); + if (entry.hfRepo) await deleteHfRepo(entry.hfRepo); + else await deleteModelPath(entry.path); setGone(true); onDeleted(entry); } catch (e) { diff --git a/src/hooks/useWorkspace.ts b/src/hooks/useWorkspace.ts index 49e157cf..76f0cd6c 100644 --- a/src/hooks/useWorkspace.ts +++ b/src/hooks/useWorkspace.ts @@ -119,6 +119,20 @@ export function useWorkspace() { return () => clearTimeout(timer); }, []); + // The backend answers the first workspace request while its library scan is + // still running (a few seconds to a minute on a big model folder) with an + // empty list and ``libraryStatus: "scanning"``. Nothing refetched afterwards, + // so a window opened right after launch showed "No models found" until the + // user reloaded it. Poll until the scan finishes. + useEffect(() => { + if (workspace.libraryStatus !== "scanning") return undefined; + const timer = setInterval(() => { + void refreshWorkspace().catch(() => undefined); + }, 2000); + return () => clearInterval(timer); + // eslint-disable-next-line react-hooks/exhaustive-deps + }, [workspace.libraryStatus]); + // Initial workspace load — retry until the backend is ready. // // The Tauri sidecar now bootstraps on a background thread so the window diff --git a/src/styles.css b/src/styles.css index 8f8e6c77..063c90ec 100644 --- a/src/styles.css +++ b/src/styles.css @@ -1196,6 +1196,14 @@ strong { font-variant-numeric: tabular-nums; } +.process-table .table-row > span:last-child { + text-align: right; +} + +.process-table .table-row { + grid-template-columns: minmax(0, 2fr) minmax(120px, 1fr) 90px 70px 84px; +} + .process-name-cell { display: flex; flex-direction: column; diff --git a/src/types/models.ts b/src/types/models.ts index 52143e65..bfad89c1 100644 --- a/src/types/models.ts +++ b/src/types/models.ts @@ -51,6 +51,8 @@ export interface LibraryItem { backend?: string | null; modelType?: string | null; sizeGb: number; + /** Size of the one GGUF that loads when the folder holds several quants. */ + loadSizeGb?: number | null; lastModified: string; actions: string[]; directoryId?: string; diff --git a/src/utils/__tests__/stickyScroll.test.ts b/src/utils/__tests__/stickyScroll.test.ts new file mode 100644 index 00000000..40d19cb4 --- /dev/null +++ b/src/utils/__tests__/stickyScroll.test.ts @@ -0,0 +1,30 @@ +import { describe, expect, it } from "vitest"; +import { newStickyScroll, stickToBottom, updateStickyScroll } from "../stickyScroll"; + +const box = (scrollHeight: number, scrollTop: number, clientHeight = 400) => ({ scrollHeight, scrollTop, clientHeight }); + +describe("stickyScroll", () => { + it("keeps following when one chunk grows the content past the threshold", () => { + const state = newStickyScroll(); + updateStickyScroll(box(1000, 600), state); + // content grows 500px, scrollTop unchanged: no scroll event moves it up + expect(updateStickyScroll(box(1500, 600), state)).toBe(true); + }); + + it("stops following when the user scrolls up, resumes at the bottom", () => { + const state = newStickyScroll(); + updateStickyScroll(box(1000, 600), state); + expect(updateStickyScroll(box(1000, 300), state)).toBe(false); + expect(updateStickyScroll(box(1200, 300), state)).toBe(false); + expect(updateStickyScroll(box(1200, 790), state)).toBe(true); + }); + + it("stickToBottom turns the follow back on", () => { + const state = newStickyScroll(); + state.stuck = false; + const el = box(2000, 0); + stickToBottom(el, state); + expect(el.scrollTop).toBe(2000); + expect(state.stuck).toBe(true); + }); +}); diff --git a/src/utils/library.ts b/src/utils/library.ts index 1e538de0..1f6c4a52 100644 --- a/src/utils/library.ts +++ b/src/utils/library.ts @@ -232,7 +232,8 @@ export function estimateLibraryItemResidentGb( item: LibraryItem, matchedVariant?: ModelVariant | null, ): number | null { - const sizeGb = Number.isFinite(item.sizeGb) && item.sizeGb > 0 ? item.sizeGb : null; + const weightsGb = item.loadSizeGb ?? item.sizeGb; + const sizeGb = Number.isFinite(weightsGb) && weightsGb > 0 ? weightsGb : null; if (sizeGb == null) { const fallback = matchedVariant?.estimatedMemoryGb; return fallback != null && Number.isFinite(fallback) ? fallback : null; @@ -253,7 +254,8 @@ export function estimateLibraryItemCompressedGb( item: LibraryItem, matchedVariant?: ModelVariant | null, ): number | null { - const sizeGb = Number.isFinite(item.sizeGb) && item.sizeGb > 0 ? item.sizeGb : null; + const weightsGb = item.loadSizeGb ?? item.sizeGb; + const sizeGb = Number.isFinite(weightsGb) && weightsGb > 0 ? weightsGb : null; if (sizeGb == null) { const fallback = matchedVariant?.estimatedCompressedMemoryGb; return fallback != null && Number.isFinite(fallback) ? fallback : null; diff --git a/src/utils/stickyScroll.ts b/src/utils/stickyScroll.ts new file mode 100644 index 00000000..4802b63a --- /dev/null +++ b/src/utils/stickyScroll.ts @@ -0,0 +1,44 @@ +/** + * Follow-the-bottom state for streaming output. + * + * Measuring "am I near the bottom" after the content has grown loses the + * follow whenever one chunk adds more than the threshold (a tall reasoning + * block, a code fence). Content growth never moves ``scrollTop``, so the + * follow only ends when the user scrolls up, and resumes when they come + * back to the bottom or press Latest. + */ +export interface StickyScroll { + top: number; + stuck: boolean; +} + +const NEAR_BOTTOM_PX = 32; + +export function newStickyScroll(): StickyScroll { + return { top: 0, stuck: true }; +} + +/** Update from a scroll event; returns whether the follow is still on. */ +export function updateStickyScroll( + element: { scrollHeight: number; scrollTop: number; clientHeight: number }, + state: StickyScroll, +): boolean { + const distance = element.scrollHeight - element.scrollTop - element.clientHeight; + if (distance < NEAR_BOTTOM_PX) { + state.stuck = true; + } else if (element.scrollTop < state.top) { + state.stuck = false; + } + state.top = element.scrollTop; + return state.stuck; +} + +/** Scroll to the end and turn the follow on. */ +export function stickToBottom( + element: { scrollHeight: number; scrollTop: number }, + state: StickyScroll, +): void { + element.scrollTop = element.scrollHeight; + state.stuck = true; + state.top = element.scrollTop; +} diff --git a/tests/test_diagnostics_routes.py b/tests/test_diagnostics_routes.py index 636699a2..a49df8a3 100644 --- a/tests/test_diagnostics_routes.py +++ b/tests/test_diagnostics_routes.py @@ -258,6 +258,53 @@ def test_shared_blob_store_is_not_deletable(self): ).json()["entries"] self.assertEqual([e["deletable"] for e in inner], [False]) + def test_shared_blob_store_is_grouped_by_repo(self): + blobs = self.models / "blobs" + (blobs / "ab").mkdir(parents=True) + (blobs / ".huggingface-shared-blobs").write_bytes(b"1") + for name, size, ref in ( + ("abc1", 300, "models--org--big/blobs/abc1"), + ("abc2", 100, "models--org--big/blobs/abc2"), + ("abc3", 50, "models--other--small/blobs/abc3"), + ): + (blobs / "ab" / name).write_bytes(b"x" * size) + (blobs / "ab" / f"{name}.refs").write_text(ref + "\n") + entries = self.client.get( + "/api/diagnostics/storage-children", params={"path": str(blobs)} + ).json()["entries"] + self.assertEqual( + [(e["repoLabel"], e["sizeBytes"]) for e in entries], + [("org/big (2 files)", 400), ("other/small (1 file)", 50)], + ) + self.assertEqual([e["hfRepo"] for e in entries], ["org/big", "other/small"]) + self.assertTrue(all(e["deletable"] for e in entries)) + + def test_delete_hf_repo_frees_it_and_keeps_other_repos(self): + import os + from unittest import mock + + hub = self.models / "hub" + for name, size in (("org--gone", 300), ("org--kept", 100)): + repo = hub / f"models--{name}" + (repo / "blobs").mkdir(parents=True) + (repo / "refs").mkdir() + (repo / "snapshots" / f"rev-{name}").mkdir(parents=True) + (repo / "blobs" / name).write_bytes(b"x" * size) + (repo / "snapshots" / f"rev-{name}" / "w.bin").symlink_to(repo / "blobs" / name) + (repo / "refs" / "main").write_text(f"rev-{name}") + with mock.patch.dict(os.environ, {"HF_HUB_CACHE": str(hub)}): + missing = self.client.post("/api/diagnostics/hf-repo/delete", json={"repo": "org/nope"}) + self.assertEqual(missing.status_code, 404) + resp = self.client.post("/api/diagnostics/hf-repo/delete", json={"repo": "org/gone"}) + self.assertEqual(resp.status_code, 200) + self.assertEqual(resp.json()["freedBytes"], 300) + self.assertFalse((hub / "models--org--gone").exists()) + self.assertTrue((hub / "models--org--kept" / "blobs" / "org--kept").exists()) + + def test_delete_hf_repo_rejects_a_path_shaped_repo(self): + resp = self.client.post("/api/diagnostics/hf-repo/delete", json={"repo": "../etc/passwd"}) + self.assertEqual(resp.status_code, 422) + def test_children_lists_files_and_folders_biggest_first(self): resp = self.client.get( "/api/diagnostics/storage-children", params={"path": str(self.models / "tool")} @@ -269,3 +316,52 @@ def test_children_refuses_paths_outside_model_directories(self): for path in (str(self.outside), str(self.models / ".." / "outside")): resp = self.client.get("/api/diagnostics/storage-children", params={"path": path}) self.assertEqual(resp.status_code, 403, path) + + +class StopProcessRouteTests(unittest.TestCase): + """The Dashboard Stop buttons: contract of the two endpoints.""" + + class _Runtime(FakeRuntime): + def stop_process(self, pid): + if pid == 1: + raise ValueError("That process serves the loaded model. Unload the model instead.") + return {"pid": pid, "action": "terminated"} + + def stop_idle(self): + return {"warmEvicted": 2, "orphansStopped": 1} + + def setUp(self): + self.tempdir = tempfile.TemporaryDirectory() + state = ChaosEngineState( + system_snapshot_provider=_fake_system_snapshot, + library_provider=lambda: [], + settings_path=Path(self.tempdir.name) / "settings.json", + benchmarks_path=Path(self.tempdir.name) / "benchmarks.json", + chat_sessions_path=Path(self.tempdir.name) / "chats.json", + library_cache_path=Path(self.tempdir.name) / "library_cache.json", + ) + state.runtime = self._Runtime() + self.client = TestClient(create_app(state=state, api_token=TEST_API_TOKEN)) + self.client.headers.update({"Authorization": f"Bearer {TEST_API_TOKEN}"}) + + def tearDown(self): + self.tempdir.cleanup() + + def test_stop_a_process(self): + resp = self.client.post("/api/models/processes/stop", json={"pid": 4321}) + self.assertEqual(resp.status_code, 200) + self.assertEqual(resp.json()["action"], "terminated") + + def test_the_loaded_models_process_is_a_conflict(self): + resp = self.client.post("/api/models/processes/stop", json={"pid": 1}) + self.assertEqual(resp.status_code, 409) + self.assertIn("Unload the model", resp.json()["detail"]) + + def test_pid_must_be_an_integer(self): + for bad in ("12", None, True): + self.assertEqual(self.client.post("/api/models/processes/stop", json={"pid": bad}).status_code, 400) + + def test_stop_idle(self): + resp = self.client.post("/api/models/processes/stop-idle") + self.assertEqual(resp.status_code, 200) + self.assertEqual(resp.json(), {"warmEvicted": 2, "orphansStopped": 1}) diff --git a/tests/test_inference.py b/tests/test_inference.py index 041fe276..3b21df50 100644 --- a/tests/test_inference.py +++ b/tests/test_inference.py @@ -1077,3 +1077,68 @@ def test_reasoning_only_reply_still_counts_and_closes(self): chunks = self._stream([{"reasoning_content": "thinking"}, {"reasoning_content": " more"}]) self.assertTrue(any(c.reasoning_done for c in chunks)) self.assertEqual(chunks[-1].completion_tokens, 2) + + +class ModelWeightsBytesTests(unittest.TestCase): + """A multi-quant GGUF folder counts as the one file that loads (warm pool, image/video gates).""" + + def test_gguf_folder_counts_the_loaded_quant_only(self) -> None: + import tempfile + from pathlib import Path + + from backend_service.inference.controller import _model_weights_bytes + + with tempfile.TemporaryDirectory() as tmp: + folder = Path(tmp) / "models--acme--Model-GGUF" / "snapshots" / "abc" + folder.mkdir(parents=True) + (folder / "Model-BF16.gguf").write_bytes(b"x" * 480) + (folder / "Model-Q4_K_M.gguf").write_bytes(b"x" * 150) + self.assertEqual(_model_weights_bytes(str(folder)), 480) # no catalog pin: the largest loads + + def test_a_folder_with_safetensors_is_sized_whole(self) -> None: + import tempfile + from pathlib import Path + + from backend_service.inference.controller import _model_weights_bytes + + with tempfile.TemporaryDirectory() as tmp: + folder = Path(tmp) / "model" + folder.mkdir() + (folder / "model.safetensors").write_bytes(b"x" * 300) + (folder / "extra.gguf").write_bytes(b"x" * 100) + self.assertEqual(_model_weights_bytes(str(folder)), 400) + + +class MtpProbeOnHubFolderTests(unittest.TestCase): + """The probe must see through a Hugging Face cache folder, not fall back to a name match.""" + + def test_repo_folder_resolves_to_its_snapshot(self) -> None: + import json + import tempfile + from pathlib import Path + + from backend_service.inference._mtp import has_mtp_heads_strict, model_has_mtp_tensors + + with tempfile.TemporaryDirectory() as tmp: + repo = Path(tmp) / "models--mlx-community--Qwen3-4B-bf16" + snap = repo / "snapshots" / "abc" + snap.mkdir(parents=True) + (snap / "model.safetensors.index.json").write_text( + json.dumps({"weight_map": {"model.layers.0.mlp.up_proj.weight": "a.safetensors"}}), encoding="utf-8" + ) + self.assertIs(model_has_mtp_tensors(str(repo)), False) + # a catalog name that carries MTP heads must not win over what is on disk + self.assertFalse(has_mtp_heads_strict("Qwen/Qwen3.5-4B", str(repo))) + + def test_single_file_checkpoint_header_is_read(self) -> None: + import json + import tempfile + from pathlib import Path + + from backend_service.inference._mtp import model_has_mtp_tensors + + with tempfile.TemporaryDirectory() as tmp: + model = Path(tmp) + header = json.dumps({"mtp.fc.weight": {"dtype": "F16"}}).encode() + (model / "model.safetensors").write_bytes(len(header).to_bytes(8, "little") + header) + self.assertIs(model_has_mtp_tensors(str(model)), True) diff --git a/tests/test_mlx_models.py b/tests/test_mlx_models.py index 5c2f7687..ad55b0d4 100644 --- a/tests/test_mlx_models.py +++ b/tests/test_mlx_models.py @@ -366,3 +366,36 @@ def apply_chat_template(self, messages, **kwargs): if __name__ == "__main__": unittest.main() + + +class FlintOnMlxTests(unittest.TestCase): + """A Flint checkpoint MLX cannot load fails with a reason and the way out.""" + + def _dir(self, *, marker: bool) -> str: + directory = Path(tempfile.mkdtemp()) + self.addCleanup(lambda: __import__("shutil").rmtree(directory, ignore_errors=True)) + (directory / "config.json").write_text(json.dumps({"model_type": "flint"}), encoding="utf-8") + if marker: + (directory / "flint-int4.json").write_text("{}", encoding="utf-8") + return str(directory) + + def test_recursives_int4_export_points_at_the_gguf(self) -> None: + from backend_service.mlx_worker_diagnostics import _reject_unsupported_quant + + with self.assertRaises(RuntimeError) as ctx: + _reject_unsupported_quant(self._dir(marker=True)) + self.assertIn("RecursiveSI/flint-int4-gguf", str(ctx.exception)) + + def test_a_stack_without_the_architecture_says_so(self) -> None: + from backend_service.mlx_worker_diagnostics import _reject_unsupported_quant + + with mock.patch("importlib.util.find_spec", return_value=None): + with self.assertRaises(RuntimeError) as ctx: + _reject_unsupported_quant(self._dir(marker=False)) + self.assertIn("do not include the Flint architecture", str(ctx.exception)) + + def test_a_stack_that_ships_it_is_left_alone(self) -> None: + from backend_service.mlx_worker_diagnostics import _reject_unsupported_quant + + with mock.patch("importlib.util.find_spec", return_value=object()): + _reject_unsupported_quant(self._dir(marker=False)) diff --git a/tests/test_non_chat_models.py b/tests/test_non_chat_models.py index 21482031..0ad5111e 100644 --- a/tests/test_non_chat_models.py +++ b/tests/test_non_chat_models.py @@ -95,3 +95,20 @@ def test_the_gguf_header_reader(self) -> None: if __name__ == "__main__": unittest.main() + + +class Fp8CompressedTensorsOnMacTests(unittest.TestCase): + def test_fp8_checkpoint_is_unsupported_on_a_mac_only(self) -> None: + from unittest import mock + + from backend_service.helpers import quantization + + config = {"quantization_config": {"quant_method": "compressed-tensors", "format": "float-quantized"}} + with mock.patch.object(quantization.sys, "platform", "darwin"): + self.assertIn("NVIDIA", quantization._unsupported_mlx_quantization_reason(config) or "") + with mock.patch.object(quantization.sys, "platform", "linux"): + self.assertIsNone(quantization._unsupported_mlx_quantization_reason(config)) + with mock.patch.object(quantization.sys, "platform", "darwin"): + self.assertIsNone(quantization._unsupported_mlx_quantization_reason( + {"quantization_config": {"quant_method": "compressed-tensors", "format": "pack-quantized"}} + )) diff --git a/tests/test_overlay_guard.py b/tests/test_overlay_guard.py index 48fc6098..3a926d67 100644 --- a/tests/test_overlay_guard.py +++ b/tests/test_overlay_guard.py @@ -54,6 +54,37 @@ def test_a_partial_old_overlay_is_moved_aside(self) -> None: self.assertFalse(any(self.extras.glob("mlx-0.31.2.dist-info"))) self.assertFalse((self.extras / "mlx").exists()) # emptied folders go too + def test_a_mixed_huggingface_hub_overlay_is_moved_whole(self) -> None: + # Two pip --target runs left 1.12.2 and 2.2.0 dist-infos over one set of files; + # transformers then sees hub 1.12.2 and refuses to import. + _install(self.bundled, "huggingface-hub", "2.2.0", ["huggingface_hub/__init__.py"]) + _install(self.extras, "huggingface-hub", "1.12.2", ["huggingface_hub/__init__.py", "huggingface_hub/old.py"]) + _install(self.extras, "huggingface-hub", "2.2.0", ["huggingface_hub/__init__.py", "huggingface_hub/new.py"]) + (self.extras / "huggingface_hub" / "__pycache__").mkdir() + (self.extras / "huggingface_hub" / "__pycache__" / "x.pyc").write_text("", encoding="utf-8") + + moved = runtime_paths.quarantine_stale_core_packages(self.extras, [self.bundled]) + + self.assertEqual(sorted(moved), ["huggingface-hub 1.12.2", "huggingface-hub 2.2.0"]) + self.assertFalse((self.extras / "huggingface_hub").exists()) + self.assertFalse(any(self.extras.glob("huggingface_hub-*.dist-info"))) + + def test_a_clean_lockstep_package_stays(self) -> None: + _install(self.bundled, "transformers", "5.20.2", ["transformers/__init__.py"]) + _install(self.extras, "transformers", "5.20.2", ["transformers/__init__.py"]) + + self.assertEqual(runtime_paths.quarantine_stale_core_packages(self.extras, [self.bundled]), []) + self.assertTrue((self.extras / "transformers" / "__init__.py").exists()) + + def test_an_older_lockstep_package_is_moved_and_the_bundle_runs(self) -> None: + _install(self.bundled, "transformers", "5.20.2", ["transformers/__init__.py"]) + _install(self.extras, "transformers", "5.7.0", ["transformers/__init__.py"]) + + moved = runtime_paths.quarantine_stale_core_packages(self.extras, [self.bundled]) + + self.assertEqual(moved, ["transformers 5.7.0"]) + self.assertFalse((self.extras / "transformers").exists()) + def test_matching_versions_and_unrelated_packages_stay(self) -> None: _install(self.extras, "mlx", "0.32.4", ["mlx/__init__.py"]) _install(self.extras, "mlx-vlm", "0.4.4", ["mlx_vlm/__init__.py"]) diff --git a/tests/test_sampler_payload.py b/tests/test_sampler_payload.py index a79f3bd2..0fa39641 100644 --- a/tests/test_sampler_payload.py +++ b/tests/test_sampler_payload.py @@ -179,3 +179,20 @@ def test_partial_override_keeps_only_set_fields(self): if __name__ == "__main__": unittest.main() + + +class ThinkingSwitchTests(unittest.TestCase): + def test_thinking_off_sends_enable_thinking_false(self): + payload: dict = {} + _apply_sampler_kwargs(payload, samplers=None, reasoning_effort=None, json_schema=None, thinking_mode="off") + self.assertEqual(payload["chat_template_kwargs"], {"enable_thinking": False}) + + def test_thinking_on_sends_enable_thinking_true(self): + payload: dict = {} + _apply_sampler_kwargs(payload, samplers=None, reasoning_effort=None, json_schema=None, thinking_mode="auto") + self.assertEqual(payload["chat_template_kwargs"], {"enable_thinking": True}) + + def test_unset_leaves_payload_alone(self): + payload: dict = {} + _apply_sampler_kwargs(payload, samplers=None, reasoning_effort=None, json_schema=None) + self.assertNotIn("chat_template_kwargs", payload) diff --git a/tests/test_tensorfold_registry.py b/tests/test_tensorfold_registry.py index c2ec700e..4b5a69bc 100644 --- a/tests/test_tensorfold_registry.py +++ b/tests/test_tensorfold_registry.py @@ -484,3 +484,36 @@ def test_startup_failure_fallbacks(self) -> None: if __name__ == "__main__": unittest.main() + + +class StockMlxStillLoadsTests(unittest.TestCase): + """An "exclusive" family stops being exclusive once the app's mlx-lm ships it.""" + + def _checkpoint(self, model_type: str): + import json + import tempfile + from pathlib import Path + + directory = Path(tempfile.mkdtemp()) + self.addCleanup(lambda: __import__("shutil").rmtree(directory, ignore_errors=True)) + (directory / "config.json").write_text(json.dumps({"model_type": model_type}), encoding="utf-8") + return directory + + def test_exclusive_when_mlx_lm_lacks_the_architecture(self) -> None: + from unittest import mock + + from backend_service.inference import _tensorfold + + with mock.patch.object(_tensorfold, "_stock_mlx_loads", return_value=False): + match = _tensorfold.match_model(model_dir=self._checkpoint("prism_hadamard_qwen35")) + self.assertIsNotNone(match) + self.assertEqual(match.tier, "exclusive") + + def test_not_exclusive_when_mlx_lm_ships_it(self) -> None: + from unittest import mock + + from backend_service.inference import _tensorfold + + with mock.patch.object(_tensorfold, "_stock_mlx_loads", return_value=True): + match = _tensorfold.match_model(model_dir=self._checkpoint("prism_hadamard_qwen35")) + self.assertIsNone(match) diff --git a/tests/test_tensorfold_routing.py b/tests/test_tensorfold_routing.py index 26d5fd6e..1a994941 100644 --- a/tests/test_tensorfold_routing.py +++ b/tests/test_tensorfold_routing.py @@ -302,3 +302,132 @@ def test_the_tracked_server_is_left_alone(self) -> None: if __name__ == "__main__": unittest.main() + + +class ColdStartNoteTests(unittest.TestCase): + def test_loading_progress_says_a_cold_start_takes_minutes(self) -> None: + from backend_service.inference import tensorfold_engine as tf + + engine = tf.TensorFoldEngine.__new__(tf.TensorFoldEngine) + engine.process = None + engine.log_path = None + messages: list[str] = [] + with mock.patch.object(tf, "_http_json", return_value={"status": "ok"}), \ + mock.patch.object(tf.TensorFoldEngine, "_server_url", return_value="http://127.0.0.1:1/health"): + engine._wait_for_server("m", lambda payload: messages.append(str(payload.get("message") or payload))) + self.assertTrue(any("cold start can take a few minutes" in m for m in messages), messages) + + +class OneLoadAtATimeTests(unittest.TestCase): + def test_a_second_request_waits_for_the_load_in_flight(self) -> None: + import threading + import time + + controller = _controller(tensorfold=True) + controller.engine = _LoadedByStub("current") # type: ignore[assignment] + finished = threading.Event() + controller._load_lock.acquire() # stands in for a model load that is still running + try: + worker = threading.Thread(target=lambda: (controller.unload_model(), finished.set()), daemon=True) + worker.start() + time.sleep(0.2) + self.assertFalse(finished.is_set(), "unload ran while a load held the lock") + finally: + controller._load_lock.release() + worker.join(timeout=2) + self.assertTrue(finished.is_set()) + + +class StopProcessTests(unittest.TestCase): + """The Stop buttons act on leftovers only and never on the loaded model or other programs.""" + + def _engine(self, pid: int): + engine = _LoadedByStub("e") + engine.process_pid = lambda: pid # type: ignore[method-assign] + return engine + + def test_the_loaded_models_process_is_refused(self) -> None: + controller = _controller(tensorfold=True) + controller.engine = self._engine(4242) # type: ignore[assignment] + with self.assertRaises(ValueError) as ctx: + controller.stop_process(4242) + self.assertIn("Unload the model", str(ctx.exception)) + + def test_a_warm_model_is_evicted_through_its_engine(self) -> None: + controller = _controller(tensorfold=True) + controller.engine = self._engine(1) # type: ignore[assignment] + warm = self._engine(5151) + controller._warm_pool["k"] = (warm, mock.Mock(name="info")) # type: ignore[assignment] + controller._warm_pool["k"][1].name = "Parked model" + with mock.patch.object(controller, "prune_stale_backend_children"): + result = controller.stop_process(5151) + self.assertEqual(result["action"], "unloaded") + self.assertEqual(warm.unloads, 1) + self.assertNotIn("k", controller._warm_pool) + + def test_a_process_this_backend_did_not_start_is_refused(self) -> None: + import os + + controller = _controller(tensorfold=True) + controller.engine = self._engine(1) # type: ignore[assignment] + # the test runner itself: a real process, but not a model process under the backend + with self.assertRaises(ValueError): + controller.stop_process(os.getpid()) + + def test_stop_idle_clears_the_warm_pool_and_prunes_without_a_grace_window(self) -> None: + controller = _controller(tensorfold=True) + controller.engine = self._engine(1) # type: ignore[assignment] + controller._warm_pool["k"] = (self._engine(7), mock.Mock()) # type: ignore[assignment] + with mock.patch.object(controller, "prune_stale_backend_children") as prune: + result = controller.stop_idle() + self.assertEqual(result["warmEvicted"], 1) + prune.assert_called_with(grace_seconds=0.0) + + +class SupersedeAndCancelTests(unittest.TestCase): + """A newer load replaces the one in flight, a repeat waits for it, and unload cancels it.""" + + class _SlowEngine(_LoadedByStub): + def __init__(self, label: str) -> None: + super().__init__(label) + self.started = __import__("threading").Event() + self.release = __import__("threading").Event() + + def load_model(self, **kwargs): + self.started.set() + if not self.release.wait(timeout=5): + raise RuntimeError("never released") + if getattr(self, "killed", False): + raise RuntimeError("worker exited unexpectedly") + return super().load_model(**kwargs) + + def unload_model(self) -> None: + self.killed = True + self.release.set() + super().unload_model() + + def _kwargs(self, repo: str) -> dict: + return dict(model_ref=repo, model_name=repo, canonical_repo=repo, source="library", backend="mlx", + path=None, runtime_target=repo) + + def test_unload_cancels_a_load_in_flight(self) -> None: + import threading + + controller = _controller(tensorfold=True) + slow = self._SlowEngine("slow") + controller._select_engine = mock.Mock(return_value=slow) + outcome: list = [] + loader = threading.Thread(target=lambda: outcome.append(self._attempt(controller, "acme/Big")), daemon=True) + loader.start() + self.assertTrue(slow.started.wait(timeout=2)) + controller.unload_model() # must not wait for the load + loader.join(timeout=3) + self.assertFalse(loader.is_alive()) + self.assertIsNone(controller.loaded_model) + self.assertIn("cancelled", str(outcome[0]).lower()) + + def _attempt(self, controller, repo): + try: + return controller.load_model(**self._kwargs(repo)) + except RuntimeError as exc: + return exc diff --git a/tests/test_text_gguf.py b/tests/test_text_gguf.py index 81ae429c..02c6c52b 100644 --- a/tests/test_text_gguf.py +++ b/tests/test_text_gguf.py @@ -251,3 +251,36 @@ def test_other_quants_in_the_cache_do_not_inflate_progress(self) -> None: scoped = hf_cache_paths._hf_repo_downloaded_bytes("acme/Model-GGUF", ["Model-Q8_0.gguf"]) self.assertEqual(whole, 507) self.assertEqual(scoped, 107) # the pinned file plus the in-progress blob + + +class LoadSizeTests(unittest.TestCase): + """A repo folder with many quants is sized by the one file that loads.""" + + def test_only_the_loaded_quant_counts(self) -> None: + import tempfile + from pathlib import Path + + from backend_service.helpers.text_gguf import gguf_load_bytes + + with tempfile.TemporaryDirectory() as tmp: + repo = Path(tmp) / "models--acme--Model-GGUF" / "snapshots" / "abc" + repo.mkdir(parents=True) + (repo / "Model-BF16.gguf").write_bytes(b"x" * 480) + (repo / "Model-Q8_0.gguf").write_bytes(b"x" * 255) + (repo / "Model-Q4_K_M.gguf").write_bytes(b"x" * 150) + (repo / "mmproj-Model.gguf").write_bytes(b"x" * 900) + # no catalog pin for this repo: the largest model file loads + self.assertEqual(gguf_load_bytes(repo), 480) + + def test_split_shards_are_summed(self) -> None: + import tempfile + from pathlib import Path + + from backend_service.helpers.text_gguf import gguf_load_bytes + + with tempfile.TemporaryDirectory() as tmp: + repo = Path(tmp) / "models--acme--Big-GGUF" / "snapshots" / "abc" + repo.mkdir(parents=True) + (repo / "Big-00001-of-00002.gguf").write_bytes(b"x" * 100) + (repo / "Big-00002-of-00002.gguf").write_bytes(b"x" * 300) + self.assertEqual(gguf_load_bytes(repo), 400)