From 2d886aeed9200885926ca6ef1af27da64c77238a Mon Sep 17 00:00:00 2001
From: Cryptopoly <31970407+cryptopoly@users.noreply.github.com>
Date: Thu, 22 Oct 2026 16:30:24 +0100
Subject: [PATCH 01/17] fix(mlx): explain why a Flint checkpoint cannot load
instead of 'Model type flint not supported'
Recursive's MLX INT4 export stores int8 embeddings in a layout only their own engine reads, and older MLX stacks lack the architecture. The worker now says so and points at the Flint GGUF.
---
backend_service/mlx_worker_diagnostics.py | 26 ++++++++++++++++++
tests/test_mlx_models.py | 33 +++++++++++++++++++++++
2 files changed, 59 insertions(+)
diff --git a/backend_service/mlx_worker_diagnostics.py b/backend_service/mlx_worker_diagnostics.py
index f9c39362..463ece3d 100644
--- a/backend_service/mlx_worker_diagnostics.py
+++ b/backend_service/mlx_worker_diagnostics.py
@@ -32,6 +32,30 @@
_UNSUPPORTED_QUANT_ALGOS = {"NVFP4", "NVINT4"}
+_FLINT_USE_GGUF = (
+ "Use the Flint INT4 GGUF (RecursiveSI/flint-int4-gguf) with the llama.cpp backend instead."
+)
+
+
+def _reject_unloadable_flint(model_dir: Path) -> None:
+ """Say why a Flint checkpoint cannot load on MLX, instead of 'Model type flint not supported'."""
+ if (model_dir / "flint-int4.json").exists():
+ raise RuntimeError(
+ "Recursive's MLX INT4 build of Flint stores its embeddings in a layout only their own "
+ "rec engine reads, so MLX cannot load it. " + _FLINT_USE_GGUF
+ )
+ shipped = [
+ name
+ for name in ("mlx_lm", "mlx_vlm")
+ if importlib.util.find_spec(f"{name}.models.flint") is not None
+ ]
+ if not shipped:
+ raise RuntimeError(
+ "The installed MLX packages do not include the Flint architecture "
+ "(mlx-vlm 0.7.10 or newer does). " + _FLINT_USE_GGUF
+ )
+
+
def _reject_unsupported_quant(model_path: str) -> None:
"""Raise early if the model uses a quantisation format MLX cannot handle."""
cfg_path = Path(model_path) / "config.json"
@@ -40,6 +64,8 @@ def _reject_unsupported_quant(model_path: str) -> None:
try:
with open(cfg_path) as f:
cfg = json.load(f)
+ if cfg.get("model_type") == "flint":
+ _reject_unloadable_flint(Path(model_path))
qcfg = cfg.get("quantization_config") or {}
algo = qcfg.get("quant_algo", "")
method = qcfg.get("quant_method", "")
diff --git a/tests/test_mlx_models.py b/tests/test_mlx_models.py
index 5c2f7687..ad55b0d4 100644
--- a/tests/test_mlx_models.py
+++ b/tests/test_mlx_models.py
@@ -366,3 +366,36 @@ def apply_chat_template(self, messages, **kwargs):
if __name__ == "__main__":
unittest.main()
+
+
+class FlintOnMlxTests(unittest.TestCase):
+ """A Flint checkpoint MLX cannot load fails with a reason and the way out."""
+
+ def _dir(self, *, marker: bool) -> str:
+ directory = Path(tempfile.mkdtemp())
+ self.addCleanup(lambda: __import__("shutil").rmtree(directory, ignore_errors=True))
+ (directory / "config.json").write_text(json.dumps({"model_type": "flint"}), encoding="utf-8")
+ if marker:
+ (directory / "flint-int4.json").write_text("{}", encoding="utf-8")
+ return str(directory)
+
+ def test_recursives_int4_export_points_at_the_gguf(self) -> None:
+ from backend_service.mlx_worker_diagnostics import _reject_unsupported_quant
+
+ with self.assertRaises(RuntimeError) as ctx:
+ _reject_unsupported_quant(self._dir(marker=True))
+ self.assertIn("RecursiveSI/flint-int4-gguf", str(ctx.exception))
+
+ def test_a_stack_without_the_architecture_says_so(self) -> None:
+ from backend_service.mlx_worker_diagnostics import _reject_unsupported_quant
+
+ with mock.patch("importlib.util.find_spec", return_value=None):
+ with self.assertRaises(RuntimeError) as ctx:
+ _reject_unsupported_quant(self._dir(marker=False))
+ self.assertIn("do not include the Flint architecture", str(ctx.exception))
+
+ def test_a_stack_that_ships_it_is_left_alone(self) -> None:
+ from backend_service.mlx_worker_diagnostics import _reject_unsupported_quant
+
+ with mock.patch("importlib.util.find_spec", return_value=object()):
+ _reject_unsupported_quant(self._dir(marker=False))
From e11f3fc583bfcf56429919fc4c4b40cbbd140894 Mon Sep 17 00:00:00 2001
From: Cryptopoly <31970407+cryptopoly@users.noreply.github.com>
Date: Thu, 22 Oct 2026 16:53:43 +0100
Subject: [PATCH 02/17] fix: the extras overlay can no longer shadow the
bundled huggingface-hub / transformers; stock-loadable models stay off
TensorFold; FP8 is flagged on a Mac
The overlay guard only covered mlx / mlx-lm / mlx-metal, so an overlay holding huggingface-hub 1.12 under a 2.2 dist-info made transformers refuse to import and turned MLX off for every model. huggingface-hub, transformers, tokenizers, safetensors and hf-xet now move aside unless the overlay holds exactly the bundled version. A TensorFold-exclusive family is exclusive only when the app's mlx-lm lacks the architecture (mlx-lm 0.32 loads Ternary Bonsai 2). A compressed-tensors FP8 checkpoint is marked unusable on macOS.
---
backend_service/helpers/quantization.py | 26 +++++++++++++++++++
backend_service/inference/_tensorfold.py | 17 +++++++++++-
backend_service/runtime_paths.py | 23 ++++++++++++++---
tests/test_non_chat_models.py | 17 ++++++++++++
tests/test_overlay_guard.py | 31 ++++++++++++++++++++++
tests/test_tensorfold_registry.py | 33 ++++++++++++++++++++++++
6 files changed, 143 insertions(+), 4 deletions(-)
diff --git a/backend_service/helpers/quantization.py b/backend_service/helpers/quantization.py
index 78fa63a2..bf655ca6 100644
--- a/backend_service/helpers/quantization.py
+++ b/backend_service/helpers/quantization.py
@@ -14,6 +14,7 @@
from __future__ import annotations
import re
+import sys
from typing import Any
@@ -51,7 +52,32 @@ def _quantization_algo_label(config: dict[str, Any] | None) -> str | None:
return None
+def _is_fp8_compressed_tensors(config: dict[str, Any] | None) -> bool:
+ """A compressed-tensors checkpoint whose weights are FP8 ("float-quantized")."""
+ if not isinstance(config, dict):
+ return False
+ payload = config.get("quantization_config")
+ if not isinstance(payload, dict) or payload.get("quant_method") != "compressed-tensors":
+ return False
+ if payload.get("format") == "float-quantized":
+ return True
+ groups = payload.get("config_groups")
+ if isinstance(groups, dict):
+ for group in groups.values():
+ weights = group.get("weights") if isinstance(group, dict) else None
+ if isinstance(weights, dict) and weights.get("type") == "float":
+ return True
+ return False
+
+
def _unsupported_mlx_quantization_reason(config: dict[str, Any] | None) -> str | None:
+ # On a Mac MLX is the only runtime, and it cannot dequantize FP8 safetensors
+ # (mlx-lm convert stops with "unsupported compressed-tensors format").
+ if sys.platform == "darwin" and _is_fp8_compressed_tensors(config):
+ return (
+ "This FP8 checkpoint (compressed-tensors) needs an NVIDIA GPU runtime such as vLLM; "
+ "MLX cannot read it. Use an MLX or GGUF build of this model on a Mac."
+ )
algo = _quantization_algo_label(config)
if not algo or algo not in _UNSUPPORTED_MLX_QUANT_ALGOS:
return None
diff --git a/backend_service/inference/_tensorfold.py b/backend_service/inference/_tensorfold.py
index 12e2fb12..79874152 100644
--- a/backend_service/inference/_tensorfold.py
+++ b/backend_service/inference/_tensorfold.py
@@ -290,12 +290,27 @@ def match_model(
return TensorFoldMatch(family=family, repo=repo, matched_by="repo")
directory = model_dir or _directory_candidate(path, runtime_target)
- family = _EXCLUSIVE_MODEL_TYPES.get(read_model_type(directory) or "")
+ model_type = read_model_type(directory) or ""
+ family = _EXCLUSIVE_MODEL_TYPES.get(model_type)
+ # "Exclusive" means stock MLX cannot load the architecture. mlx-lm 0.32 ships
+ # prism_hadamard_qwen35, so a small Ternary Bonsai 2 loads there, and
+ # TensorFold itself refuses every checkpoint but the 27B.
+ if family is not None and _stock_mlx_loads(model_type):
+ family = None
if family is not None:
return TensorFoldMatch(family=family, repo=None, matched_by="config")
return None
+def _stock_mlx_loads(model_type: str) -> bool:
+ import importlib.util
+
+ try:
+ return importlib.util.find_spec(f"mlx_lm.models.{model_type}") is not None
+ except (ImportError, ValueError):
+ return False
+
+
def _directory_candidate(*candidates: str | None) -> Path | None:
for candidate in candidates:
if not candidate:
diff --git a/backend_service/runtime_paths.py b/backend_service/runtime_paths.py
index 2e59099c..566bb23a 100644
--- a/backend_service/runtime_paths.py
+++ b/backend_service/runtime_paths.py
@@ -112,6 +112,13 @@ def ensure_extras_on_sys_path() -> list[Path]:
# brings mlx without mlx-lm) would run the new mlx-lm on the old mlx.
_BUNDLED_CORE = ("mlx", "mlx-metal", "mlx-lm")
+# Packages transformers / mlx-lm import in lock-step with each other. Successive
+# ``pip install --target`` runs into the overlay layer several versions of these
+# on top of one another (huggingface-hub 1.12 files under a 2.2 dist-info), so
+# the overlay is not just stale, it is a mix. Unless the overlay holds exactly
+# the bundled version, every copy of the package goes and the bundled one runs.
+_LOCKSTEP_CORE = ("huggingface-hub", "transformers", "tokenizers", "safetensors", "hf-xet")
+
def _normalise(name: str) -> str:
return name.lower().replace("_", "-").replace(".", "-")
@@ -161,12 +168,16 @@ def quarantine_stale_core_packages(extras_dir: Path, bundled_dirs: list[Path]) -
root = extras_dir.resolve()
moved: list[str] = []
emptied: set[Path] = set()
- for name in _BUNDLED_CORE:
+ for name in _BUNDLED_CORE + _LOCKSTEP_CORE:
want = bundled.get(name)
if want is None:
continue
- for version, info in overlay.get(name, []):
- if version == want:
+ copies = overlay.get(name, [])
+ # A lock-step package that is clean (one copy, the bundled version) stays;
+ # anything else is moved whole, including a copy that matches the bundle.
+ move_all = name in _LOCKSTEP_CORE and (len(copies) > 1 or any(v != want for v, _ in copies))
+ for version, info in copies:
+ if version == want and not move_all:
continue
target = extras_dir.parent / "stale-core" / f"{name}-{version}"
try:
@@ -192,6 +203,12 @@ def quarantine_stale_core_packages(extras_dir: Path, bundled_dirs: list[Path]) -
# the overlay root; rmdir refuses any folder that still holds files.
for folder in sorted(emptied, key=lambda p: len(p.parts), reverse=True):
while folder != root and root in folder.parents:
+ try:
+ leftovers = [child for child in folder.iterdir() if child.name != "__pycache__"]
+ if not leftovers:
+ shutil.rmtree(folder / "__pycache__", ignore_errors=True)
+ except OSError:
+ break
try:
folder.rmdir()
except OSError:
diff --git a/tests/test_non_chat_models.py b/tests/test_non_chat_models.py
index 21482031..0ad5111e 100644
--- a/tests/test_non_chat_models.py
+++ b/tests/test_non_chat_models.py
@@ -95,3 +95,20 @@ def test_the_gguf_header_reader(self) -> None:
if __name__ == "__main__":
unittest.main()
+
+
+class Fp8CompressedTensorsOnMacTests(unittest.TestCase):
+ def test_fp8_checkpoint_is_unsupported_on_a_mac_only(self) -> None:
+ from unittest import mock
+
+ from backend_service.helpers import quantization
+
+ config = {"quantization_config": {"quant_method": "compressed-tensors", "format": "float-quantized"}}
+ with mock.patch.object(quantization.sys, "platform", "darwin"):
+ self.assertIn("NVIDIA", quantization._unsupported_mlx_quantization_reason(config) or "")
+ with mock.patch.object(quantization.sys, "platform", "linux"):
+ self.assertIsNone(quantization._unsupported_mlx_quantization_reason(config))
+ with mock.patch.object(quantization.sys, "platform", "darwin"):
+ self.assertIsNone(quantization._unsupported_mlx_quantization_reason(
+ {"quantization_config": {"quant_method": "compressed-tensors", "format": "pack-quantized"}}
+ ))
diff --git a/tests/test_overlay_guard.py b/tests/test_overlay_guard.py
index 48fc6098..3a926d67 100644
--- a/tests/test_overlay_guard.py
+++ b/tests/test_overlay_guard.py
@@ -54,6 +54,37 @@ def test_a_partial_old_overlay_is_moved_aside(self) -> None:
self.assertFalse(any(self.extras.glob("mlx-0.31.2.dist-info")))
self.assertFalse((self.extras / "mlx").exists()) # emptied folders go too
+ def test_a_mixed_huggingface_hub_overlay_is_moved_whole(self) -> None:
+ # Two pip --target runs left 1.12.2 and 2.2.0 dist-infos over one set of files;
+ # transformers then sees hub 1.12.2 and refuses to import.
+ _install(self.bundled, "huggingface-hub", "2.2.0", ["huggingface_hub/__init__.py"])
+ _install(self.extras, "huggingface-hub", "1.12.2", ["huggingface_hub/__init__.py", "huggingface_hub/old.py"])
+ _install(self.extras, "huggingface-hub", "2.2.0", ["huggingface_hub/__init__.py", "huggingface_hub/new.py"])
+ (self.extras / "huggingface_hub" / "__pycache__").mkdir()
+ (self.extras / "huggingface_hub" / "__pycache__" / "x.pyc").write_text("", encoding="utf-8")
+
+ moved = runtime_paths.quarantine_stale_core_packages(self.extras, [self.bundled])
+
+ self.assertEqual(sorted(moved), ["huggingface-hub 1.12.2", "huggingface-hub 2.2.0"])
+ self.assertFalse((self.extras / "huggingface_hub").exists())
+ self.assertFalse(any(self.extras.glob("huggingface_hub-*.dist-info")))
+
+ def test_a_clean_lockstep_package_stays(self) -> None:
+ _install(self.bundled, "transformers", "5.20.2", ["transformers/__init__.py"])
+ _install(self.extras, "transformers", "5.20.2", ["transformers/__init__.py"])
+
+ self.assertEqual(runtime_paths.quarantine_stale_core_packages(self.extras, [self.bundled]), [])
+ self.assertTrue((self.extras / "transformers" / "__init__.py").exists())
+
+ def test_an_older_lockstep_package_is_moved_and_the_bundle_runs(self) -> None:
+ _install(self.bundled, "transformers", "5.20.2", ["transformers/__init__.py"])
+ _install(self.extras, "transformers", "5.7.0", ["transformers/__init__.py"])
+
+ moved = runtime_paths.quarantine_stale_core_packages(self.extras, [self.bundled])
+
+ self.assertEqual(moved, ["transformers 5.7.0"])
+ self.assertFalse((self.extras / "transformers").exists())
+
def test_matching_versions_and_unrelated_packages_stay(self) -> None:
_install(self.extras, "mlx", "0.32.4", ["mlx/__init__.py"])
_install(self.extras, "mlx-vlm", "0.4.4", ["mlx_vlm/__init__.py"])
diff --git a/tests/test_tensorfold_registry.py b/tests/test_tensorfold_registry.py
index c2ec700e..4b5a69bc 100644
--- a/tests/test_tensorfold_registry.py
+++ b/tests/test_tensorfold_registry.py
@@ -484,3 +484,36 @@ def test_startup_failure_fallbacks(self) -> None:
if __name__ == "__main__":
unittest.main()
+
+
+class StockMlxStillLoadsTests(unittest.TestCase):
+ """An "exclusive" family stops being exclusive once the app's mlx-lm ships it."""
+
+ def _checkpoint(self, model_type: str):
+ import json
+ import tempfile
+ from pathlib import Path
+
+ directory = Path(tempfile.mkdtemp())
+ self.addCleanup(lambda: __import__("shutil").rmtree(directory, ignore_errors=True))
+ (directory / "config.json").write_text(json.dumps({"model_type": model_type}), encoding="utf-8")
+ return directory
+
+ def test_exclusive_when_mlx_lm_lacks_the_architecture(self) -> None:
+ from unittest import mock
+
+ from backend_service.inference import _tensorfold
+
+ with mock.patch.object(_tensorfold, "_stock_mlx_loads", return_value=False):
+ match = _tensorfold.match_model(model_dir=self._checkpoint("prism_hadamard_qwen35"))
+ self.assertIsNotNone(match)
+ self.assertEqual(match.tier, "exclusive")
+
+ def test_not_exclusive_when_mlx_lm_ships_it(self) -> None:
+ from unittest import mock
+
+ from backend_service.inference import _tensorfold
+
+ with mock.patch.object(_tensorfold, "_stock_mlx_loads", return_value=True):
+ match = _tensorfold.match_model(model_dir=self._checkpoint("prism_hadamard_qwen35"))
+ self.assertIsNone(match)
From 6e19fd29520f5ca30777398181ce5e8d52ca78f5 Mon Sep 17 00:00:00 2001
From: Cryptopoly <31970407+cryptopoly@users.noreply.github.com>
Date: Thu, 22 Oct 2026 17:18:29 +0100
Subject: [PATCH 03/17] fix(ui): show Fit in memory and Fused attention only
where an engine acts on them
llama.cpp uses both (retry with fewer layers; --flash-attn). MLX and vLLM record Fit in memory without using it, and Fused attention reaches only a non-Native MLX cache strategy, so the switches did nothing on most models.
---
src/components/RuntimeControls.tsx | 8 ++++++++
1 file changed, 8 insertions(+)
diff --git a/src/components/RuntimeControls.tsx b/src/components/RuntimeControls.tsx
index 1665c2ae..4a4aff1e 100644
--- a/src/components/RuntimeControls.tsx
+++ b/src/components/RuntimeControls.tsx
@@ -633,6 +633,11 @@ export function RuntimeControls({
+ {/* Only llama.cpp acts on these two. MLX and vLLM record Fit in memory
+ without using it, and Fused attention only reaches an MLX cache
+ strategy that is not Native, so showing them elsewhere offered
+ switches that did nothing. */}
+ {isGgufBackend ? (<>
{system.runningLlmProcesses.length ? (
diff --git a/src/styles.css b/src/styles.css
index 02f8e348..063c90ec 100644
--- a/src/styles.css
+++ b/src/styles.css
@@ -1196,14 +1196,12 @@ strong {
font-variant-numeric: tabular-nums;
}
-.process-table .table-row {
- grid-template-columns: minmax(0, 2fr) minmax(120px, 1fr) 90px 70px 64px;
+.process-table .table-row > span:last-child {
+ text-align: right;
}
-.process-actions {
- display: flex;
- justify-content: flex-end;
- margin-bottom: 8px;
+.process-table .table-row {
+ grid-template-columns: minmax(0, 2fr) minmax(120px, 1fr) 90px 70px 84px;
}
.process-name-cell {
From 9223e3212f7f2247d66e69a07358f0fdffff636d Mon Sep 17 00:00:00 2001
From: Cryptopoly <31970407+cryptopoly@users.noreply.github.com>
Date: Thu, 22 Oct 2026 19:28:17 +0100
Subject: [PATCH 11/17] fix(runtime): unload cancels a load in flight; a newer
load replaces it instead of queueing
The load lock added earlier made every repeat request wait its turn, so a model that was slow to start (a 27B TensorFold cold start) turned each further click or chat message into another full reload in sequence, and Unload itself waited behind them. Now: unload ends the load in flight (and queued loads give up), a request for a different model aborts the one running, a repeat of the running request waits for it and reuses the result, and an aborted load does not fall back to another engine.
---
backend_service/inference/controller.py | 91 ++++++++++++++++++++++---
tests/test_tensorfold_routing.py | 49 +++++++++++++
2 files changed, 132 insertions(+), 8 deletions(-)
diff --git a/backend_service/inference/controller.py b/backend_service/inference/controller.py
index 4769cffe..2f4a84f1 100644
--- a/backend_service/inference/controller.py
+++ b/backend_service/inference/controller.py
@@ -21,7 +21,7 @@
import time
from dataclasses import dataclass
from pathlib import Path
-from threading import Lock, RLock, Thread
+from threading import Event, Lock, RLock, Thread
from collections.abc import Callable, Iterator
from typing import Any
@@ -169,6 +169,12 @@ def __init__(self, *, background_probe: bool = False) -> None:
# first was still loading, replace ``self.engine`` and orphan the first
# engine's process (a 21 GB TensorFold worker kept running, starving the next model).
self._load_lock = RLock()
+ # Newest request wins: a load for another model aborts the one in flight,
+ # a repeat of the same request waits for it, and unload cancels any load.
+ self._latest_load_key: str | None = None
+ self._inflight_load_key: str | None = None
+ self._loaded_key: str | None = None
+ self._load_aborted = Event()
self._loading_progress: dict[str, Any] | None = None
self._loading_log_tail: list[str] = []
self._recent_orphaned_workers: list[dict[str, Any]] = []
@@ -904,8 +910,63 @@ def _make_room_keeping_active(self, required_free_bytes: int) -> bool:
free += self._pop_oldest_warm_entry()
return free >= required_free_bytes
- @_one_load_at_a_time
- def load_model(
+ def load_model(self, **kwargs: Any) -> LoadedModelInfo:
+ key = self._warm_pool_key(
+ model_ref=kwargs["model_ref"],
+ runtime_target=kwargs.get("runtime_target"),
+ path=kwargs.get("path"),
+ cache_strategy=kwargs.get("cache_strategy", "native"),
+ cache_bits=kwargs.get("cache_bits", 0),
+ fp16_layers=kwargs.get("fp16_layers", 0),
+ fused_attention=kwargs.get("fused_attention", False),
+ fit_model_in_memory=kwargs.get("fit_model_in_memory", True),
+ context_tokens=kwargs.get("context_tokens", 8192),
+ speculative_decoding=kwargs.get("speculative_decoding", False),
+ )
+ self._latest_load_key = key
+ waited = False
+ if not self._load_lock.acquire(timeout=0.2):
+ waited = True
+ if self._inflight_load_key not in (None, key):
+ self._abort_inflight_load()
+ self._load_lock.acquire()
+ try:
+ if self._latest_load_key != key:
+ raise RuntimeError("A newer load request replaced this one.")
+ # A repeat of the request that just finished loading is already served;
+ # a request that did not have to wait is a deliberate reload.
+ if waited and self.loaded_model is not None and self._loaded_key == key:
+ return self.loaded_model
+ self._load_aborted.clear()
+ self._inflight_load_key = key
+ try:
+ loaded = self._load_model_locked(**kwargs)
+ finally:
+ self._inflight_load_key = None
+ self._loaded_key = key
+ return loaded
+ finally:
+ self._load_lock.release()
+
+ def _abort_inflight_load(self) -> None:
+ """Make the load in flight fail now by ending its engine's process."""
+ self._load_aborted.set()
+ engine = self.engine
+ pid = engine.process_pid() if engine is not None else None
+ if pid:
+ try:
+ import psutil
+
+ psutil.Process(int(pid)).kill()
+ return
+ except Exception: # noqa: BLE001 - already gone
+ pass
+ try:
+ engine.unload_model()
+ except Exception: # noqa: BLE001
+ pass
+
+ def _load_model_locked(
self,
*,
model_ref: str,
@@ -1032,6 +1093,13 @@ def _internal_progress(progress: dict[str, Any]) -> None:
try:
loaded = self.engine.load_model(**_load_kwargs)
except RuntimeError as _startup_exc:
+ if self._load_aborted.is_set():
+ self.loaded_model = None
+ self.runtime_note = None
+ self._loading_progress = None
+ self._loading_log_tail = []
+ self.prune_stale_backend_children()
+ raise RuntimeError("Model load cancelled.") from _startup_exc
# An accelerator engine's startup failure → fall back to the
# standard MLX worker (see ``_startup_fallback_label``).
fallback_from = self._startup_fallback_label(
@@ -1110,12 +1178,19 @@ def update_profile(
def loading_progress(self) -> tuple[dict[str, Any] | None, list[str]]:
return self._loading_progress, list(self._loading_log_tail)
- @_one_load_at_a_time
def unload_model(self) -> None:
- self.engine.unload_model()
- self.loaded_model = None
- self.runtime_note = None
- self.prune_stale_backend_children()
+ self._latest_load_key = None # queued loads give up instead of reloading
+ if not self._load_lock.acquire(timeout=0.2):
+ self._abort_inflight_load()
+ self._load_lock.acquire()
+ try:
+ self.engine.unload_model()
+ self.loaded_model = None
+ self._loaded_key = None
+ self.runtime_note = None
+ self.prune_stale_backend_children()
+ finally:
+ self._load_lock.release()
def generate(
self,
diff --git a/tests/test_tensorfold_routing.py b/tests/test_tensorfold_routing.py
index 336bb03f..1a994941 100644
--- a/tests/test_tensorfold_routing.py
+++ b/tests/test_tensorfold_routing.py
@@ -382,3 +382,52 @@ def test_stop_idle_clears_the_warm_pool_and_prunes_without_a_grace_window(self)
result = controller.stop_idle()
self.assertEqual(result["warmEvicted"], 1)
prune.assert_called_with(grace_seconds=0.0)
+
+
+class SupersedeAndCancelTests(unittest.TestCase):
+ """A newer load replaces the one in flight, a repeat waits for it, and unload cancels it."""
+
+ class _SlowEngine(_LoadedByStub):
+ def __init__(self, label: str) -> None:
+ super().__init__(label)
+ self.started = __import__("threading").Event()
+ self.release = __import__("threading").Event()
+
+ def load_model(self, **kwargs):
+ self.started.set()
+ if not self.release.wait(timeout=5):
+ raise RuntimeError("never released")
+ if getattr(self, "killed", False):
+ raise RuntimeError("worker exited unexpectedly")
+ return super().load_model(**kwargs)
+
+ def unload_model(self) -> None:
+ self.killed = True
+ self.release.set()
+ super().unload_model()
+
+ def _kwargs(self, repo: str) -> dict:
+ return dict(model_ref=repo, model_name=repo, canonical_repo=repo, source="library", backend="mlx",
+ path=None, runtime_target=repo)
+
+ def test_unload_cancels_a_load_in_flight(self) -> None:
+ import threading
+
+ controller = _controller(tensorfold=True)
+ slow = self._SlowEngine("slow")
+ controller._select_engine = mock.Mock(return_value=slow)
+ outcome: list = []
+ loader = threading.Thread(target=lambda: outcome.append(self._attempt(controller, "acme/Big")), daemon=True)
+ loader.start()
+ self.assertTrue(slow.started.wait(timeout=2))
+ controller.unload_model() # must not wait for the load
+ loader.join(timeout=3)
+ self.assertFalse(loader.is_alive())
+ self.assertIsNone(controller.loaded_model)
+ self.assertIn("cancelled", str(outcome[0]).lower())
+
+ def _attempt(self, controller, repo):
+ try:
+ return controller.load_model(**self._kwargs(repo))
+ except RuntimeError as exc:
+ return exc
From 65afc6ddeb3c2d645cbd4a022177fe3b890c7cd4 Mon Sep 17 00:00:00 2001
From: Cryptopoly <31970407+cryptopoly@users.noreply.github.com>
Date: Thu, 22 Oct 2026 19:45:25 +0100
Subject: [PATCH 12/17] fix(chat): block send while a model loads; sticky
follow-the-bottom for streams; per-request enable_thinking for llama.cpp
---
.venv | 1 +
backend_service/inference/llama_cpp_engine.py | 7 +++
node_modules | 1 +
src/App.tsx | 26 ++++++----
src/features/chat/ChatComposer.tsx | 13 +++--
src/features/chat/ChatTab.tsx | 1 +
src/features/chat/CompareView.tsx | 11 ++--
src/features/chat/HtmlChallengeTab.tsx | 51 ++++---------------
src/utils/__tests__/stickyScroll.test.ts | 30 +++++++++++
src/utils/stickyScroll.ts | 44 ++++++++++++++++
tests/test_sampler_payload.py | 17 +++++++
11 files changed, 145 insertions(+), 57 deletions(-)
create mode 120000 .venv
create mode 120000 node_modules
create mode 100644 src/utils/__tests__/stickyScroll.test.ts
create mode 100644 src/utils/stickyScroll.ts
diff --git a/.venv b/.venv
new file mode 120000
index 00000000..d9a17cf1
--- /dev/null
+++ b/.venv
@@ -0,0 +1 @@
+../ChaosEngineAI/.venv
\ No newline at end of file
diff --git a/backend_service/inference/llama_cpp_engine.py b/backend_service/inference/llama_cpp_engine.py
index 33b505ec..42e4dfea 100644
--- a/backend_service/inference/llama_cpp_engine.py
+++ b/backend_service/inference/llama_cpp_engine.py
@@ -156,6 +156,7 @@ def _apply_sampler_kwargs(
samplers: dict[str, Any] | None,
reasoning_effort: str | None,
json_schema: dict[str, Any] | None,
+ thinking_mode: str | None = None,
) -> None:
"""Merge Phase 2.2 sampler overrides into a chat-completions payload.
@@ -172,6 +173,11 @@ def _apply_sampler_kwargs(
payload[key] = value
if reasoning_effort:
payload["reasoning_effort"] = reasoning_effort
+ if thinking_mode is not None:
+ # Templates that gate reasoning on ``enable_thinking`` (Qwen3 family,
+ # LFM2.5, ...) reason by default; the server-wide ``--reasoning off``
+ # does not reach every one, so say it per request.
+ payload["chat_template_kwargs"] = {"enable_thinking": thinking_mode != "off"}
if json_schema:
payload["response_format"] = {
"type": "json_schema",
@@ -974,6 +980,7 @@ def stream_generate(
samplers=samplers,
reasoning_effort=reasoning_effort,
json_schema=json_schema,
+ thinking_mode=thinking_mode,
)
url = self._server_url("/v1/chat/completions")
data = json.dumps(payload).encode("utf-8")
diff --git a/node_modules b/node_modules
new file mode 120000
index 00000000..1447ca2a
--- /dev/null
+++ b/node_modules
@@ -0,0 +1 @@
+../ChaosEngineAI/node_modules
\ No newline at end of file
diff --git a/src/App.tsx b/src/App.tsx
index 49535817..fbd8cc86 100644
--- a/src/App.tsx
+++ b/src/App.tsx
@@ -120,6 +120,7 @@ import {
import { useMtplxInstall } from "./hooks/useMtplxInstall";
import { useTensorfoldInstall } from "./hooks/useTensorfoldInstall";
import type { TensorfoldLaunchControls } from "./components/tensorfoldSupport";
+import { newStickyScroll, stickToBottom, updateStickyScroll, type StickyScroll } from "./utils/stickyScroll";
export default function App() {
// FU-042: i18n hook — used for the workspace header tab label /
@@ -892,20 +893,27 @@ export default function App() {
});
}, [benchmarkOption?.key, setBenchmarkDraft]);
- // Auto-scroll chat
+ // Auto-scroll chat: follow the stream until the user scrolls up (see
+ // utils/stickyScroll). A new chat or a new message turns the follow on.
+ const chatSticky = useRef(newStickyScroll());
const lastMessageLength = activeChat?.messages[activeChat.messages.length - 1]?.text?.length ?? 0;
+ useEffect(() => {
+ if (activeTab !== "chat") return;
+ const el = chatScrollRef.current;
+ if (!el) return;
+ const onScroll = () => { updateStickyScroll(el, chatSticky.current); };
+ el.addEventListener("scroll", onScroll, { passive: true });
+ return () => el.removeEventListener("scroll", onScroll);
+ }, [activeTab, activeChat?.id, chatScrollRef]);
+ useEffect(() => {
+ chatSticky.current.stuck = true;
+ }, [activeChat?.id, activeChat?.messages.length]);
useEffect(() => {
if (activeTab !== "chat") return;
const handle = requestAnimationFrame(() => {
const el = chatScrollRef.current;
- if (!el) return;
- // Only auto-scroll if the user is already near the bottom.
- // This lets users scroll up to read earlier content during streaming
- // without being yanked back to the bottom on every token.
- const nearBottom = el.scrollHeight - el.scrollTop - el.clientHeight < 120;
- if (nearBottom) {
- el.scrollTop = el.scrollHeight;
- }
+ if (!el || !chatSticky.current.stuck) return;
+ stickToBottom(el, chatSticky.current);
});
return () => cancelAnimationFrame(handle);
}, [activeTab, activeChat?.id, activeChat?.messages.length, lastMessageLength, chatScrollRef]);
diff --git a/src/features/chat/ChatComposer.tsx b/src/features/chat/ChatComposer.tsx
index c065d011..977a882a 100644
--- a/src/features/chat/ChatComposer.tsx
+++ b/src/features/chat/ChatComposer.tsx
@@ -25,6 +25,8 @@ export interface ChatComposerProps {
draftMessage: string;
pendingImages: string[];
loadedModelRef: string | undefined;
+ /** A model is loading or reloading; sending would hit the previous model. */
+ modelLoading?: boolean;
loadedModelCapabilities?: ModelCapabilities | null;
thinkingMode: ChatThinkingMode;
reasoningEffort: ReasoningEffortLevel;
@@ -78,6 +80,7 @@ export function ChatComposer({
draftMessage,
pendingImages,
loadedModelRef,
+ modelLoading = false,
loadedModelCapabilities,
thinkingMode,
reasoningEffort,
@@ -240,7 +243,7 @@ export function ChatComposer({
event.preventDefault();
// Mirror the Send button's disabled state — no-op when no
// model is loaded so users don't trigger a confusing 500.
- if (!loadedModelRef) return;
+ if (!loadedModelRef || modelLoading) return;
void onSendMessage();
}
}}
@@ -412,9 +415,13 @@ export function ChatComposer({
className="primary-button"
type="button"
onClick={() => void onSendMessage()}
- disabled={!loadedModelRef}
+ disabled={!loadedModelRef || modelLoading}
title={
- !loadedModelRef
+ modelLoading
+ ? t("composer.sendLoadingTooltip", {
+ defaultValue: "Wait for the model to finish loading",
+ })
+ : !loadedModelRef
? t("composer.sendDisabledTooltip", {
defaultValue: "Load a model first to send messages",
})
diff --git a/src/features/chat/ChatTab.tsx b/src/features/chat/ChatTab.tsx
index f83f4769..e6a3105c 100644
--- a/src/features/chat/ChatTab.tsx
+++ b/src/features/chat/ChatTab.tsx
@@ -462,6 +462,7 @@ export function ChatTab({
draftMessage={draftMessage}
pendingImages={pendingImages}
loadedModelRef={loadedModelRef}
+ modelLoading={modelBusyLabel !== null}
loadedModelCapabilities={loadedModelCapabilities ?? null}
thinkingMode={thinkingMode}
reasoningEffort={reasoningEffort}
diff --git a/src/features/chat/CompareView.tsx b/src/features/chat/CompareView.tsx
index 7df5110e..e21bcb50 100644
--- a/src/features/chat/CompareView.tsx
+++ b/src/features/chat/CompareView.tsx
@@ -28,6 +28,7 @@ import {
resolvedSpeculativeMode,
runtimeOutcomeWarning,
} from "./runtimeDetails";
+import { newStickyScroll, stickToBottom, updateStickyScroll, type StickyScroll } from "../../utils/stickyScroll";
export const compareTargets = ["a", "b", "c", "d"] as const;
export type CompareTarget = typeof compareTargets[number];
@@ -395,6 +396,7 @@ export function CompareView({
const [pickerDraftSettings, setPickerDraftSettings] = useState(() => cloneLaunchSettings(launchSettings));
const [resultAtBottom, setResultAtBottom] = useState>(emptyAtBottom);
const abortRef = useRef(null);
+ const resultSticky = useRef>({});
const resultRefs = useRef>({
a: null,
b: null,
@@ -429,7 +431,7 @@ export function CompareView({
useEffect(() => {
const handles = slots
- .filter((slot) => resultAtBottom[slot.id])
+ .filter((slot) => resultSticky.current[slot.id]?.stuck ?? true)
.map((slot) => requestAnimationFrame(() => scrollResultToBottom(slot.id)));
return () => handles.forEach((handle) => cancelAnimationFrame(handle));
// modelStates changes on each streamed token; that is intentional so
@@ -458,14 +460,15 @@ export function CompareView({
function handleResultScroll(target: CompareTarget) {
const element = resultRefs.current[target];
if (!element) return;
- const atBottom = element.scrollHeight - element.scrollTop - element.clientHeight < 32;
- setResultAtBottom((current) => ({ ...current, [target]: atBottom }));
+ const sticky = (resultSticky.current[target] ??= newStickyScroll());
+ const atBottom = updateStickyScroll(element, sticky);
+ setResultAtBottom((current) => current[target] === atBottom ? current : { ...current, [target]: atBottom });
}
function scrollResultToBottom(target: CompareTarget) {
const element = resultRefs.current[target];
if (!element) return;
- element.scrollTop = element.scrollHeight;
+ stickToBottom(element, (resultSticky.current[target] ??= newStickyScroll()));
setResultAtBottom((current) => current[target] ? current : { ...current, [target]: true });
}
diff --git a/src/features/chat/HtmlChallengeTab.tsx b/src/features/chat/HtmlChallengeTab.tsx
index 733fe78f..4ac7c2ab 100644
--- a/src/features/chat/HtmlChallengeTab.tsx
+++ b/src/features/chat/HtmlChallengeTab.tsx
@@ -75,6 +75,7 @@ import { ChallengePickerModal } from "./html_challenge/ChallengePickerModal";
import { ChallengePromptLibraryModal } from "./html_challenge/ChallengePromptLibraryModal";
import { ChallengeSetupPanel } from "./html_challenge/ChallengeSetupPanel";
import { ChallengeSlotPanel } from "./html_challenge/ChallengeSlotPanel";
+import { newStickyScroll, stickToBottom, updateStickyScroll, type StickyScroll } from "../../utils/stickyScroll";
interface HtmlChallengeTabProps {
modelOptions: ChatModelOption[];
@@ -149,6 +150,7 @@ export function HtmlChallengeTab({
const [pickerInitialKey, setPickerInitialKey] = useState("");
const [pickerInitialSettings, setPickerInitialSettings] = useState(() => cloneLaunchSettings(launchSettings));
const abortRef = useRef(null);
+ const streamSticky = useRef>({});
const streamRefs = useRef>({
a: null,
b: null,
@@ -168,13 +170,6 @@ export function HtmlChallengeTab({
d: null,
});
const activePreviewSlotRef = useRef(null);
- // FU-036: timestamp of the most recent programmatic scroll-to-bottom for
- // each slot. ``handleStreamScroll`` ignores scroll events fired in the
- // immediate window after one (the browser fires ``scroll`` for both
- // user wheel input AND ``element.scrollTop = …`` writes; without the
- // guard the post-write event re-flips ``streamAtBottom`` to true even
- // when the user just scrolled away).
- const lastProgrammaticScrollRef = useRef>({});
const textModelOptions = modelOptions.filter(isTextModelOption);
const selectedBySlot = Object.fromEntries(
@@ -282,30 +277,11 @@ export function HtmlChallengeTab({
}, []);
useEffect(() => {
- // FU-036 (2026-05-10): re-measure scroll position inside the rAF
- // before yanking back to bottom. ``setStreamAtBottom`` from the
- // ``onScroll`` handler is async, so a streaming chunk that arrives
- // a few ms after the user wheel-scrolls would otherwise see the
- // stale ``streamAtBottom[slot.id] === true`` from the previous
- // render and snap the box back down — felt as a jerk that fought
- // the user's scroll. Re-measuring against the live DOM closes the
- // race; if the user has moved away in the gap, we drop tracking
- // for that slot instead of stomping their scroll position.
+ // Follow state lives in a ref (see utils/stickyScroll): content growth
+ // never ends the follow, only the user scrolling up does.
const handles = slots
- .filter((slot) => streamAtBottom[slot.id])
- .map((slot) => requestAnimationFrame(() => {
- const element = streamRefs.current[slot.id];
- if (!element) return;
- const stillNearBottom =
- element.scrollHeight - element.scrollTop - element.clientHeight < 32;
- if (stillNearBottom) {
- scrollStreamToBottom(slot.id);
- } else {
- setStreamAtBottom((current) => current[slot.id]
- ? { ...current, [slot.id]: false }
- : current);
- }
- }));
+ .filter((slot) => streamSticky.current[slot.id]?.stuck ?? true)
+ .map((slot) => requestAnimationFrame(() => scrollStreamToBottom(slot.id)));
return () => handles.forEach((handle) => cancelAnimationFrame(handle));
}, [slots, slotStates, streamAtBottom]);
@@ -349,24 +325,17 @@ export function HtmlChallengeTab({
}
function handleStreamScroll(target: CompareTarget) {
- // Ignore the scroll event the browser fires immediately after our
- // own ``element.scrollTop = …`` write. Without this guard, the
- // post-write event re-flipped ``streamAtBottom`` true and the next
- // chunk would yank the box back even when the user had since
- // scrolled away.
- const lastProgrammatic = lastProgrammaticScrollRef.current[target] ?? 0;
- if (performance.now() - lastProgrammatic < 80) return;
const element = streamRefs.current[target];
if (!element) return;
- const atBottom = element.scrollHeight - element.scrollTop - element.clientHeight < 32;
- setStreamAtBottom((current) => ({ ...current, [target]: atBottom }));
+ const sticky = (streamSticky.current[target] ??= newStickyScroll());
+ const atBottom = updateStickyScroll(element, sticky);
+ setStreamAtBottom((current) => current[target] === atBottom ? current : { ...current, [target]: atBottom });
}
function scrollStreamToBottom(target: CompareTarget) {
const element = streamRefs.current[target];
if (!element) return;
- lastProgrammaticScrollRef.current[target] = performance.now();
- element.scrollTop = element.scrollHeight;
+ stickToBottom(element, (streamSticky.current[target] ??= newStickyScroll()));
setStreamAtBottom((current) => current[target] ? current : { ...current, [target]: true });
}
diff --git a/src/utils/__tests__/stickyScroll.test.ts b/src/utils/__tests__/stickyScroll.test.ts
new file mode 100644
index 00000000..40d19cb4
--- /dev/null
+++ b/src/utils/__tests__/stickyScroll.test.ts
@@ -0,0 +1,30 @@
+import { describe, expect, it } from "vitest";
+import { newStickyScroll, stickToBottom, updateStickyScroll } from "../stickyScroll";
+
+const box = (scrollHeight: number, scrollTop: number, clientHeight = 400) => ({ scrollHeight, scrollTop, clientHeight });
+
+describe("stickyScroll", () => {
+ it("keeps following when one chunk grows the content past the threshold", () => {
+ const state = newStickyScroll();
+ updateStickyScroll(box(1000, 600), state);
+ // content grows 500px, scrollTop unchanged: no scroll event moves it up
+ expect(updateStickyScroll(box(1500, 600), state)).toBe(true);
+ });
+
+ it("stops following when the user scrolls up, resumes at the bottom", () => {
+ const state = newStickyScroll();
+ updateStickyScroll(box(1000, 600), state);
+ expect(updateStickyScroll(box(1000, 300), state)).toBe(false);
+ expect(updateStickyScroll(box(1200, 300), state)).toBe(false);
+ expect(updateStickyScroll(box(1200, 790), state)).toBe(true);
+ });
+
+ it("stickToBottom turns the follow back on", () => {
+ const state = newStickyScroll();
+ state.stuck = false;
+ const el = box(2000, 0);
+ stickToBottom(el, state);
+ expect(el.scrollTop).toBe(2000);
+ expect(state.stuck).toBe(true);
+ });
+});
diff --git a/src/utils/stickyScroll.ts b/src/utils/stickyScroll.ts
new file mode 100644
index 00000000..4802b63a
--- /dev/null
+++ b/src/utils/stickyScroll.ts
@@ -0,0 +1,44 @@
+/**
+ * Follow-the-bottom state for streaming output.
+ *
+ * Measuring "am I near the bottom" after the content has grown loses the
+ * follow whenever one chunk adds more than the threshold (a tall reasoning
+ * block, a code fence). Content growth never moves ``scrollTop``, so the
+ * follow only ends when the user scrolls up, and resumes when they come
+ * back to the bottom or press Latest.
+ */
+export interface StickyScroll {
+ top: number;
+ stuck: boolean;
+}
+
+const NEAR_BOTTOM_PX = 32;
+
+export function newStickyScroll(): StickyScroll {
+ return { top: 0, stuck: true };
+}
+
+/** Update from a scroll event; returns whether the follow is still on. */
+export function updateStickyScroll(
+ element: { scrollHeight: number; scrollTop: number; clientHeight: number },
+ state: StickyScroll,
+): boolean {
+ const distance = element.scrollHeight - element.scrollTop - element.clientHeight;
+ if (distance < NEAR_BOTTOM_PX) {
+ state.stuck = true;
+ } else if (element.scrollTop < state.top) {
+ state.stuck = false;
+ }
+ state.top = element.scrollTop;
+ return state.stuck;
+}
+
+/** Scroll to the end and turn the follow on. */
+export function stickToBottom(
+ element: { scrollHeight: number; scrollTop: number },
+ state: StickyScroll,
+): void {
+ element.scrollTop = element.scrollHeight;
+ state.stuck = true;
+ state.top = element.scrollTop;
+}
diff --git a/tests/test_sampler_payload.py b/tests/test_sampler_payload.py
index a79f3bd2..0fa39641 100644
--- a/tests/test_sampler_payload.py
+++ b/tests/test_sampler_payload.py
@@ -179,3 +179,20 @@ def test_partial_override_keeps_only_set_fields(self):
if __name__ == "__main__":
unittest.main()
+
+
+class ThinkingSwitchTests(unittest.TestCase):
+ def test_thinking_off_sends_enable_thinking_false(self):
+ payload: dict = {}
+ _apply_sampler_kwargs(payload, samplers=None, reasoning_effort=None, json_schema=None, thinking_mode="off")
+ self.assertEqual(payload["chat_template_kwargs"], {"enable_thinking": False})
+
+ def test_thinking_on_sends_enable_thinking_true(self):
+ payload: dict = {}
+ _apply_sampler_kwargs(payload, samplers=None, reasoning_effort=None, json_schema=None, thinking_mode="auto")
+ self.assertEqual(payload["chat_template_kwargs"], {"enable_thinking": True})
+
+ def test_unset_leaves_payload_alone(self):
+ payload: dict = {}
+ _apply_sampler_kwargs(payload, samplers=None, reasoning_effort=None, json_schema=None)
+ self.assertNotIn("chat_template_kwargs", payload)
From bc24a1d09b66cec0ee487152d865a73c67d987fe Mon Sep 17 00:00:00 2001
From: Cryptopoly <31970407+cryptopoly@users.noreply.github.com>
Date: Mon, 26 Oct 2026 07:47:01 +0000
Subject: [PATCH 13/17] feat(diagnostics): break the shared Hugging Face blob
store down by repo in Disk usage
---
backend_service/routes/diagnostics.py | 57 +++++++++++++++++++++++++++
tests/test_diagnostics_routes.py | 20 ++++++++++
2 files changed, 77 insertions(+)
diff --git a/backend_service/routes/diagnostics.py b/backend_service/routes/diagnostics.py
index 2cd4655b..13e6b979 100644
--- a/backend_service/routes/diagnostics.py
+++ b/backend_service/routes/diagnostics.py
@@ -265,6 +265,59 @@ def diagnostics_storage_top(request: Request, limit: int = 20) -> dict[str, Any]
}
+def _shared_blobs_by_repo(blobs_root: Path, source_label: str) -> list[dict[str, Any]]:
+ """Group the shared Hugging Face blob store by the repo each blob belongs to.
+
+ huggingface_hub 1.x keeps one ```` file plus a ``.refs`` note
+ per blob, sharded under two-letter folders; the note names the repo
+ (``models--org--name/blobs/``) that fetched it. A blob several repos
+ share is counted once, under the first repo listed.
+ """
+ groups: dict[str, dict[str, Any]] = {}
+ try:
+ shards = [d for d in blobs_root.iterdir() if d.is_dir()]
+ except OSError:
+ return []
+ for shard in shards:
+ try:
+ files = list(shard.iterdir())
+ except OSError:
+ continue
+ for blob in files:
+ if blob.suffix == ".refs" or not blob.is_file():
+ continue
+ try:
+ size = blob.stat().st_size
+ refs = [
+ line.split("/", 1)[0]
+ for line in blob.with_name(blob.name + ".refs").read_text().splitlines()
+ if line.strip()
+ ]
+ except OSError:
+ continue
+ repo_dir = refs[0] if refs else "unreferenced"
+ kind, _, rest = repo_dir.partition("--")
+ label = rest.replace("--", "/", 1) if kind in {"models", "datasets", "spaces"} and rest else repo_dir
+ group = groups.setdefault(label, {"size": 0, "mtime": 0.0, "files": 0, "shared": 0, "kind": kind})
+ group["size"] += size
+ group["files"] += 1
+ group["shared"] += 1 if len(set(refs)) > 1 else 0
+ group["mtime"] = max(group["mtime"], blob.stat().st_mtime)
+ return [
+ {
+ "path": str(blobs_root),
+ "repoLabel": f"{label} ({g['files']} file{'s' if g['files'] != 1 else ''})",
+ "sizeBytes": g["size"],
+ "sizeGb": _bytes_to_gb(g["size"]),
+ "sourceKind": source_label,
+ "lastModified": g["mtime"],
+ "expandable": False,
+ "deletable": False,
+ }
+ for label, g in groups.items()
+ ]
+
+
@router.get("/api/diagnostics/storage-children")
def diagnostics_storage_children(request: Request, path: str) -> dict[str, Any]:
"""Direct children of one folder from the Disk usage table, biggest first.
@@ -284,6 +337,10 @@ def diagnostics_storage_children(request: Request, path: str) -> dict[str, Any]:
break
else:
raise HTTPException(status_code=403, detail="Path is outside the configured model directories.")
+ if (target / ".huggingface-shared-blobs").exists():
+ grouped = _shared_blobs_by_repo(target, source_label)
+ grouped.sort(key=lambda e: e["sizeBytes"], reverse=True)
+ return {"path": str(target), "entries": grouped[:200]}
try:
children = list(target.iterdir())
except OSError as exc:
diff --git a/tests/test_diagnostics_routes.py b/tests/test_diagnostics_routes.py
index 0eaaec4d..091c08a7 100644
--- a/tests/test_diagnostics_routes.py
+++ b/tests/test_diagnostics_routes.py
@@ -258,6 +258,26 @@ def test_shared_blob_store_is_not_deletable(self):
).json()["entries"]
self.assertEqual([e["deletable"] for e in inner], [False])
+ def test_shared_blob_store_is_grouped_by_repo(self):
+ blobs = self.models / "blobs"
+ (blobs / "ab").mkdir(parents=True)
+ (blobs / ".huggingface-shared-blobs").write_bytes(b"1")
+ for name, size, ref in (
+ ("abc1", 300, "models--org--big/blobs/abc1"),
+ ("abc2", 100, "models--org--big/blobs/abc2"),
+ ("abc3", 50, "models--other--small/blobs/abc3"),
+ ):
+ (blobs / "ab" / name).write_bytes(b"x" * size)
+ (blobs / "ab" / f"{name}.refs").write_text(ref + "\n")
+ entries = self.client.get(
+ "/api/diagnostics/storage-children", params={"path": str(blobs)}
+ ).json()["entries"]
+ self.assertEqual(
+ [(e["repoLabel"], e["sizeBytes"]) for e in entries],
+ [("org/big (2 files)", 400), ("other/small (1 file)", 50)],
+ )
+ self.assertFalse(any(e["deletable"] for e in entries))
+
def test_children_lists_files_and_folders_biggest_first(self):
resp = self.client.get(
"/api/diagnostics/storage-children", params={"path": str(self.models / "tool")}
From 9ff2babd565e6a0c0034249f544d3819fd6f4cd1 Mon Sep 17 00:00:00 2001
From: Cryptopoly <31970407+cryptopoly@users.noreply.github.com>
Date: Mon, 26 Oct 2026 08:05:43 +0000
Subject: [PATCH 14/17] feat(diagnostics): delete a cached Hugging Face repo
from Disk usage through the cache API
---
backend_service/routes/diagnostics.py | 43 +++++++++++++++++++++-
src/api/admin.ts | 6 +++
src/api/index.ts | 1 +
src/features/settings/DiagnosticsPanel.tsx | 6 ++-
tests/test_diagnostics_routes.py | 29 ++++++++++++++-
5 files changed, 81 insertions(+), 4 deletions(-)
diff --git a/backend_service/routes/diagnostics.py b/backend_service/routes/diagnostics.py
index 13e6b979..902beb00 100644
--- a/backend_service/routes/diagnostics.py
+++ b/backend_service/routes/diagnostics.py
@@ -31,6 +31,7 @@
from typing import Any
from fastapi import APIRouter, HTTPException, Request
+from pydantic import BaseModel, Field
router = APIRouter()
@@ -198,8 +199,10 @@ def _storage_entry(child: Path, source_label: str, *, dirs_only: bool) -> dict[s
if repo_label.startswith("models--"):
# HF cache layout: ``models--{org}--{repo}`` -> ``{org}/{repo}``
repo_label = repo_label[len("models--"):].replace("--", "/", 1)
+ hf_repo = repo_label if child.name.startswith("models--") and "/" in repo_label else None
return {
"path": str(child),
+ "hfRepo": hf_repo,
"repoLabel": repo_label,
"sizeBytes": size_bytes,
"sizeGb": _bytes_to_gb(size_bytes),
@@ -312,12 +315,50 @@ def _shared_blobs_by_repo(blobs_root: Path, source_label: str) -> list[dict[str,
"sourceKind": source_label,
"lastModified": g["mtime"],
"expandable": False,
- "deletable": False,
+ "deletable": g["kind"] == "models" and "/" in label,
+ "hfRepo": label if g["kind"] == "models" and "/" in label else None,
}
for label, g in groups.items()
]
+class DeleteHfRepoRequest(BaseModel):
+ repo: str = Field(min_length=3, max_length=256, pattern=r"^[\w.\-]+/[\w.\-]+$")
+
+
+@router.post("/api/diagnostics/hf-repo/delete")
+def diagnostics_delete_hf_repo(request: Request, body: DeleteHfRepoRequest) -> dict[str, Any]:
+ """Remove one cached Hugging Face model repo, including blobs in the shared store.
+
+ Goes through huggingface_hub's cache API so a blob another repo links to is
+ kept and only this repo's own files are freed. Unloads the model first when
+ it is the one loaded.
+ """
+ from backend_service.helpers.hf_cache_paths import _hf_hub_cache_root
+
+ state: ChaosEngineState = request.app.state.chaosengine
+ try:
+ from huggingface_hub import scan_cache_dir
+
+ info = scan_cache_dir(_hf_hub_cache_root())
+ except Exception as exc: # noqa: BLE001 - cache layout errors come in many types
+ raise HTTPException(status_code=500, detail=f"Cannot read the Hugging Face cache: {exc}") from exc
+ repo = next((r for r in info.repos if r.repo_type == "model" and r.repo_id == body.repo), None)
+ if repo is None:
+ raise HTTPException(status_code=404, detail=f"{body.repo} is not in the Hugging Face cache.")
+ loaded = getattr(state.runtime, "loaded_model", None)
+ if loaded and body.repo in {getattr(loaded, "ref", None), getattr(loaded, "canonicalRepo", None)}:
+ state.runtime.unload_model()
+ plan = info.delete_revisions(*[rev.commit_hash for rev in repo.revisions])
+ freed = int(plan.expected_freed_size)
+ try:
+ plan.execute()
+ except OSError as exc:
+ raise HTTPException(status_code=500, detail=f"Failed to delete {body.repo}: {exc}") from exc
+ state.add_log("library", "info", f"Deleted cached repo {body.repo} ({freed} bytes).")
+ return {"deleted": body.repo, "freedBytes": freed, "library": state._library(force=True)}
+
+
@router.get("/api/diagnostics/storage-children")
def diagnostics_storage_children(request: Request, path: str) -> dict[str, Any]:
"""Direct children of one folder from the Disk usage table, biggest first.
diff --git a/src/api/admin.ts b/src/api/admin.ts
index eab230ed..03152ba4 100644
--- a/src/api/admin.ts
+++ b/src/api/admin.ts
@@ -127,6 +127,8 @@ export interface StorageTopEntry {
lastModified: number;
expandable?: boolean;
deletable?: boolean;
+ /** Set for a cached Hugging Face repo: deleted through the cache API, not by path. */
+ hfRepo?: string | null;
}
export interface StorageTopResponse {
@@ -146,6 +148,10 @@ export async function fetchStorageTop(limit = 20): Promise {
);
}
+export async function deleteHfRepo(repo: string): Promise<{ deleted: string; freedBytes: number }> {
+ return await postJson<{ deleted: string; freedBytes: number }>("/api/diagnostics/hf-repo/delete", { repo });
+}
+
export async function fetchStorageChildren(path: string): Promise {
const res = await fetchJson<{ entries: StorageTopEntry[] }>(
`/api/diagnostics/storage-children?path=${encodeURIComponent(path)}`,
diff --git a/src/api/index.ts b/src/api/index.ts
index d53b6f55..19ac2550 100644
--- a/src/api/index.ts
+++ b/src/api/index.ts
@@ -637,6 +637,7 @@ export type {
export {
fetchDiagnosticsLogTail,
fetchDiagnosticsSnapshot,
+ deleteHfRepo,
fetchStorageChildren,
fetchStorageTop,
getModelMoveStatus,
diff --git a/src/features/settings/DiagnosticsPanel.tsx b/src/features/settings/DiagnosticsPanel.tsx
index 1a26d5f1..cf032d87 100644
--- a/src/features/settings/DiagnosticsPanel.tsx
+++ b/src/features/settings/DiagnosticsPanel.tsx
@@ -3,6 +3,7 @@ import { useTranslation } from "react-i18next";
import { Panel } from "../../components/Panel";
import {
fetchDiagnosticsSnapshot,
+ deleteHfRepo,
deleteModelPath,
fetchStorageChildren,
fetchStorageTop,
@@ -565,13 +566,14 @@ function StorageRow({
const remove = async () => {
const size = entry.sizeGb !== null ? `${entry.sizeGb.toFixed(1)} GB` : "unknown size";
const confirmed = window.confirm(
- `Permanently delete ${entry.path} (${size})?\n\nThis removes it from disk and cannot be undone.`,
+ `Permanently delete ${entry.hfRepo ?? entry.path} (${size})?\n\nThis removes it from disk and cannot be undone.`,
);
if (!confirmed) return;
setDeleting(true);
setErr(null);
try {
- await deleteModelPath(entry.path);
+ if (entry.hfRepo) await deleteHfRepo(entry.hfRepo);
+ else await deleteModelPath(entry.path);
setGone(true);
onDeleted(entry);
} catch (e) {
diff --git a/tests/test_diagnostics_routes.py b/tests/test_diagnostics_routes.py
index 091c08a7..a49df8a3 100644
--- a/tests/test_diagnostics_routes.py
+++ b/tests/test_diagnostics_routes.py
@@ -276,7 +276,34 @@ def test_shared_blob_store_is_grouped_by_repo(self):
[(e["repoLabel"], e["sizeBytes"]) for e in entries],
[("org/big (2 files)", 400), ("other/small (1 file)", 50)],
)
- self.assertFalse(any(e["deletable"] for e in entries))
+ self.assertEqual([e["hfRepo"] for e in entries], ["org/big", "other/small"])
+ self.assertTrue(all(e["deletable"] for e in entries))
+
+ def test_delete_hf_repo_frees_it_and_keeps_other_repos(self):
+ import os
+ from unittest import mock
+
+ hub = self.models / "hub"
+ for name, size in (("org--gone", 300), ("org--kept", 100)):
+ repo = hub / f"models--{name}"
+ (repo / "blobs").mkdir(parents=True)
+ (repo / "refs").mkdir()
+ (repo / "snapshots" / f"rev-{name}").mkdir(parents=True)
+ (repo / "blobs" / name).write_bytes(b"x" * size)
+ (repo / "snapshots" / f"rev-{name}" / "w.bin").symlink_to(repo / "blobs" / name)
+ (repo / "refs" / "main").write_text(f"rev-{name}")
+ with mock.patch.dict(os.environ, {"HF_HUB_CACHE": str(hub)}):
+ missing = self.client.post("/api/diagnostics/hf-repo/delete", json={"repo": "org/nope"})
+ self.assertEqual(missing.status_code, 404)
+ resp = self.client.post("/api/diagnostics/hf-repo/delete", json={"repo": "org/gone"})
+ self.assertEqual(resp.status_code, 200)
+ self.assertEqual(resp.json()["freedBytes"], 300)
+ self.assertFalse((hub / "models--org--gone").exists())
+ self.assertTrue((hub / "models--org--kept" / "blobs" / "org--kept").exists())
+
+ def test_delete_hf_repo_rejects_a_path_shaped_repo(self):
+ resp = self.client.post("/api/diagnostics/hf-repo/delete", json={"repo": "../etc/passwd"})
+ self.assertEqual(resp.status_code, 422)
def test_children_lists_files_and_folders_biggest_first(self):
resp = self.client.get(
From 9414be53699bee71bc8779a733f07caf23260cc3 Mon Sep 17 00:00:00 2001
From: Cryptopoly <31970407+cryptopoly@users.noreply.github.com>
Date: Mon, 26 Oct 2026 08:30:13 +0000
Subject: [PATCH 15/17] docs: v0.9.8 notes lead with stop controls, load
cancel, streaming follow and disk-usage repo delete
---
CHANGELOG.md | 2 ++
RELEASE_NOTES_v0.9.8.md | 11 +++++++++--
2 files changed, 11 insertions(+), 2 deletions(-)
diff --git a/CHANGELOG.md b/CHANGELOG.md
index a1951c17..2f93f23a 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -19,6 +19,8 @@ First release since v0.9.4; versions 0.9.5 to 0.9.7 were internal. Full notes in
- New models: LFM2.5 24B-A2B (MLX 4/8-bit and GGUF), Muse Glimmer 30B GGUF (vision, DFlash 2 sidecar), Kolibri 1 GGUF Q3_K_M, Qwen3.8 Flash Next GGUF, Qwen3.6 35B-A3B MTP through TensorFold, Qwen-Image-2.1 Turbo and FLUX 3 Image Klein through mflux.
- Image input for MLX vision models loaded through mlx-vlm (Gemma 4 on Apple Silicon).
- Memory and speed estimates rebuilt for chat, image and video: real KV-cache layout per architecture, memory pressure from available memory, decode speed from this machine's history or its memory bandwidth, an advisory check before a chat model loads, and engine-aware image and video estimates (FU-110).
+- Dashboard process list: Stop per leftover model process and a Stop idle button; unloading cancels a load in flight and a newer load replaces the one running.
+- Disk usage breaks the shared Hugging Face blob store down by repo, and a cached repo can be deleted there through the cache API.
- `release-constraints.txt` pins every package in release builds to the tested versions.
### Fixed
diff --git a/RELEASE_NOTES_v0.9.8.md b/RELEASE_NOTES_v0.9.8.md
index fce3d6da..62461233 100644
--- a/RELEASE_NOTES_v0.9.8.md
+++ b/RELEASE_NOTES_v0.9.8.md
@@ -7,6 +7,7 @@ The first release since v0.9.4. Versions 0.9.5 to 0.9.7 were internal builds, so
- **DFlash on Apple Silicon finally runs.** Since dflash-mlx 0.1.5, every DFlash reply had quietly fallen back to normal decoding while the load note still said DFlash was active. Fixed: Qwen3-4B now writes at 113 to 138 tokens per second against 53 without it, with identical output.
- **Newer MLX underneath.** Releases now ship mlx 0.32, mlx-lm 0.32 and transformers 5.20. Setup installs resolve mlx-vlm 0.7.10 and diffusers 0.41. Current Gemma 4 conversions load (the old mlx-vlm could not read them), and every release build pins the exact versions it was tested with.
- **Images in chat on Apple Silicon.** Vision models loaded through mlx-vlm (Gemma 4) now see attached images. Before, the image was always dropped.
+- **Models stop loading when you ask.** Unload cancels a load in flight, and picking a different model replaces the one that is loading instead of queueing behind it. The Dashboard lists leftover model processes with a Stop button each and a Stop idle button, so a stuck worker can no longer hold 20 GB of memory with no way out. Send is disabled while a model loads, so a message can no longer go to the previous model.
- **Honest memory and speed estimates.** The launch screen works out the real KV-cache size for each model's architecture and estimates speed from your own past runs or your machine's memory bandwidth. It warns before you load a chat model that may not fit. Image and video estimates know which engine they run on and count a chat model that is already loaded.
### New models
@@ -31,7 +32,7 @@ The first release since v0.9.4. Versions 0.9.5 to 0.9.7 were internal builds, so
### Faster generation
- **DFlash** (Apple Silicon) fixed, as above.
-- **Thinking Off now works** for Qwen 3 / 3.5 / 3.6 / 3.8 on Apple Silicon. Before, they reasoned anyway and spent the reply budget on it.
+- **Thinking Off now works** for Qwen 3 / 3.5 / 3.6 / 3.8 on Apple Silicon, and the switch is sent to llama.cpp models too. Before, they reasoned anyway and spent the reply budget on it.
- **TensorFold 0.6.6** (Apple Silicon): exact speculative decoding with no change in output. Install it from the launch settings.
- **llama.cpp b11810** bundled: DFlash 2, EAGLE-3 and draft-free `ngram-mod`, picked automatically from the drafter files next to a GGUF.
- **MTPLX 2.12**: reasoning now streams correctly.
@@ -54,7 +55,9 @@ A Voice group in the sidebar: speech-to-text and text-to-speech with Studio, Mod
- **Chat > My Models lists chat models only.** 3D tool parts, text encoders, voice models and bare weight files no longer show up as chat models.
- **Settings > Diagnostics is split into three tabs**: Diagnostics, Accelerators and Disk usage.
-- **Disk usage rows expand and can be deleted.** The shared Hugging Face `blobs` store is protected.
+- **Disk usage rows expand and can be deleted.** The shared Hugging Face `blobs` store is now broken down by repo, and any cached repo can be deleted from there; blobs another repo still uses are kept.
+- **Streaming follows the bottom.** In Chat, Compare and the HTML challenge the view now stays with the newest text until you scroll up. Before, one long chunk made it stop following, and Latest had to be pressed again and again.
+- **Multi-quant GGUF repos show the size of the one file that loads**, not the whole folder, in the library and the memory bar. A model that fits no longer looks too big to run.
### Fixes
@@ -68,6 +71,10 @@ A Voice group in the sidebar: speech-to-text and text-to-speech with Studio, Mod
- **Dead catalog links** repaired; an audit script checks every row.
- **Release builds** include the `dflash` package, the Setup installers and a pinned llama.cpp build, so GGUF models work out of the box.
- **Tool use on Apple Silicon** no longer loops.
+- **Loading** a model no longer hangs forever when a second load is requested: repeated requests for the same model share one load.
+- **My Models and the chat picker** no longer come up empty while the first library scan is still running.
+- **MTPLX** is no longer chosen for models without an MTP head, and the TensorFold toggle names DFlash 2 and MTP drafting. A cold TensorFold start says it can take a few minutes.
+- **Extras overlay**: an older huggingface-hub or transformers left by a Setup install can no longer shadow the ones the app ships.
- **MLX probe**: one stray kill of the startup check at launch no longer turns MLX off until you restart; it is retried once. The launch screen's context setting is capped at the model's own window, so a stored 262K no longer triggers a false memory warning on a 32K model.
- **Hub listings** work on Python installs without a certificate bundle (Discover came back empty there).
- **Downloads** skip weight shards that no index file uses, and the Gemma 4 entries are merged into one family.
From b7925c9e423eb83fafc17520b8996c4c22a8bbc4 Mon Sep 17 00:00:00 2001
From: Cryptopoly <31970407+cryptopoly@users.noreply.github.com>
Date: Mon, 26 Oct 2026 08:33:58 +0000
Subject: [PATCH 16/17] fix: stop tracking the worktree .venv symlink (it broke
every CI venv step)
---
.gitignore | 1 +
.venv | 1 -
2 files changed, 1 insertion(+), 1 deletion(-)
delete mode 120000 .venv
diff --git a/.gitignore b/.gitignore
index 7b856ece..985aa2dd 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,5 +1,6 @@
__pycache__/
*.pyc
+.venv
.venv/
*.egg-info/
build/
diff --git a/.venv b/.venv
deleted file mode 120000
index d9a17cf1..00000000
--- a/.venv
+++ /dev/null
@@ -1 +0,0 @@
-../ChaosEngineAI/.venv
\ No newline at end of file
From 6471a30378cfeca5ca68fb8d69fbf309930cb0ea Mon Sep 17 00:00:00 2001
From: Cryptopoly <31970407+cryptopoly@users.noreply.github.com>
Date: Mon, 26 Oct 2026 08:34:11 +0000
Subject: [PATCH 17/17] fix: stop tracking the worktree node_modules symlink
---
.gitignore | 1 +
node_modules | 1 -
2 files changed, 1 insertion(+), 1 deletion(-)
delete mode 120000 node_modules
diff --git a/.gitignore b/.gitignore
index 985aa2dd..ea276a44 100644
--- a/.gitignore
+++ b/.gitignore
@@ -6,6 +6,7 @@ __pycache__/
build/
dist/
.DS_Store
+node_modules
node_modules/
src-tauri/target/
src-tauri/resources/embedded/
diff --git a/node_modules b/node_modules
deleted file mode 120000
index 1447ca2a..00000000
--- a/node_modules
+++ /dev/null
@@ -1 +0,0 @@
-../ChaosEngineAI/node_modules
\ No newline at end of file