Fix download progress counts; repair build.yml; gate docs deploy

.github/workflows/build.yml

@@ -63,11 +63,11 @@ jobs:
       - name: cargo check
         run: cargo check --manifest-path src-tauri/Cargo.toml --all-targets
       - name: Python venv + install
-        shell: ${{ matrix.shell }}
+        shell: bash
         run: |
           python -m venv .venv
           ${{ matrix.venv-activate }}
-          pip install --upgrade pip
+          python -m pip install --upgrade pip
           # [images] brings torch+diffusers+accelerate+pillow+safetensors, which
           # tests/test_video_runtime.py needs because probe() does a real
           # ``import torch`` after the mocked _find_missing returns. Without
@@ -76,7 +76,7 @@ jobs:
           # assertion fails.
           pip install -e ".[desktop,dev,images]"
       - name: Python tests with coverage
-        shell: ${{ matrix.shell }}
+        shell: bash
         run: |
           ${{ matrix.venv-activate }}
           python -m pytest tests/ -v --tb=short \

.github/workflows/deploy-docs.yml

@@ -36,6 +36,10 @@ concurrency:
 
 jobs:
   build-and-deploy:
+    # Runs once the deploy key exists: add the SITE_REPO_DEPLOY_KEY secret, then set the
+    # repository variable SITE_DEPLOY_ENABLED to true. Until then the job is skipped
+    # instead of failing on the push to ChaosEngineAI-Site.
+    if: vars.SITE_DEPLOY_ENABLED == 'true'
     runs-on: ubuntu-latest
     timeout-minutes: 10
     steps:

backend_service/helpers/hf_cache_paths.py

@@ -38,10 +38,18 @@ def _hf_repo_cache_dir(repo_id: str) -> Path:
     return _hf_hub_cache_root() / f"models--{repo_id.replace('/', '--')}"
 
 
-def _hf_repo_downloaded_bytes(repo_id: str) -> int:
+def _hf_repo_downloaded_bytes(repo_id: str, allow_patterns: list[str] | None = None) -> int:
+    """Bytes of ``repo_id`` in the cache.
+
+    With ``allow_patterns`` (a download that fetches only some files), count
+    just the snapshot files those patterns select plus in-progress blobs, so a
+    repo that also holds other quants does not show more than its own total.
+    """
     from backend_service.helpers.discovery import _path_size_bytes
 
     cache_dir = _hf_repo_cache_dir(repo_id)
+    if allow_patterns:
+        return _matching_cache_bytes(cache_dir, allow_patterns)
     try:
         if not cache_dir.exists():
             return 0
@@ -53,6 +61,36 @@ def _hf_repo_downloaded_bytes(repo_id: str) -> int:
         return 0
 
 
+def _matching_cache_bytes(cache_dir: Path, allow_patterns: list[str]) -> int:
+    import fnmatch
+
+    seen: set[str] = set()
+    total = 0
+    snapshots = cache_dir / "snapshots"
+    try:
+        for entry in snapshots.rglob("*"):
+            relative = entry.relative_to(snapshots).as_posix().split("/", 1)
+            name = relative[1] if len(relative) > 1 else relative[0]
+            if not any(fnmatch.fnmatch(name, pattern) for pattern in allow_patterns):
+                continue
+            try:
+                real = entry.resolve()
+                if not real.is_file() or str(real) in seen:
+                    continue
+                seen.add(str(real))
+                total += real.stat().st_size
+            except OSError:
+                continue
+        for blob in (cache_dir / "blobs").glob("*.incomplete"):
+            try:
+                total += blob.stat().st_size
+            except OSError:
+                continue
+    except OSError:
+        return total
+    return total
+
+
 def _hf_repo_snapshot_dir(repo_id: str) -> Path | None:
     cache_dir = _hf_repo_cache_dir(repo_id)
     snapshots_dir = cache_dir / "snapshots"

backend_service/state/downloads.py

@@ -121,6 +121,15 @@ def start_download(
         if pinned_bytes > 0:
             total_gb = _bytes_to_gb(pinned_bytes)
 
+    # The files this download fetches: progress counts only those, not other
+    # quants of the repo that an earlier download left in the cache.
+    scope_patterns = (
+        allow_patterns
+        or _video_repo_allow_patterns(repo)
+        or _image_repo_allow_patterns(repo)
+        or text_patterns
+    )
+    downloaded_gb = _bytes_to_gb(_hf_repo_downloaded_bytes(repo, scope_patterns))
     initial_progress = 0.0
     if isinstance(total_gb, (int, float)) and total_gb > 0 and downloaded_gb > 0:
         initial_progress = min(0.99, downloaded_gb / float(total_gb))
@@ -150,7 +159,7 @@ def _download_worker():
 
         def _progress_worker() -> None:
             while not stop_progress.wait(1.0):
-                downloaded_bytes = _hf_repo_downloaded_bytes(repo)
+                downloaded_bytes = _hf_repo_downloaded_bytes(repo, scope_patterns)
                 downloaded_gb_local = _bytes_to_gb(downloaded_bytes)
                 with state._lock:
                     current = state._downloads.get(repo)
@@ -254,7 +263,7 @@ def _progress_worker() -> None:
                     return
                 cancelled = state._download_cancel.get(repo, False)
             if cancelled:
-                downloaded_gb_local = _bytes_to_gb(_hf_repo_downloaded_bytes(repo))
+                downloaded_gb_local = _bytes_to_gb(_hf_repo_downloaded_bytes(repo, scope_patterns))
                 with state._lock:
                     current = state._downloads.get(repo)
                     if current is None or state._download_tokens.get(repo) != download_token:
@@ -292,7 +301,7 @@ def _progress_worker() -> None:
                     state._downloads[repo]["error"] = validation_error
                     state.add_log("library", "error", validation_error)
                 return
-            downloaded_gb_local = _bytes_to_gb(_hf_repo_downloaded_bytes(repo))
+            downloaded_gb_local = _bytes_to_gb(_hf_repo_downloaded_bytes(repo, scope_patterns))
             with state._lock:
                 if state._download_tokens.get(repo) != download_token:
                     return
@@ -437,7 +446,14 @@ def cancel_download(state: ChaosEngineState, repo: str) -> dict[str, Any]:
         except Exception:
             pass
 
-    downloaded_gb = _bytes_to_gb(_hf_repo_downloaded_bytes(repo))
+    from backend_service.helpers.text_gguf import text_repo_allow_patterns
+
+    scope_patterns = (
+        _video_repo_allow_patterns(repo)
+        or _image_repo_allow_patterns(repo)
+        or text_repo_allow_patterns(repo)
+    )
+    downloaded_gb = _bytes_to_gb(_hf_repo_downloaded_bytes(repo, scope_patterns))
     with state._lock:
         current = state._downloads.get(repo)
         if current is None:

scripts/check-version-sync.mjs

@@ -35,7 +35,7 @@ const sources = [
     "src-tauri/Cargo.lock",
     () => {
       const name = read("src-tauri/Cargo.toml").match(/^name\s*=\s*"([^"]+)"/m)?.[1];
-      const block = read("src-tauri/Cargo.lock").match(new RegExp(`name = "${name}"\\nversion = "([^"]+)"`));
+      const block = read("src-tauri/Cargo.lock").match(new RegExp(`name = "${name}"\\r?\\nversion = "([^"]+)"`));
       return block?.[1];
     },
   ],

tests/test_library_cache_persistence.py

@@ -109,7 +109,7 @@ def test_load_rejects_corrupt_payload(self):
 
 class StateIntegrationTests(unittest.TestCase):
     def setUp(self) -> None:
-        self.tmp = tempfile.TemporaryDirectory()
+        self.tmp = tempfile.TemporaryDirectory(ignore_cleanup_errors=True)
         tmpdir = Path(self.tmp.name)
         self.cache_path = tmpdir / "library_cache.json"
         self.kwargs = dict(

tests/test_mflux_engine.py

@@ -137,6 +137,7 @@ def _fake_command(directory: Path, body: str) -> Path:
     return script
 
 
+@unittest.skipIf(os.name == "nt", "the engine runs shell-script stand-ins; mflux is Apple Silicon only")
 class RunTests(unittest.TestCase):
     def setUp(self) -> None:
         self._tmp = tempfile.TemporaryDirectory()
@@ -244,6 +245,7 @@ def test_flux_still_falls_back_to_diffusers(self) -> None:
         self.assertIn("mflux failed", status["message"])
 
 
+@unittest.skipIf(os.name == "nt", "the installer is a bash script; mflux is Apple Silicon only")
 class SetupRouteTests(unittest.TestCase):
     def setUp(self) -> None:
         self._tmp = tempfile.TemporaryDirectory()

tests/test_mlx_models.py

@@ -139,7 +139,10 @@ def test_missing_or_broken_config_is_false(self) -> None:
 class KolibriTinyModelTests(unittest.TestCase):
     @classmethod
     def setUpClass(cls) -> None:
-        import mlx.core as mx
+        try:
+            import mlx.core as mx
+        except ImportError:
+            raise unittest.SkipTest("mlx is not installed")
 
         cls.mx = mx
         path = Path(mlx_models.__file__).with_name("kolibri1.py")

tests/test_mlx_prompt_cache.py

@@ -7,6 +7,7 @@
 full prefill.
 """
 
+import importlib.util
 import unittest
 from unittest import mock
 
@@ -59,6 +60,7 @@ def test_compression_strategy_passthrough(self):
         self.assertIsNone(acq.commit_tokens)
 
 
+@unittest.skipUnless(importlib.util.find_spec("mlx_lm"), "mlx-lm is not installed")
 class AcquireNativeTests(unittest.TestCase):
     def _patches(self, *, can_trim=True, trim=lambda c, n: n, fresh_label="fresh"):
         return (

tests/test_sdcpp_image.py

@@ -98,7 +98,7 @@ def test_returns_none_when_no_env_no_managed(self):
 
     def test_returns_env_path_when_set(self):
         with patch.dict(os.environ, {}, clear=False):
-            tmp = Path("/tmp/sdcpp-img-test-binary")
+            tmp = Path(__import__("tempfile").gettempdir()) / "sdcpp-img-test-binary"
             tmp.write_text("")
             try:
                 os.environ["CHAOSENGINE_SDCPP_BIN"] = str(tmp)

tests/test_sdcpp_video.py

@@ -90,7 +90,7 @@ def test_returns_none_when_no_env_no_managed(self):
 
     def test_returns_env_path_when_set(self):
         with patch.dict(os.environ, {}, clear=False):
-            tmp = Path("/tmp/sdcpp-test-binary")
+            tmp = Path(__import__("tempfile").gettempdir()) / "sdcpp-test-binary"
             tmp.write_text("")
             try:
                 os.environ["CHAOSENGINE_SDCPP_BIN"] = str(tmp)

tests/test_setup_routes.py

@@ -211,7 +211,7 @@ def test_mlx_video_install_cannot_pull_a_newer_mlx(self):
             self.client.post("/api/setup/install-package", json={"package": "mlx-video"})
         cmd = mock_run.call_args[0][0]
         self.assertIn("mlx-vlm>=0.6.3", cmd)
-        self.assertEqual(cmd[cmd.index("-c") + 1], "/tmp/x-extras/c.txt")
+        self.assertEqual(Path(cmd[cmd.index("-c") + 1]), Path("/tmp/x-extras/c.txt"))
         self.assertFalse(any(arg.startswith("mlx>=") or arg.startswith("mlx<") for arg in cmd), cmd)
 
     def test_every_overlay_install_passes_the_mlx_constraint_file(self):

tests/test_tensorfold_routing.py

@@ -184,7 +184,12 @@ def test_a_failed_tested_checkpoint_falls_back_to_standard_mlx(self) -> None:
         controller = _controller(tensorfold=True)
         controller._select_engine = mock.Mock(return_value=_real_tensorfold_engine_that_fails(controller, "Metal exploded"))
         fallback = _LoadedByStub("standard MLX")
-        with mock.patch("backend_service.inference.controller.MLXWorkerEngine", return_value=fallback):
+        # ``load_model`` re-probes capabilities; pin the probe so a host without mlx
+        # (CI on Linux) still sees the usable MLX these tests set up.
+        with mock.patch(
+            "backend_service.inference.controller.get_backend_capabilities",
+            return_value=controller.capabilities,
+        ), mock.patch("backend_service.inference.controller.MLXWorkerEngine", return_value=fallback):
             loaded = self._load(controller, _NATIVE)
         self.assertIs(controller.engine, fallback)
         self.assertIn("TensorFold startup failed (Metal exploded); using standard MLX.", loaded.runtimeNote or "")
@@ -217,7 +222,12 @@ def test_an_explicit_request_for_an_unlisted_model_falls_back_too(self) -> None:
         controller = _controller(tensorfold=True)
         controller._select_engine = mock.Mock(return_value=_real_tensorfold_engine_that_fails(controller, "unsupported model_type"))
         fallback = _LoadedByStub("standard MLX")
-        with mock.patch("backend_service.inference.controller.MLXWorkerEngine", return_value=fallback):
+        # ``load_model`` re-probes capabilities; pin the probe so a host without mlx
+        # (CI on Linux) still sees the usable MLX these tests set up.
+        with mock.patch(
+            "backend_service.inference.controller.get_backend_capabilities",
+            return_value=controller.capabilities,
+        ), mock.patch("backend_service.inference.controller.MLXWorkerEngine", return_value=fallback):
             loaded = self._load(controller, _COMMUNITY)
         self.assertIn("using standard MLX", loaded.runtimeNote or "")
 

tests/test_text_gguf.py

@@ -222,3 +222,32 @@ def test_one_pin_per_repo(self) -> None:
 
 if __name__ == "__main__":
     unittest.main()
+
+
+@unittest.skipIf(__import__("os").name == "nt", "needs symlinks")
+class DownloadedBytesScopeTests(unittest.TestCase):
+    """Progress counts only the files a pinned download fetches."""
+
+    def test_other_quants_in_the_cache_do_not_inflate_progress(self) -> None:
+        import tempfile
+        from pathlib import Path
+        from unittest import mock
+
+        from backend_service.helpers import hf_cache_paths
+
+        with tempfile.TemporaryDirectory() as tmp:
+            repo_dir = Path(tmp) / "models--acme--Model-GGUF"
+            snapshot = repo_dir / "snapshots" / "abc"
+            blobs = repo_dir / "blobs"
+            snapshot.mkdir(parents=True)
+            blobs.mkdir()
+            for name, size in {"Model-Q8_0.gguf": 100, "Model-Q4_K_M.gguf": 400}.items():
+                blob = blobs / f"sha-{name}"
+                blob.write_bytes(b"x" * size)
+                (snapshot / name).symlink_to(blob)
+            (blobs / "partial.incomplete").write_bytes(b"x" * 7)
+            with mock.patch.object(hf_cache_paths, "_hf_repo_cache_dir", return_value=repo_dir):
+                whole = hf_cache_paths._hf_repo_downloaded_bytes("acme/Model-GGUF")
+                scoped = hf_cache_paths._hf_repo_downloaded_bytes("acme/Model-GGUF", ["Model-Q8_0.gguf"])
+        self.assertEqual(whole, 507)
+        self.assertEqual(scoped, 107)  # the pinned file plus the in-progress blob

tests/test_video_routes.py

@@ -367,8 +367,16 @@ def test_outputs_is_empty_until_generation_lands(self):
 class VideoPreloadRouteTests(unittest.TestCase):
     def setUp(self) -> None:
         self.client, self.tempdir, self.env_snapshot = make_client()
+        # Same as the generate tests (FU-060): preload runs the memory gate, and a
+        # small CI runner would refuse a 10 GB model before the route logic runs.
+        self._memory_patch = mock.patch(
+            "backend_service.helpers.memory_gate.snapshot_memory_signals",
+            return_value=(64.0, 10.0),
+        )
+        self._memory_patch.start()
 
     def tearDown(self) -> None:
+        self._memory_patch.stop()
         restore_env(self.env_snapshot)
         self.tempdir.cleanup()