From 42e03c55909e6a4a04622d8b2f7d846ab22f86ff Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dean=20Qui=C3=B1anola?= Date: Tue, 7 Jul 2026 21:56:44 -0700 Subject: [PATCH 1/6] docs(examples): add network-volume warm cache (VolumeCache) example --- .../05_volume_warm_cache/README.md | 69 ++++++++++++ .../05_volume_warm_cache/gpu_worker.py | 102 ++++++++++++++++++ .../05_volume_warm_cache/requirements.txt | 12 +++ 05_data_workflows/README.md | 10 ++ 4 files changed, 193 insertions(+) create mode 100644 05_data_workflows/05_volume_warm_cache/README.md create mode 100644 05_data_workflows/05_volume_warm_cache/gpu_worker.py create mode 100644 05_data_workflows/05_volume_warm_cache/requirements.txt diff --git a/05_data_workflows/05_volume_warm_cache/README.md b/05_data_workflows/05_volume_warm_cache/README.md new file mode 100644 index 0000000..7859e86 --- /dev/null +++ b/05_data_workflows/05_volume_warm_cache/README.md @@ -0,0 +1,69 @@ +# 05_volume_warm_cache + +Warm-cache a model on a Runpod network volume with `VolumeCache`, so cold starts +restore the model from the volume instead of re-downloading it. + +## Status + +`VolumeCache` is a `runpod-python` serverless primitive added in +**SLS-367 / [runpod-python PR #531](https://github.com/runpod/runpod-python/pull/531)**. +It is not yet in a released `runpod-python`. This example runs end-to-end once: + +1. PR #531 is merged and released, and +2. the deployed **flash-worker** image is built against a `runpod-python` that + includes `VolumeCache` (the model load runs inside the remote GPU worker, so + it is that image's `runpod`, not this example's local env, that must have it). + +Until then this example is illustrative. `requirements.txt` pins `runpod` to the +PR branch so the local environment and any custom worker build can resolve +`VolumeCache` today; switch it to the released version once available. + +## Overview + +`VolumeCache` keeps a browsable mirror of local cache directories on a mounted +network volume and reconciles them on each use: + +- **On enter** (`hydrate`): copy files that are missing or newer on the volume + into the container. +- **On exit** (`sync`): copy files that are missing or newer in the container + back to the volume (in the background). + +The GPU worker wraps its Stable Diffusion load in `with VolumeCache(...)`. The +first cold worker downloads the weights and syncs them to the volume; every later +cold worker hydrates from the volume and skips the download. + +## How this differs from `01_network_volumes` + +`01_network_volumes` sets `HF_HUB_CACHE` to a path **on** the volume, so every +model read hits the network mount. This example keeps the Hugging Face cache on +**local disk** and uses `VolumeCache` to mirror it to the volume — inference +reads stay local (fast) while cold starts stay warm (persisted). + +| | 01_network_volumes | 02_volume_warm_cache | +| --- | --- | --- | +| Model cache location | On the network volume (`HF_HUB_CACHE` → `/runpod-volume/...`) | Local disk (default `~/.cache/huggingface`) | +| Volume role | Live cache (read/written directly) | Warm mirror (hydrated in, synced out) | +| Inference read speed | Network mount | Local disk | +| Cold-start download | Skipped (reads from volume) | Skipped (restored to local disk) | + +## Quick Start + +```bash +uv pip install -r 05_data_workflows/05_volume_warm_cache/requirements.txt +uv run flash login # or set RUNPOD_API_KEY in .env +uv run flash dev # serves at http://localhost:8888 +``` + +Generate an image (provisions a real GPU worker with the volume attached): + +```bash +curl -X POST http://localhost:8888/gpu_worker/runsync \ + -H "Content-Type: application/json" \ + -d '{"prompt": "a sunset over mountains"}' +``` + +## What You'll Learn + +- How to warm-cache model weights across cold starts with `runpod.serverless.VolumeCache` +- Why mirroring a local cache to a volume differs from mounting the cache on the volume +- The `with VolumeCache(dirs=[...]):` closure pattern inside a Flash `@Endpoint` diff --git a/05_data_workflows/05_volume_warm_cache/gpu_worker.py b/05_data_workflows/05_volume_warm_cache/gpu_worker.py new file mode 100644 index 0000000..7dec85a --- /dev/null +++ b/05_data_workflows/05_volume_warm_cache/gpu_worker.py @@ -0,0 +1,102 @@ +# GPU worker that warm-caches its model on a network volume with VolumeCache. +# +# Contrast with 01_network_volumes, which points HF_HUB_CACHE straight at the +# volume so every read hits the network mount. Here the Hugging Face cache stays +# on fast local disk and VolumeCache mirrors it to the volume: inference reads +# are local, while cold starts stay warm because the model is restored from the +# volume instead of re-downloaded. +# +# STATUS: VolumeCache ships in runpod-python (SLS-367 / PR #531). This example +# runs once that change is released and the flash-worker image includes it; see +# README.md. Until then it is illustrative. +# +# run with: flash dev +import logging + +from runpod_flash import Endpoint, GpuType, NetworkVolume + +logger = logging.getLogger(__name__) + +volume = NetworkVolume( + name="flash-05-02-volume", + size=50, +) + + +@Endpoint( + name="02_volume_warm_cache", + gpu=GpuType.NVIDIA_GEFORCE_RTX_5090, + workers=(0, 3), + idle_timeout=300, + volume=volume, + dependencies=["torch", "diffusers", "transformers", "accelerate"], +) +class WarmCachedSD: + def __init__(self): + # Imports live inside the handler: @Endpoint ships only the function + # body to the worker, so module-level imports/constants aren't available + # remotely. VolumeCache comes from the worker's runpod-python. + import logging + import os + + import torch + from diffusers import StableDiffusionPipeline + from runpod.serverless import VolumeCache + + self.logger = logging.getLogger(__name__) + + # HF cache stays on local disk (default ~/.cache/huggingface). We do NOT + # point HF_HUB_CACHE at the volume. Instead VolumeCache hydrates that + # local dir from the volume before the load and syncs new files back + # after: the first cold worker downloads once; every later cold worker + # restores from the volume and skips the download. + hf_cache = os.environ.get("HF_HOME", os.path.expanduser("~/.cache/huggingface")) + + self.logger.info("Loading Stable Diffusion (warm cache via VolumeCache)...") + with VolumeCache(dirs=[hf_cache]): + self.pipe = StableDiffusionPipeline.from_pretrained( + "runwayml/stable-diffusion-v1-5", + torch_dtype=torch.float16, + safety_checker=None, + use_safetensors=True, + requires_safety_checker=False, + low_cpu_mem_usage=True, + ).to("cuda") + + self.pipe.enable_attention_slicing() + self.logger.info("Stable Diffusion ready.") + + async def generate_image(self, prompt: str) -> dict: + """Generate a single image from a prompt.""" + self.logger.info(f"Generating image for: '{prompt}'") + + image = self.pipe( + prompt=prompt, + num_inference_steps=20, + guidance_scale=7.5, + width=512, + height=512, + ).images[0] + + import datetime + import os + + output_dir = "/runpod-volume/generated_images" + os.makedirs(output_dir, exist_ok=True) + timestamp = datetime.datetime.now().strftime("%Y%m%d_%H%M%S") + image_path = os.path.join(output_dir, f"sd_generated_{timestamp}.png") + image.save(image_path) + self.logger.info(f"Image saved to: {image_path}") + + return { + "prompt": prompt, + "image_path": image_path, + "timestamp": timestamp, + } + + +if __name__ == "__main__": + import asyncio + + sd = WarmCachedSD() + asyncio.run(sd.generate_image("a cute labrador retriever surfing a wave")) diff --git a/05_data_workflows/05_volume_warm_cache/requirements.txt b/05_data_workflows/05_volume_warm_cache/requirements.txt new file mode 100644 index 0000000..70e873b --- /dev/null +++ b/05_data_workflows/05_volume_warm_cache/requirements.txt @@ -0,0 +1,12 @@ +# VolumeCache ships in runpod-python via SLS-367 / PR #531 (not yet released). +# Pin runpod to the PR branch so `from runpod.serverless import VolumeCache` +# resolves today. Switch to a released version (e.g. runpod>=X.Y.Z) once merged. +runpod @ git+https://github.com/runpod/runpod-python.git@deanquinanola/sls-367-network-volume-warm-cache-for-serverless-volumecache + +runpod-flash>=1.8.0 + +# Model libraries used by the GPU worker +torch +diffusers +transformers +accelerate diff --git a/05_data_workflows/README.md b/05_data_workflows/README.md index 87be705..f222fb6 100644 --- a/05_data_workflows/README.md +++ b/05_data_workflows/README.md @@ -49,6 +49,16 @@ ETL workflows for data processing. - Audio transcription + analysis - Image batch processing +### [05_volume_warm_cache](./05_volume_warm_cache/) +Warm-cache model weights across cold starts with `runpod.serverless.VolumeCache`. + +**What you'll learn:** +- Mirroring a local model cache to a network volume and reconciling on each use +- Why mirroring differs from mounting the cache directly on the volume +- The `with VolumeCache(dirs=[...]):` closure inside a Flash `@Endpoint` + +> Requires `runpod-python` with `VolumeCache` (SLS-367 / PR #531); see the example README for status. + ### 04_s3_integration _(coming soon)_ Cloud storage integration (S3, R2, etc.). From 8a1cd865723ca01271ea897a2571ca9f1042b652 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dean=20Qui=C3=B1anola?= Date: Wed, 8 Jul 2026 08:50:07 -0700 Subject: [PATCH 2/6] docs(examples): add direct vs VolumeCache cold-start benchmark harness --- .../05_volume_warm_cache/README.md | 8 ++ .../05_volume_warm_cache/benchmark/README.md | 56 ++++++++ .../benchmark/bench_direct.py | 82 ++++++++++++ .../benchmark/bench_volumecache.py | 99 ++++++++++++++ .../benchmark/benchmark.py | 121 ++++++++++++++++++ 5 files changed, 366 insertions(+) create mode 100644 05_data_workflows/05_volume_warm_cache/benchmark/README.md create mode 100644 05_data_workflows/05_volume_warm_cache/benchmark/bench_direct.py create mode 100644 05_data_workflows/05_volume_warm_cache/benchmark/bench_volumecache.py create mode 100644 05_data_workflows/05_volume_warm_cache/benchmark/benchmark.py diff --git a/05_data_workflows/05_volume_warm_cache/README.md b/05_data_workflows/05_volume_warm_cache/README.md index 7859e86..199e5b2 100644 --- a/05_data_workflows/05_volume_warm_cache/README.md +++ b/05_data_workflows/05_volume_warm_cache/README.md @@ -62,6 +62,14 @@ curl -X POST http://localhost:8888/gpu_worker/runsync \ -d '{"prompt": "a sunset over mountains"}' ``` +## Benchmark: direct vs VolumeCache + +`benchmark/` contains an A/B that measures cold-start model-load time for the two +caching strategies (direct HF-on-volume vs local-disk-mirrored-to-volume), +reporting first-run vs warm load time and volume/local storage. See +[benchmark/README.md](./benchmark/README.md). The direct arm is runnable today; +the VolumeCache arm needs a flash-worker image that includes VolumeCache (#531). + ## What You'll Learn - How to warm-cache model weights across cold starts with `runpod.serverless.VolumeCache` diff --git a/05_data_workflows/05_volume_warm_cache/benchmark/README.md b/05_data_workflows/05_volume_warm_cache/benchmark/README.md new file mode 100644 index 0000000..5bba8e0 --- /dev/null +++ b/05_data_workflows/05_volume_warm_cache/benchmark/README.md @@ -0,0 +1,56 @@ +# Benchmark: direct volume cache vs VolumeCache + +An A/B that measures **cold-start model-load time** for two ways of caching a +model on a Runpod network volume: + +- **direct** (`bench_direct.py`) — `HF_HOME`/`HF_HUB_CACHE` point at a directory + **on** the volume; the model is read from the network mount on every cold start. +- **volumecache** (`bench_volumecache.py`) — the HF cache stays on **local disk** + and `runpod.serverless.VolumeCache` hydrates it from the volume before load and + syncs new files back after; reads are local, the volume is a warm mirror. + +Both load the same model (`stable-diffusion-v1-5`) and self-report metrics; +`benchmark.py` drives them and prints a comparison. + +## What it measures + +| Metric | Meaning | +| --- | --- | +| first-run load | Cold start with an empty cache — includes the download (and, for volumecache, the sync back). One-time per endpoint. | +| warm load (mean/min) | Cold start with the cache already populated — the number that matters at scale. For volumecache this **includes** the hydrate copy. | +| hydrate | Time volumecache spent copying volume → local (0 for direct). | +| volume / local | Bytes stored on the volume and on local disk (volumecache double-stores). | + +## Prerequisites and status + +- Both endpoints need a network volume attached (the workers declare one). +- **Direct arm: runnable on Flash today.** +- **VolumeCache arm requires a flash-worker image that includes `VolumeCache` + (SLS-367 / [runpod-python PR #531](https://github.com/runpod/runpod-python/pull/531)).** + It cannot be added at runtime via `dependencies` because the worker's `runpod` + is already imported before the handler runs. Until flash-worker ships it, run + the direct arm alone with `--skip-volumecache`. + +This harness has **not been run yet** — no results are published here. Numbers +should be filled in from a real run. + +## Running + +```bash +# From this benchmark/ directory (so flash dev serves only the two bench workers): +uv run flash dev # provisions real GPU endpoints; serves at :8888 + +# In another shell — direct arm only (runnable today): +python benchmark.py --skip-volumecache --warm-trials 3 + +# Full A/B (once the flash-worker image includes VolumeCache): +python benchmark.py --warm-trials 3 +``` + +Confirm the exact routes at `http://localhost:8888/docs` and pass +`--direct-route` / `--vc-route` if they differ. `--cold-wait` (default 45s) must +exceed the workers' `idle_timeout` (30s) so each trial is a fresh cold start. + +> Running provisions real GPU workers and a network volume — it costs money and +> cold starts are slow. Tear down with Ctrl+C on `flash dev` (or `flash undeploy`) +> and verify the endpoints are gone afterward. diff --git a/05_data_workflows/05_volume_warm_cache/benchmark/bench_direct.py b/05_data_workflows/05_volume_warm_cache/benchmark/bench_direct.py new file mode 100644 index 0000000..db5258b --- /dev/null +++ b/05_data_workflows/05_volume_warm_cache/benchmark/bench_direct.py @@ -0,0 +1,82 @@ +# Benchmark worker A: DIRECT caching. +# +# HF_HOME / HF_HUB_CACHE point at a directory ON the network volume, so the model +# is read directly from the (network) mount on every cold start. This is the +# 01_network_volumes approach. Paired with bench_volumecache.py; driven by +# benchmark.py. +# +# Runnable on Flash today (no VolumeCache dependency). +import logging + +from runpod_flash import Endpoint, GpuType, NetworkVolume + +logger = logging.getLogger(__name__) + +# Shared volume so both strategies compete on the same storage medium. +volume = NetworkVolume(name="flash-05-bench-volume", size=50) + +HF_ON_VOLUME = "/runpod-volume/hf" + + +@Endpoint( + name="bench_direct", + gpu=GpuType.NVIDIA_GEFORCE_RTX_5090, + workers=(0, 1), # single worker so each call is one clean measurement + idle_timeout=30, # scale to 0 between trials -> next call is a cold start + volume=volume, + env={"HF_HOME": HF_ON_VOLUME, "HF_HUB_CACHE": HF_ON_VOLUME}, + dependencies=["torch", "diffusers", "transformers", "accelerate"], +) +class BenchDirect: + def __init__(self): + # Imports/paths live inside the body: @Endpoint ships only the function + # body to the worker. + import logging + import os + import time + + import torch + from diffusers import StableDiffusionPipeline + + self.logger = logging.getLogger(__name__) + + hf = os.environ.get("HF_HOME", "/runpod-volume/hf") + # First run = the on-volume cache was empty before this cold start. + self._first_run = not (os.path.isdir(hf) and any(os.scandir(hf))) + + t0 = time.perf_counter() + self.pipe = StableDiffusionPipeline.from_pretrained( + "runwayml/stable-diffusion-v1-5", + torch_dtype=torch.float16, + safety_checker=None, + requires_safety_checker=False, + use_safetensors=True, + low_cpu_mem_usage=True, + ).to("cuda") + self._load_seconds = time.perf_counter() - t0 + self.logger.info( + f"[direct] load={self._load_seconds:.2f}s first_run={self._first_run}" + ) + + async def benchmark(self) -> dict: + """Report the cold-start metrics captured during __init__.""" + import os + + def dir_bytes(path): + total = 0 + for root, _dirs, files in os.walk(path): + for name in files: + try: + total += os.path.getsize(os.path.join(root, name)) + except OSError: + pass + return total + + return { + "mode": "direct", + "first_run": self._first_run, + "load_seconds": round(self._load_seconds, 2), + "hydrate_seconds": 0.0, + "volume_bytes": dir_bytes("/runpod-volume/hf"), + "local_bytes": 0, + } diff --git a/05_data_workflows/05_volume_warm_cache/benchmark/bench_volumecache.py b/05_data_workflows/05_volume_warm_cache/benchmark/bench_volumecache.py new file mode 100644 index 0000000..4ff0816 --- /dev/null +++ b/05_data_workflows/05_volume_warm_cache/benchmark/bench_volumecache.py @@ -0,0 +1,99 @@ +# Benchmark worker B: VOLUMECACHE strategy. +# +# HF cache stays on local disk; runpod.serverless.VolumeCache hydrates it from +# the network volume before the load and syncs new files back after. Reads are +# local; the volume is a warm mirror. Paired with bench_direct.py; driven by +# benchmark.py. +# +# PREREQUISITE: the deployed flash-worker image must include VolumeCache +# (SLS-367 / runpod-python PR #531). It cannot be added at runtime via the +# dependencies list because the worker's runpod is already imported before the +# handler runs. Until flash-worker ships it, this arm will fail to import +# VolumeCache — see benchmark/README.md. +import logging + +from runpod_flash import Endpoint, GpuType, NetworkVolume + +logger = logging.getLogger(__name__) + +# Same volume as bench_direct so both strategies compete on the same medium. +volume = NetworkVolume(name="flash-05-bench-volume", size=50) + + +@Endpoint( + name="bench_volumecache", + gpu=GpuType.NVIDIA_GEFORCE_RTX_5090, + workers=(0, 1), + idle_timeout=30, + volume=volume, + dependencies=["torch", "diffusers", "transformers", "accelerate"], +) +class BenchVolumeCache: + def __init__(self): + import logging + import os + import time + + import torch + from diffusers import StableDiffusionPipeline + from runpod.serverless import VolumeCache + + self.logger = logging.getLogger(__name__) + + # HF cache on local disk (default). VolumeCache mirrors it to the volume. + hf = os.path.expanduser("~/.cache/huggingface") + self._hf = hf + self._vc = VolumeCache(dirs=[hf]) + + # Mirror location is {volume}/.cache/{RUNPOD_ENDPOINT_ID}; empty = first run. + self._mirror = os.path.join( + "/runpod-volume", ".cache", os.environ.get("RUNPOD_ENDPOINT_ID", "") + ) + self._first_run = not ( + os.path.isdir(self._mirror) and any(os.scandir(self._mirror)) + ) + + t0 = time.perf_counter() + self._files_hydrated = self._vc.hydrate() # volume -> local + self._hydrate_seconds = time.perf_counter() - t0 + + self.pipe = StableDiffusionPipeline.from_pretrained( + "runwayml/stable-diffusion-v1-5", + torch_dtype=torch.float16, + safety_checker=None, + requires_safety_checker=False, + use_safetensors=True, + low_cpu_mem_usage=True, + ).to("cuda") + # Total cold-start-to-ready time, including the hydrate copy. + self._load_seconds = time.perf_counter() - t0 + + self._vc.sync() # local -> volume, in the background + self.logger.info( + f"[volumecache] load={self._load_seconds:.2f}s " + f"hydrate={self._hydrate_seconds:.2f}s first_run={self._first_run}" + ) + + async def benchmark(self) -> dict: + """Report the cold-start metrics captured during __init__.""" + import os + + def dir_bytes(path): + total = 0 + for root, _dirs, files in os.walk(path): + for name in files: + try: + total += os.path.getsize(os.path.join(root, name)) + except OSError: + pass + return total + + return { + "mode": "volumecache", + "first_run": self._first_run, + "files_hydrated": self._files_hydrated, + "load_seconds": round(self._load_seconds, 2), + "hydrate_seconds": round(self._hydrate_seconds, 2), + "local_bytes": dir_bytes(self._hf), + "volume_bytes": dir_bytes(self._mirror), + } diff --git a/05_data_workflows/05_volume_warm_cache/benchmark/benchmark.py b/05_data_workflows/05_volume_warm_cache/benchmark/benchmark.py new file mode 100644 index 0000000..e2e4c29 --- /dev/null +++ b/05_data_workflows/05_volume_warm_cache/benchmark/benchmark.py @@ -0,0 +1,121 @@ +#!/usr/bin/env python3 +"""Drive the direct vs VolumeCache cold-start A/B and print a comparison. + +Each worker measures its own model-load time during a cold start (in __init__) +and returns it from `benchmark()`. This runner triggers cold starts, collects +first-run and warm-run metrics for both strategies, and prints a table. + +Prerequisites: +- Both endpoints deployed/served (see benchmark/README.md). The volumecache arm + additionally requires a flash-worker image that includes VolumeCache (#531). +- The workers use idle_timeout=30, so waiting >~40s between trials lets the + worker scale to 0, making the next call a fresh cold start. + +Usage: + python benchmark.py --base-url http://localhost:8888 --warm-trials 3 + +Confirm the exact routes and response shape at /docs; adjust +--direct-route / --vc-route if your deployment differs. +""" + +import argparse +import json +import statistics +import time +import urllib.request + + +def call(base_url: str, route: str, timeout: int) -> dict: + """POST an empty job to a worker and return its benchmark metrics.""" + url = f"{base_url.rstrip('/')}/{route.strip('/')}" + body = json.dumps({"input": {}}).encode() + req = urllib.request.Request( + url, data=body, headers={"Content-Type": "application/json"} + ) + with urllib.request.urlopen(req, timeout=timeout) as resp: + payload = json.loads(resp.read()) + # flash runsync wraps the return value under "output"; tolerate either shape. + return payload.get("output", payload) + + +def run_arm(base_url, route, warm_trials, cold_wait, timeout): + """First-run (cold, empty cache) + N warm cold-starts for one strategy.""" + results = {"first_run": None, "warm": []} + print(f" [{route}] first run (empty cache; includes download)...") + results["first_run"] = call(base_url, route, timeout) + for i in range(warm_trials): + print( + f" [{route}] waiting {cold_wait}s for scale-to-0, then warm cold start {i + 1}/{warm_trials}..." + ) + time.sleep(cold_wait) + results["warm"].append(call(base_url, route, timeout)) + return results + + +def gb(n): + return f"{(n or 0) / (1024**3):.2f} GB" + + +def summarize(name, arm): + warm_loads = [r["load_seconds"] for r in arm["warm"]] or [float("nan")] + first = arm["first_run"] + warm0 = arm["warm"][0] if arm["warm"] else {} + return { + "name": name, + "first_run_load": first.get("load_seconds"), + "warm_load_mean": round(statistics.fmean(warm_loads), 2), + "warm_load_min": round(min(warm_loads), 2), + "hydrate_seconds": warm0.get("hydrate_seconds", 0.0), + "volume_bytes": warm0.get("volume_bytes", first.get("volume_bytes")), + "local_bytes": warm0.get("local_bytes", first.get("local_bytes")), + } + + +def main(): + ap = argparse.ArgumentParser() + ap.add_argument("--base-url", default="http://localhost:8888") + ap.add_argument("--direct-route", default="bench_direct/runsync") + ap.add_argument("--vc-route", default="bench_volumecache/runsync") + ap.add_argument("--warm-trials", type=int, default=3) + ap.add_argument( + "--cold-wait", type=int, default=45, help="seconds to wait for scale-to-0" + ) + ap.add_argument("--timeout", type=int, default=1200) + ap.add_argument( + "--skip-volumecache", + action="store_true", + help="run only the direct arm (e.g. before flash-worker ships VolumeCache)", + ) + args = ap.parse_args() + + print("Direct arm:") + direct = run_arm( + args.base_url, args.direct_route, args.warm_trials, args.cold_wait, args.timeout + ) + rows = [summarize("direct", direct)] + + if not args.skip_volumecache: + print("VolumeCache arm:") + vc = run_arm( + args.base_url, args.vc_route, args.warm_trials, args.cold_wait, args.timeout + ) + rows.append(summarize("volumecache", vc)) + + print("\n=== Results ===") + header = f"{'strategy':<14}{'first-run load':>16}{'warm load (mean)':>18}{'warm load (min)':>17}{'hydrate':>10}{'volume':>12}{'local':>12}" + print(header) + print("-" * len(header)) + for r in rows: + print( + f"{r['name']:<14}" + f"{str(r['first_run_load']) + 's':>16}" + f"{str(r['warm_load_mean']) + 's':>18}" + f"{str(r['warm_load_min']) + 's':>17}" + f"{str(r['hydrate_seconds']) + 's':>10}" + f"{gb(r['volume_bytes']):>12}" + f"{gb(r['local_bytes']):>12}" + ) + + +if __name__ == "__main__": + main() From 46beb28915e8975f271c80876e89c6e2c251918c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dean=20Qui=C3=B1anola?= Date: Wed, 8 Jul 2026 13:21:54 -0700 Subject: [PATCH 3/6] fix(examples): honor HF_HOME (flash-worker sets /hf-cache) in volumecache bench --- .../05_volume_warm_cache/benchmark/bench_volumecache.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/05_data_workflows/05_volume_warm_cache/benchmark/bench_volumecache.py b/05_data_workflows/05_volume_warm_cache/benchmark/bench_volumecache.py index 4ff0816..4618d60 100644 --- a/05_data_workflows/05_volume_warm_cache/benchmark/bench_volumecache.py +++ b/05_data_workflows/05_volume_warm_cache/benchmark/bench_volumecache.py @@ -40,8 +40,10 @@ def __init__(self): self.logger = logging.getLogger(__name__) - # HF cache on local disk (default). VolumeCache mirrors it to the volume. - hf = os.path.expanduser("~/.cache/huggingface") + # HF cache on local disk. The flash-worker image sets HF_HOME=/hf-cache, + # so honor it (falling back to the HF default) — VolumeCache mirrors that + # local dir to the volume. + hf = os.environ.get("HF_HOME") or os.path.expanduser("~/.cache/huggingface") self._hf = hf self._vc = VolumeCache(dirs=[hf]) From cdcbd3c059cbae9388a45169e6b0d3106d6b2f3c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dean=20Qui=C3=B1anola?= Date: Wed, 8 Jul 2026 18:09:15 -0700 Subject: [PATCH 4/6] test(examples): add rigorous pod-based VolumeCache vs direct benchmark --- .../benchmark/bench_volumecache.py | 5 +- .../benchmark/rigorous/README.md | 72 ++++++ .../benchmark/rigorous/benchmark.py | 225 ++++++++++++++++++ 3 files changed, 301 insertions(+), 1 deletion(-) create mode 100644 05_data_workflows/05_volume_warm_cache/benchmark/rigorous/README.md create mode 100644 05_data_workflows/05_volume_warm_cache/benchmark/rigorous/benchmark.py diff --git a/05_data_workflows/05_volume_warm_cache/benchmark/bench_volumecache.py b/05_data_workflows/05_volume_warm_cache/benchmark/bench_volumecache.py index 4618d60..4851e19 100644 --- a/05_data_workflows/05_volume_warm_cache/benchmark/bench_volumecache.py +++ b/05_data_workflows/05_volume_warm_cache/benchmark/bench_volumecache.py @@ -24,7 +24,10 @@ name="bench_volumecache", gpu=GpuType.NVIDIA_GEFORCE_RTX_5090, workers=(0, 1), - idle_timeout=30, + # Keep the worker alive long enough for the background mirror-back sync to + # finish before scale-to-0 (otherwise the volume mirror never populates and + # the next cold worker can't hydrate). + idle_timeout=180, volume=volume, dependencies=["torch", "diffusers", "transformers", "accelerate"], ) diff --git a/05_data_workflows/05_volume_warm_cache/benchmark/rigorous/README.md b/05_data_workflows/05_volume_warm_cache/benchmark/rigorous/README.md new file mode 100644 index 0000000..dd5c157 --- /dev/null +++ b/05_data_workflows/05_volume_warm_cache/benchmark/rigorous/README.md @@ -0,0 +1,72 @@ +# Rigorous benchmark: direct-on-volume vs VolumeCache + +A controlled A/B of two model-caching strategies on a Runpod network volume: + +- **direct** — HF cache lives on the volume; every load reads the model from the + network mount. +- **volumecache** — HF cache on local disk, mirrored to the volume; a fresh + worker hydrates (copies volume → local) once, then loads from local. + +This is a **Pod-based** benchmark, not serverless — on purpose. + +## Why not the serverless harness + +The serverless benchmark (one directory up) couldn't produce trustworthy numbers: + +- **Page cache.** A model loaded once sits in RAM; the next load hits RAM, not + disk, and looks instant (~1s). The serverless "warm" numbers were page-cache + hits, not storage reads. +- **No control over cold starts.** Forcing scale-to-0 via idle timeout + sleep + was unreliable; workers persisted and warm reuse contaminated the results. +- **Wrong question.** A single cold start doesn't reveal the tradeoff. On one + load, VolumeCache does *more* I/O than direct — it reads the model from the + volume during hydrate *and* again from local during load, while direct reads + from the volume once. VolumeCache can only win when the model is loaded enough + times per worker for cheaper local reads to amortize the one-time hydrate. + +This harness fixes all three: it runs as root and **drops the OS page cache +before every timed read**, controls cache state explicitly, repeats N trials for +variance, and reports the **per-reload cost and the breakeven point**. + +## What it measures (per strategy, N trials, warmup discarded, cold page cache) + +| Metric | Meaning | +| --- | --- | +| direct load (volume) | Load the model reading from the volume mount. | +| vc hydrate (volume→local) | Copy the mirror from volume to local disk. | +| vc first load (local) | Load right after hydrate (files warm in cache, as a real worker sees). | +| vc reload (local) | Load with the model already local, cold page cache — the steady-state local read. | +| storage | Bytes on the volume (direct + mirror) and local disk. | +| breakeven | How many loads per worker before VolumeCache beats direct. | + +## Running + +Provision a **Pod** with a GPU and the network volume attached (mounted at +`/runpod-volume`), then on the pod (as root): + +```bash +pip install "runpod @ git+https://github.com/runpod/runpod-python.git@deanquinanola/sls-367-network-volume-warm-cache-for-serverless-volumecache" \ + diffusers transformers accelerate torch +python benchmark.py --trials 5 --reloads 5 +``` + +First run downloads the model once to seed both caches (needs internet + HF +access); subsequent runs reuse them. + +## Interpreting + +- If **vc reload < direct load**, local reads beat volume reads; VolumeCache pays + off after `hydrate / (direct_load − vc_reload)` loads per worker. +- If **vc reload ≥ direct load**, the volume read isn't the bottleneck, so + VolumeCache only adds a hydrate copy + double storage — direct-on-volume is the + simpler, faster choice for this model/volume. + +The point isn't a single winner: it's the crossover. Frequent reloads or a slow +volume favor VolumeCache; a single load per cold start on a fast volume favors +direct. + +## Note + +This script uses `runpod.serverless.VolumeCache` directly (no Flash). It is +Flash-independent and could equally live in the runpod-python repo; it sits here +alongside the rest of the VolumeCache benchmark work. diff --git a/05_data_workflows/05_volume_warm_cache/benchmark/rigorous/benchmark.py b/05_data_workflows/05_volume_warm_cache/benchmark/rigorous/benchmark.py new file mode 100644 index 0000000..34c1c12 --- /dev/null +++ b/05_data_workflows/05_volume_warm_cache/benchmark/rigorous/benchmark.py @@ -0,0 +1,225 @@ +#!/usr/bin/env python3 +"""Rigorous A/B: HF cache directly on a network volume vs runpod VolumeCache. + +Runs on a Runpod **Pod** (GPU + a network volume at /runpod-volume), as root. +Deliberately NOT serverless: a pod gives a stable environment, root access to +drop the OS page cache between reads, and no cold-start scheduling noise — so we +measure the actual I/O tradeoff instead of worker-boot jitter. + +Why this design (vs the naive serverless benchmark): +- Page cache is controlled. A model loaded once sits in RAM; a second load then + hits RAM, not disk, and looks instant. We drop the page cache before every + timed read so we measure true storage reads, not cache hits. +- We measure the PER-RELOAD cost and the BREAKEVEN, not one cold start. Key + insight: on a single load, VolumeCache does more I/O than direct (it reads the + model from the volume during hydrate AND reads it again from local during + load; direct reads from the volume once). VolumeCache only wins when the model + is (re)loaded enough times per worker for cheaper local reads to amortize the + one-time hydrate copy. This harness finds that crossover. + +Setup on the pod: + pip install "runpod @ git+https://github.com/runpod/runpod-python.git@deanquinanola/sls-367-network-volume-warm-cache-for-serverless-volumecache" \ + diffusers transformers accelerate torch + python benchmark.py --trials 5 --reloads 5 + +Requires root (for /proc/sys/vm/drop_caches) and a mounted network volume. +""" + +import argparse +import gc +import os +import shutil +import statistics +import subprocess +import time + +MODEL = "runwayml/stable-diffusion-v1-5" +VOLUME = "/runpod-volume" +VOLUME_HF = f"{VOLUME}/bench-hf-direct" # direct strategy: HF cache ON the volume +LOCAL_HF = "/root/bench-hf-local" # volumecache strategy: HF cache on local disk +VC_NAMESPACE = "bench-volumecache" + + +def drop_caches(): + """Flush dirty pages and drop the OS page cache so the next read hits storage.""" + subprocess.run(["sync"], check=True) + try: + with open("/proc/sys/vm/drop_caches", "w") as fh: + fh.write("3\n") + except PermissionError: + raise SystemExit( + "Need root to drop the page cache (/proc/sys/vm/drop_caches). " + "Run this on a Pod as root." + ) + + +def dir_bytes(path): + total = 0 + for root, _dirs, files in os.walk(path): + for name in files: + try: + total += os.path.getsize(os.path.join(root, name)) + except OSError: + pass + return total + + +def load_model(cache_dir): + """Load the pipeline from cache_dir; return load seconds. Frees it after.""" + import torch + from diffusers import StableDiffusionPipeline + + t0 = time.perf_counter() + pipe = StableDiffusionPipeline.from_pretrained( + MODEL, + cache_dir=cache_dir, + torch_dtype=torch.float16, + safety_checker=None, + requires_safety_checker=False, + use_safetensors=True, + low_cpu_mem_usage=True, + local_files_only=True, # measure load from cache, never a download + ).to("cuda") + dt = time.perf_counter() - t0 + del pipe + gc.collect() + torch.cuda.empty_cache() + return dt + + +def stats(label, xs, unit="s"): + xs = list(xs) + return ( + f"{label:<28} n={len(xs)} " + f"mean={statistics.fmean(xs):.2f}{unit} " + f"median={statistics.median(xs):.2f}{unit} " + f"min={min(xs):.2f}{unit} max={max(xs):.2f}{unit} " + f"stdev={statistics.pstdev(xs):.2f}{unit}" + ) + + +def seed(vc): + """One-time: populate the on-volume direct cache and the VolumeCache mirror.""" + if dir_bytes(VOLUME_HF) == 0: + print("seeding direct cache on volume (download)...") + os.makedirs(VOLUME_HF, exist_ok=True) + _download(VOLUME_HF) + if dir_bytes(LOCAL_HF) == 0: + print("seeding local cache (download)...") + os.makedirs(LOCAL_HF, exist_ok=True) + _download(LOCAL_HF) + print("syncing local -> volume mirror (blocking)...") + t0 = time.perf_counter() + vc.sync(background=False) + print( + f" sync completed in {time.perf_counter() - t0:.1f}s; " + f"mirror={dir_bytes(vc._mirror_root) / 1e9:.2f}GB" + ) + + +def _download(cache_dir): + import torch + from diffusers import StableDiffusionPipeline + + StableDiffusionPipeline.from_pretrained( + MODEL, + cache_dir=cache_dir, + torch_dtype=torch.float16, + safety_checker=None, + requires_safety_checker=False, + use_safetensors=True, + ) + del torch + + +def main(): + ap = argparse.ArgumentParser() + ap.add_argument( + "--trials", type=int, default=5, help="first-load trials per strategy" + ) + ap.add_argument("--reloads", type=int, default=5, help="reload trials per strategy") + args = ap.parse_args() + + from runpod.serverless import VolumeCache + + if not os.path.isdir(VOLUME): + raise SystemExit(f"No network volume mounted at {VOLUME}.") + + vc = VolumeCache(dirs=[LOCAL_HF], namespace=VC_NAMESPACE, volume_path=VOLUME) + seed(vc) + + model_gb = dir_bytes(VOLUME_HF) / 1e9 + + # --- Direct: every load reads the model from the volume (cold page cache) --- + direct_loads = [] + for i in range(args.trials + 1): # +1 warmup, discarded + drop_caches() + dt = load_model(VOLUME_HF) + if i: + direct_loads.append(dt) + print(f" direct load [{i}] {dt:.2f}s{' (warmup)' if not i else ''}") + + # --- VolumeCache first load on a fresh worker: hydrate (volume->local) + load --- + vc_hydrate, vc_first_load = [], [] + for i in range(args.trials + 1): + shutil.rmtree( + LOCAL_HF, ignore_errors=True + ) # simulate a fresh worker's empty local disk + drop_caches() + t0 = time.perf_counter() + vc.hydrate() # copy mirror (volume) -> local + h = time.perf_counter() - t0 + # Load from local. In reality the just-copied files are warm in page cache + # (a real cold worker benefits from this), so we do NOT drop caches here. + ll = load_model(LOCAL_HF) + if i: + vc_hydrate.append(h) + vc_first_load.append(ll) + print( + f" vc first [{i}] hydrate={h:.2f}s load={ll:.2f}s{' (warmup)' if not i else ''}" + ) + + # --- Reload cost with the model already on local disk (cold page cache each) --- + vc_reloads = [] + for i in range(args.reloads + 1): + drop_caches() + dt = load_model(LOCAL_HF) + if i: + vc_reloads.append(dt) + print(f" vc reload (local) [{i}] {dt:.2f}s{' (warmup)' if not i else ''}") + + # --- Report --- + print("\n" + "=" * 72) + print(f"Model: {MODEL} (~{model_gb:.2f}GB) volume={VOLUME}") + print("=" * 72) + print(stats("direct load (volume)", direct_loads)) + print(stats("vc hydrate (volume->local)", vc_hydrate)) + print(stats("vc first load (local)", vc_first_load)) + print(stats("vc reload (local)", vc_reloads)) + print( + f"storage: volume(direct)={dir_bytes(VOLUME_HF) / 1e9:.2f}GB " + f"volume(mirror)={dir_bytes(vc._mirror_root) / 1e9:.2f}GB " + f"local={dir_bytes(LOCAL_HF) / 1e9:.2f}GB" + ) + + d = statistics.fmean(direct_loads) # per-load cost, direct (volume) + h = statistics.fmean(vc_hydrate) # one-time hydrate cost, volumecache + r = statistics.fmean(vc_reloads) # per-load cost, volumecache (local) + print("\n--- when does VolumeCache win? ---") + print(f"direct cost for K loads = K * {d:.2f}s") + print(f"volumecache cost for K loads = {h:.2f}s (hydrate) + K * {r:.2f}s") + if r < d: + breakeven = h / (d - r) + print( + f"local reload is {d - r:.2f}s cheaper than a volume load; " + f"VolumeCache breaks even at K = {breakeven:.1f} loads per worker." + ) + else: + print( + f"local reload ({r:.2f}s) is NOT cheaper than a volume load ({d:.2f}s) here, " + f"so VolumeCache never wins for this model/volume — direct is simpler and faster." + ) + + +if __name__ == "__main__": + main() From 2639c3fb09729a46f672b8b68f0731dcd27fb03e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dean=20Qui=C3=B1anola?= Date: Wed, 8 Jul 2026 19:58:17 -0700 Subject: [PATCH 5/6] test(examples): rigorous benchmark results (fadvise eviction, breakeven ~1 load) --- .../benchmark/rigorous/README.md | 22 ++++++++++ .../benchmark/rigorous/benchmark.py | 41 ++++++++++++------- 2 files changed, 48 insertions(+), 15 deletions(-) diff --git a/05_data_workflows/05_volume_warm_cache/benchmark/rigorous/README.md b/05_data_workflows/05_volume_warm_cache/benchmark/rigorous/README.md index dd5c157..03741e4 100644 --- a/05_data_workflows/05_volume_warm_cache/benchmark/rigorous/README.md +++ b/05_data_workflows/05_volume_warm_cache/benchmark/rigorous/README.md @@ -65,6 +65,28 @@ The point isn't a single winner: it's the crossover. Frequent reloads or a slow volume favor VolumeCache; a single load per cold start on a fast volume favors direct. +## Results (measured 2026-07-09) + +RTX 4090 pod, EU-RO-1, 50GB MooseFS network volume, `stable-diffusion-v1-5` +(~8.5GB), 5 trials each, page cache evicted (`posix_fadvise`) before every read. + +| metric | mean | stdev | +| --- | --- | --- | +| direct load (from volume) | **6.45s** | 0.19 | +| vc hydrate (volume→local copy) | 4.81s | 0.04 | +| vc first load (from local, after hydrate) | 2.21s | 0.02 | +| vc reload (from local, cold cache) | **1.76s** | 0.01 | + +Storage: direct = 8.5GB on the volume; VolumeCache = ~4.3GB volume mirror + local copy (roughly double). + +**Reading local is ~3.7× faster than reading the MooseFS volume** (1.76s vs 6.45s) — the volume read is a real bottleneck. + +**Breakeven ≈ 1 load per worker:** +- Load **once** per cold worker: direct ≈ 6.45s vs VolumeCache ≈ hydrate 4.81s + load 2.21s ≈ **7.0s** — direct is simpler and marginally faster. +- Load **twice or more** per worker: VolumeCache wins decisively — each extra reload is 1.76s vs 6.45s. `direct = K×6.45` vs `volumecache = 4.81 + K×1.76`. + +**Takeaway:** on this network volume, VolumeCache pays off when a worker (re)loads the model more than once (or for read-heavy repeated access); for a single load per cold start, caching directly on the volume is the simpler choice with equivalent latency. Both beat re-downloading from Hugging Face. + ## Note This script uses `runpod.serverless.VolumeCache` directly (no Flash). It is diff --git a/05_data_workflows/05_volume_warm_cache/benchmark/rigorous/benchmark.py b/05_data_workflows/05_volume_warm_cache/benchmark/rigorous/benchmark.py index 34c1c12..1fc0650 100644 --- a/05_data_workflows/05_volume_warm_cache/benchmark/rigorous/benchmark.py +++ b/05_data_workflows/05_volume_warm_cache/benchmark/rigorous/benchmark.py @@ -40,17 +40,26 @@ VC_NAMESPACE = "bench-volumecache" -def drop_caches(): - """Flush dirty pages and drop the OS page cache so the next read hits storage.""" - subprocess.run(["sync"], check=True) - try: - with open("/proc/sys/vm/drop_caches", "w") as fh: - fh.write("3\n") - except PermissionError: - raise SystemExit( - "Need root to drop the page cache (/proc/sys/vm/drop_caches). " - "Run this on a Pod as root." - ) +def evict(path): + """Drop each file's pages from the page cache so the next read hits storage. + + Containers can't write /proc/sys/vm/drop_caches (read-only, needs a + privileged host), so we evict per file with posix_fadvise(DONTNEED) — which + needs no root and targets exactly the tree we're about to read. + """ + subprocess.run(["sync"], check=False) + for root, _dirs, files in os.walk(path): + for name in files: + try: + fd = os.open(os.path.join(root, name), os.O_RDONLY) + except OSError: + continue + try: + os.posix_fadvise(fd, 0, 0, os.POSIX_FADV_DONTNEED) + except OSError: + pass + finally: + os.close(fd) def dir_bytes(path): @@ -78,7 +87,6 @@ def load_model(cache_dir): requires_safety_checker=False, use_safetensors=True, low_cpu_mem_usage=True, - local_files_only=True, # measure load from cache, never a download ).to("cuda") dt = time.perf_counter() - t0 del pipe @@ -118,6 +126,9 @@ def seed(vc): def _download(cache_dir): + # Download only what this load config needs (~4GB safetensors), not the full + # ~33GB repo (all .ckpt/.bin variants) — the volume quota and container disk + # can't hold two full copies. import torch from diffusers import StableDiffusionPipeline @@ -153,7 +164,7 @@ def main(): # --- Direct: every load reads the model from the volume (cold page cache) --- direct_loads = [] for i in range(args.trials + 1): # +1 warmup, discarded - drop_caches() + evict(VOLUME_HF) dt = load_model(VOLUME_HF) if i: direct_loads.append(dt) @@ -165,7 +176,7 @@ def main(): shutil.rmtree( LOCAL_HF, ignore_errors=True ) # simulate a fresh worker's empty local disk - drop_caches() + evict(vc._mirror_root) # hydrate should read the mirror cold from the volume t0 = time.perf_counter() vc.hydrate() # copy mirror (volume) -> local h = time.perf_counter() - t0 @@ -182,7 +193,7 @@ def main(): # --- Reload cost with the model already on local disk (cold page cache each) --- vc_reloads = [] for i in range(args.reloads + 1): - drop_caches() + evict(LOCAL_HF) dt = load_model(LOCAL_HF) if i: vc_reloads.append(dt) From b058f0123812926ded6a474f397c4f6ab7c4faf5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dean=20Qui=C3=B1anola?= Date: Fri, 10 Jul 2026 08:48:21 -0700 Subject: [PATCH 6/6] test(examples): add quadrant transport benchmark for VolumeCache adaptive transport Serverless CPU-endpoint benchmark sweeping file-shape quadrants (few/many x small/large + mixed HF) across serial/parallel/tar transport in both mirror and hydrate directions. Provides the empirical basis for the 256 KiB size-bucketed adaptive transport. Includes recorded results. --- .../05_volume_warm_cache/benchmark/.gitignore | 1 + .../benchmark/quadrant/README.md | 205 +++++++++++++++ .../benchmark/quadrant/bench_quadrant.py | 248 ++++++++++++++++++ .../benchmark/quadrant/benchmark.py | 136 ++++++++++ .../benchmark/quadrant/results.json | 188 +++++++++++++ 5 files changed, 778 insertions(+) create mode 100644 05_data_workflows/05_volume_warm_cache/benchmark/.gitignore create mode 100644 05_data_workflows/05_volume_warm_cache/benchmark/quadrant/README.md create mode 100644 05_data_workflows/05_volume_warm_cache/benchmark/quadrant/bench_quadrant.py create mode 100644 05_data_workflows/05_volume_warm_cache/benchmark/quadrant/benchmark.py create mode 100644 05_data_workflows/05_volume_warm_cache/benchmark/quadrant/results.json diff --git a/05_data_workflows/05_volume_warm_cache/benchmark/.gitignore b/05_data_workflows/05_volume_warm_cache/benchmark/.gitignore new file mode 100644 index 0000000..ddc7a88 --- /dev/null +++ b/05_data_workflows/05_volume_warm_cache/benchmark/.gitignore @@ -0,0 +1 @@ +.flash/ diff --git a/05_data_workflows/05_volume_warm_cache/benchmark/quadrant/README.md b/05_data_workflows/05_volume_warm_cache/benchmark/quadrant/README.md new file mode 100644 index 0000000..b3e4200 --- /dev/null +++ b/05_data_workflows/05_volume_warm_cache/benchmark/quadrant/README.md @@ -0,0 +1,205 @@ +# Quadrant benchmark: which transport strategy for which tree shape? + +The `rigorous/` benchmark answered *should you cache on the volume at all*. This +one answers the follow-up: **when VolumeCache moves a tree between local disk and +the network volume, which transport strategy is fastest — and does the answer +depend on the tree's shape?** + +VolumeCache today copies files one at a time. That is not obviously optimal. +Moving a cache tree pays two competing costs: + +- **bulk throughput** — bytes/sec the volume can stream. Dominates for large + files; the wire is the limit. +- **per-file metadata** — one open/create/stat round-trip per file over the + network mount. Dominates for many small files; latency is the limit. + +So the right strategy should depend on the tree's *shape*. This harness sweeps a +grid of shapes and, for each, times three strategies in both directions. + +## Why serverless (not a pod) + +This runs as a **Runpod serverless worker**, deliberately: + +- `/runpod-volume` on a worker is the **real MooseFS mount VolumeCache ships + against**. A pod is only a proxy for it; measuring on the actual substrate is + what makes the thresholds trustworthy. +- It's a **CPU endpoint, no GPU** — this is storage transport, not compute. +- Page cache is dropped before every timed op with `posix_fadvise(DONTNEED)`, + which needs no root and works inside the container — so none of the pod-only + machinery (privileged `drop_caches`, cold-start control) is required. +- We control cache/dir state explicitly *within one invocation*, so there are no + cold-start games to play — a single request measures clean transport. + +`VolumeCache` itself is not imported: the `serial` strategy **is** what +VolumeCache does today (per-file copy); `parallel` and `tar` are the candidates +we're evaluating to add. So this needs neither a GPU nor the #531 worker image. + +## The quadrants (tree shapes) + +| profile | files | per-file | total | stresses | +| --- | --- | --- | --- | --- | +| `few_small` | 8 | 4 MiB | 32 MiB | nothing — a cheap control | +| `few_large` | 8 | 256 MiB | 2 GiB | bulk throughput | +| `many_small` | 40,000 | 16 KiB | ~640 MiB | per-file metadata | +| `many_medium` | 3,000 | 1 MiB | ~3 GiB | the crossover zone | +| `mixed_hf` | 4 × 256 MiB + 8,000 × 8 KiB | — | ~2 GiB | both at once (like a real HF cache) | + +Sized to fit a CPU endpoint's container disk (vCPU × 10 GB — `CPU3G_4_16` gives +~40 GB) and to keep each profile's invocation to a few minutes. + +## The strategies + +| strategy | mirror (local → volume) | hydrate (volume → local) | +| --- | --- | --- | +| `serial` | `copy2` each file, one at a time (today's VolumeCache) | same, reversed | +| `parallel` | `copy2` each file across N threads (default 16) | same, reversed | +| `tar` | one `tar` archive written to the volume (packed, no compression) | `tar -x` from the volume | + +`tar` collapses N per-file metadata round-trips into a single sequential stream, +**but only if the volume stores the archive packed** — which the +write-once/read-many serverless cache lifecycle makes safe (one atomic writer +during mirror, readers only afterward). + +## Method + +- **N trials + 1 discarded warmup** per strategy per direction; reports mean, + stdev, and MiB/s. +- **Cold page cache before every timed op** via `posix_fadvise(DONTNEED)` on the + side about to be read. +- **Correctness gate.** After every hydrate the reconstructed tree is compared + byte-for-byte against the source; a mismatch aborts the run. +- **Idempotent, zero-filled sources** written to overlay disk (not tmpfs). `tar` + runs without compression and MooseFS doesn't dedup, so the bytes on the wire + are real. +- **One profile per invocation** — the full sweep moves hundreds of GB, too long + for a single request. The driver loops the profiles. + +## Running + +1. Deploy the worker (from this directory): + + ```bash + flash deploy bench_quadrant.py + ``` + + It's a CPU endpoint with the `flash-05-bench-volume` network volume attached; + no GPU, no extra dependencies. + +2. Drive it. Against a deployed endpoint: + + ```bash + RUNPOD_API_KEY=... python benchmark.py --endpoint-id --trials 2 + ``` + + Or against a local `flash dev` server (which dispatches to a real worker): + + ```bash + flash dev # in one shell + python benchmark.py --base-url http://localhost:8888 --trials 2 + ``` + + Confirm the route at `http://localhost:8888/docs` and pass `--route` if it + differs. `--only many_small,few_large` runs a subset. + +## Reading the output + +Per profile you get mean seconds and MiB/s per strategy per direction, then a +summary matrix — the fastest strategy for each quadrant: + +``` +quadrant files size mirror hydrate +few / large 8 2.00GiB parallel parallel +many / small 40000 0.62GiB tar tar +mixed (HF-like) 8004 2.06GiB ??? ??? +``` + +The crossover is the whole point: + +- **`tar` wins** → per-file metadata dominated; packing paid off. +- **`parallel` wins** → throughput dominated and threads hid the round-trips. +- **`serial` ties** → the tree was too small for strategy to matter. + +Where the winner *flips* between `many_small` and `few_large` is the empirical +threshold (file count and/or mean file size) an adaptive VolumeCache should +switch on. The `mixed_hf` row shows whether one strategy serves a realistic tree +or whether a **bucketed hybrid** (tar the small-file tail, parallel-copy the +large-file head) is warranted. + +## Results (measured 2026-07-09) + +Runpod CPU serverless (`cpu3g-4-16`), 50 GB MooseFS network volume in EU-RO-1, +page cache evicted before every timed op. `trials=2` except `many_small` +(`trials=1`). Raw data in `results.json`. + +### Summary matrix — fastest strategy per quadrant + +| quadrant | files | mean file | mirror winner | hydrate winner | +| --- | ---: | ---: | --- | --- | +| few / small | 8 | 4 MiB | parallel | parallel | +| few / large | 8 | 256 MiB | parallel | parallel | +| many / medium | 3,000 | 1 MiB | parallel | parallel | +| mixed (HF-like) | 8,004 | ~139 KiB* | **tar** | **tar** | +| many / small | 40,000 | 16 KiB | **tar** | **tar** | + +\* `mixed_hf` is bimodal: 4 × 256 MiB weights + 8,000 × 8 KiB metadata. + +### Per-profile mirror (local → volume), mean seconds (MiB/s) + +| profile | serial | parallel | tar | +| --- | --- | --- | --- | +| few_small | 0.09s (351) | **0.05s (659)** | 0.09s (355) | +| few_large | 2.67s (767) | **0.59s (3445)** | 5.32s (385) | +| many_medium | 31.5s (95) | **4.03s (745)** | 9.36s (321) | +| mixed_hf | 92.0s (12) | 12.6s (86) | **3.48s (312)** | +| many_small | 428.7s (1.5) | 62.0s (10) | **5.91s (106)** | + +Hydrate (volume → local) tracks the same winners; see `results.json`. + +### What the crossover says + +The deciding variable is **mean file size** — throughput vs per-file metadata: + +- **Large files (≥ ~1 MiB), any count → `parallel`.** On `few_large` and + `many_medium`, 16 threads overlap the modest metadata round-trips and saturate + throughput (3445 / 745 MiB/s). `tar` is pure overhead here — it serializes + everything through one stream and actually *lost* `few_large` mirror at 5.32s + vs parallel's 0.59s. +- **Many small files (≤ ~16 KiB) → `tar`.** On `many_small`, per-file metadata + dominates: `tar` collapses 40,000 round-trips into one sequential stream and + mirrors in **5.9s vs 62s parallel vs 428s serial** — **72× faster than serial, + 10× faster than parallel**. +- **`serial` (today's VolumeCache) never wins**, and degrades catastrophically + with file count: 428s to mirror 40k tiny files (1.5 MiB/s). It is not viable as + the sole strategy for small-file-heavy caches. +- **The mixed tree validates a bucketed hybrid.** `mixed_hf` is 1 GiB of big + weights + 8,000 tiny files; `tar` won overall (3.48s) because the small-file + *count* dominates the round-trip cost — but those 4 big files would move + fastest via `parallel`. One uniform strategy leaves time on the table. + +### Adaptive-transport thresholds for VolumeCache + +1. **Profile the tree during the walk** (free): file count and a size histogram. +2. **Bucket by size at ~256 KiB.** + - Files **< 256 KiB** → **tar** them into one packed archive on the volume + (metadata collapse). The crossover sits between `many_medium` (1 MiB → + parallel) and `mixed_hf`/`many_small` (≤139 KiB → tar); 256 KiB is a safe + switch point. + - Files **≥ 256 KiB** → **parallel** per-file copy (throughput + overlap). +3. **Degenerate cases collapse to one bucket:** an all-large tree is pure + parallel; an all-tiny tree is a single tar — no hybrid overhead. +4. **Write an on-volume manifest** recording the layout (which files are in the + archive vs copied) so hydrate is deterministic and can run the inverse in + parallel. The write-once/read-many serverless lifecycle makes the packed + archive safe: one atomic writer during mirror, readers only afterward. + +Net: replacing `serial` with this size-bucketed tar+parallel hybrid turns the +worst case (40k small files: 428s) into ~6s and keeps the large-file case at +full parallel throughput. + +## Note + +The `many_small` (`serial`, 40k files) arm ran 428s and pushed the invocation to +1,408s total — long enough that the driver's `runsync` returned before the job +finished. The result was recovered from the endpoint's job-status API. In +production a small-file cache would use `tar`, not `serial`, so this is a +benchmark artifact, not a usage path. diff --git a/05_data_workflows/05_volume_warm_cache/benchmark/quadrant/bench_quadrant.py b/05_data_workflows/05_volume_warm_cache/benchmark/quadrant/bench_quadrant.py new file mode 100644 index 0000000..3899403 --- /dev/null +++ b/05_data_workflows/05_volume_warm_cache/benchmark/quadrant/bench_quadrant.py @@ -0,0 +1,248 @@ +# Quadrant transport benchmark, as a Runpod serverless worker. +# +# Answers: when VolumeCache moves a cache tree between local disk and the network +# volume, which transport strategy is fastest — and does it depend on the tree's +# shape? Two costs compete: bulk throughput (large files) vs per-file metadata +# round-trips over the network mount (many small files). The right strategy +# should depend on file count x file size, so this sweeps a grid of shapes. +# +# Runs on serverless ON PURPOSE (not a pod): /runpod-volume here is the real +# MooseFS mount VolumeCache ships against, so we measure the actual substrate. +# CPU endpoint, no GPU — this is storage transport, not compute. VolumeCache +# itself isn't imported: the `serial` strategy IS what VolumeCache does today +# (per-file copy); `parallel` and `tar` are the candidates we're evaluating to +# add. So this needs neither a GPU nor the #531 worker image. +# +# One invocation runs ONE profile (the full sweep moves hundreds of GB — too +# long for a single request); the driver (benchmark.py) loops the profiles. +# +# Everything lives inside the handler method: @Endpoint ships only the function +# body to the worker, so module-level helpers/constants wouldn't exist remotely. +# +# deploy with: flash deploy (see README.md) +import logging + +from runpod_flash import CpuInstanceType, Endpoint, NetworkVolume + +logger = logging.getLogger(__name__) + +# Same volume as the other bench arms so every strategy competes on one medium. +volume = NetworkVolume(name="flash-05-bench-volume", size=50) + + +@Endpoint( + name="bench_quadrant", + cpu=CpuInstanceType.CPU3G_4_16, # 4 vCPU / 16GB RAM / ~40GB disk, no GPU + workers=(0, 1), + idle_timeout=60, + volume=volume, +) +class BenchQuadrant: + def __init__(self): + pass + + async def run( + self, profile: str = "many_small", trials: int = 2, workers: int = 16 + ) -> dict: + """Benchmark one profile: 3 strategies x 2 directions, cold page cache. + + Returns per-strategy mean mirror/hydrate seconds + MiB/s, plus the tree's + file count and total bytes. A byte-for-byte gate aborts on any mismatch. + """ + import os + import shutil + import statistics + import subprocess + import time + from concurrent.futures import ThreadPoolExecutor + + KIB, MIB, GIB = 1 << 10, 1 << 20, 1 << 30 + WRITE_CHUNK = 8 << 20 + ZERO = b"\x00" * WRITE_CHUNK + VOLUME = "/runpod-volume" + VOL_ROOT = f"{VOLUME}/quadrant" + LOCAL_SRC = "/quadrant-local/src" # overlay disk, not tmpfs + LOCAL_DST = "/quadrant-local/dst" + + # (files uniform) name -> (count, size_bytes); mixed uses parts below. + catalog = { + "few_small": [(8, 4 * MIB)], + "few_large": [(8, 256 * MIB)], + "many_small": [(40_000, 16 * KIB)], + "many_medium": [(3_000, 1 * MIB)], + "mixed_hf": [(4, 256 * MIB), (8_000, 8 * KIB)], + } + if profile not in catalog: + raise ValueError(f"unknown profile {profile!r}; have {list(catalog)}") + + def write_file(path, size): + fd = os.open(path, os.O_WRONLY | os.O_CREAT | os.O_TRUNC, 0o644) + try: + left = size + while left > 0: + n = WRITE_CHUNK if left >= WRITE_CHUNK else left + os.write(fd, ZERO if n == WRITE_CHUNK else ZERO[:n]) + left -= n + finally: + os.close(fd) + + def generate(root, parts): + # Fan files across subdirs (1000/dir) so a huge flat directory index + # doesn't dominate; idempotent so re-invocations reuse the tree. + base = 0 + for count, size in parts: + for i in range(count): + sub = os.path.join(root, f"p{base}", f"d{i // 1000:04d}") + os.makedirs(sub, exist_ok=True) + tgt = os.path.join(sub, f"f{i:06d}.bin") + if os.path.exists(tgt) and os.path.getsize(tgt) == size: + continue + write_file(tgt, size) + base += 1 + + def tree_stats(path): + count = total = 0 + for r, _d, fs in os.walk(path): + for name in fs: + try: + total += os.path.getsize(os.path.join(r, name)) + count += 1 + except OSError: + pass + return count, total + + def evict(path): + # Drop pages so the next read hits storage. Containers can't write + # /proc/sys/vm/drop_caches, so evict per file with posix_fadvise. + subprocess.run(["sync"], check=False) + if not hasattr(os, "posix_fadvise"): + return + for r, _d, fs in os.walk(path): + for name in fs: + try: + fd = os.open(os.path.join(r, name), os.O_RDONLY) + except OSError: + continue + try: + os.posix_fadvise(fd, 0, 0, os.POSIX_FADV_DONTNEED) + except OSError: + pass + finally: + os.close(fd) + + def pairs(src, dst): + for r, _d, fs in os.walk(src): + rel = os.path.relpath(r, src) + for name in fs: + yield os.path.join(r, name), os.path.join(dst, rel, name) + + def copy_serial(src, dst): + for s, d in pairs(src, dst): + os.makedirs(os.path.dirname(d), exist_ok=True) + shutil.copy2(s, d) + + def copy_parallel(src, dst): + for r, _d, _fs in os.walk(src): + os.makedirs(os.path.join(dst, os.path.relpath(r, src)), exist_ok=True) + with ThreadPoolExecutor(max_workers=workers) as pool: + list(pool.map(lambda p: shutil.copy2(*p), pairs(src, dst))) + + def tar_pack(src, area): + subprocess.run( + ["tar", "-C", src, "-cf", os.path.join(area, "cache.tar"), "."], + check=True, + ) + + def tar_unpack(area, dst): + subprocess.run( + ["tar", "-C", dst, "-xf", os.path.join(area, "cache.tar")], check=True + ) + + # strategy -> (pack(src, area), unpack(area, dst), evict_source(area)) + mirror_sub = "mirror" + strategies = { + "serial": ( + lambda s, a: copy_serial(s, os.path.join(a, mirror_sub)), + lambda a, d: copy_serial(os.path.join(a, mirror_sub), d), + lambda a: os.path.join(a, mirror_sub), + ), + "parallel": ( + lambda s, a: copy_parallel(s, os.path.join(a, mirror_sub)), + lambda a, d: copy_parallel(os.path.join(a, mirror_sub), d), + lambda a: os.path.join(a, mirror_sub), + ), + } + have_tar = shutil.which("tar") is not None + if have_tar: + strategies["tar"] = (tar_pack, tar_unpack, lambda a: a) + + def timed(fn): + t0 = time.perf_counter() + fn() + return time.perf_counter() - t0 + + # --- generate the source tree once (reused across strategies) --- + src = os.path.join(LOCAL_SRC, profile) + os.makedirs(src, exist_ok=True) + os.makedirs(VOL_ROOT, exist_ok=True) + generate(src, catalog[profile]) + count, total = tree_stats(src) + + results = {} + for name, (pack, unpack, evict_src) in strategies.items(): + area = os.path.join(VOL_ROOT, profile, name) + dst = os.path.join(LOCAL_DST, profile, name) + mirror_t, hydrate_t = [], [] + for i in range(trials + 1): # +1 warmup, discarded + shutil.rmtree(area, ignore_errors=True) + os.makedirs(area, exist_ok=True) + evict(src) + m = timed(lambda: pack(src, area)) + + shutil.rmtree(dst, ignore_errors=True) + os.makedirs(dst, exist_ok=True) + evict(evict_src(area)) + h = timed(lambda: unpack(area, dst)) + + got_count, got_bytes = tree_stats(dst) + if got_bytes != total: + raise RuntimeError( + f"{name}: hydrate mismatch — expected {total} bytes, " + f"got {got_bytes} across {got_count} files" + ) + if i: + mirror_t.append(m) + hydrate_t.append(h) + shutil.rmtree(area, ignore_errors=True) + shutil.rmtree(dst, ignore_errors=True) + + def mbps(secs): + return round((total / MIB) / secs, 1) if secs else 0.0 + + mm, hh = statistics.fmean(mirror_t), statistics.fmean(hydrate_t) + results[name] = { + "mirror_s": round(mm, 3), + "hydrate_s": round(hh, 3), + "mirror_mibps": mbps(mm), + "hydrate_mibps": mbps(hh), + "mirror_stdev": round(statistics.pstdev(mirror_t), 3), + "hydrate_stdev": round(statistics.pstdev(hydrate_t), 3), + } + + # Clean the volume so repeated runs and the sibling arms start fresh. + shutil.rmtree(os.path.join(VOL_ROOT, profile), ignore_errors=True) + + mirror_winner = min(results, key=lambda k: results[k]["mirror_s"]) + hydrate_winner = min(results, key=lambda k: results[k]["hydrate_s"]) + return { + "profile": profile, + "file_count": count, + "total_bytes": total, + "total_gib": round(total / GIB, 3), + "trials": trials, + "workers": workers, + "tar_available": have_tar, + "strategies": results, + "mirror_winner": mirror_winner, + "hydrate_winner": hydrate_winner, + } diff --git a/05_data_workflows/05_volume_warm_cache/benchmark/quadrant/benchmark.py b/05_data_workflows/05_volume_warm_cache/benchmark/quadrant/benchmark.py new file mode 100644 index 0000000..2161407 --- /dev/null +++ b/05_data_workflows/05_volume_warm_cache/benchmark/quadrant/benchmark.py @@ -0,0 +1,136 @@ +#!/usr/bin/env python3 +"""Drive the quadrant transport benchmark and print the crossover matrix. + +The worker (bench_quadrant.py) runs one profile per invocation — 3 strategies +(serial / parallel / tar) x 2 directions (mirror local->volume, hydrate +volume->local) against the real /runpod-volume mount, with the page cache +dropped before every timed op. This runner calls it once per profile and renders +the fastest strategy per tree shape — the empirical basis for adaptive +transport thresholds in VolumeCache. + +Two ways to reach the worker: + + flash dev (local dev server, dispatches to a real worker): + python benchmark.py --base-url http://localhost:8888 + + deployed endpoint (flash deploy): + RUNPOD_API_KEY=... python benchmark.py --endpoint-id + +Confirm the exact route at /docs (dev) — pass --route if it differs. +Results are printed and written to results.json for the README. +""" + +import argparse +import json +import os +import time +import urllib.request + +PROFILES = ("few_small", "few_large", "many_small", "many_medium", "mixed_hf") +QUADRANT_LABEL = { + "few_small": "few / small", + "few_large": "few / large", + "many_small": "many / small", + "many_medium": "many / medium", + "mixed_hf": "mixed (HF-like)", +} + + +def _post(url: str, body: dict, headers: dict, timeout: int) -> dict: + data = json.dumps(body).encode() + req = urllib.request.Request(url, data=data, headers=headers) + with urllib.request.urlopen(req, timeout=timeout) as resp: + return json.loads(resp.read()) + + +def call_profile(args, profile: str) -> dict: + """Invoke the worker for one profile; return its output dict.""" + payload = { + "input": {"profile": profile, "trials": args.trials, "workers": args.workers} + } + headers = {"Content-Type": "application/json"} + if args.endpoint_id: + url = f"https://api.runpod.ai/v2/{args.endpoint_id}/runsync" + key = os.environ.get("RUNPOD_API_KEY") + if not key: + raise SystemExit("RUNPOD_API_KEY is required with --endpoint-id") + headers["Authorization"] = f"Bearer {key}" + else: + url = f"{args.base_url.rstrip('/')}/{args.route.strip('/')}" + resp = _post(url, payload, headers, args.timeout) + # flash runsync / runpod both wrap the return under "output". + return resp.get("output", resp) + + +def _mibps_row(name: str, s: dict) -> str: + return ( + f" {name:<10} " + f"{s['mirror_s']:>7.2f}s ({s['mirror_mibps']:>7.1f}) " + f"{s['hydrate_s']:>7.2f}s ({s['hydrate_mibps']:>7.1f})" + ) + + +def main() -> None: + ap = argparse.ArgumentParser(description=__doc__) + ap.add_argument("--base-url", default="http://localhost:8888") + ap.add_argument("--route", default="bench_quadrant/run/runsync") + ap.add_argument( + "--endpoint-id", default="", help="deployed endpoint id (uses RUNPOD_API_KEY)" + ) + ap.add_argument("--trials", type=int, default=2) + ap.add_argument("--workers", type=int, default=16) + ap.add_argument("--timeout", type=int, default=1800) + ap.add_argument( + "--only", default="", help="comma-separated profiles (default: all)" + ) + ap.add_argument("--out", default="results.json") + args = ap.parse_args() + + selected = [p for p in PROFILES if not args.only or p in args.only.split(",")] + if not selected: + raise SystemExit(f"No profiles matched --only={args.only!r}") + + results = {} + for profile in selected: + print(f"\n=== {profile} ({QUADRANT_LABEL[profile]}) ===") + t0 = time.perf_counter() + out = call_profile(args, profile) + results[profile] = out + strat = out["strategies"] + print( + f" {out['file_count']} files, {out['total_gib']:.2f}GiB " + f"(took {time.perf_counter() - t0:.0f}s)" + ) + print(f" {'strategy':<10} {'mirror (L->V)':<20} {'hydrate (V->L)':<20}") + for name in strat: + print(_mibps_row(name, strat[name])) + print( + f" winner: mirror={out['mirror_winner']} hydrate={out['hydrate_winner']}" + ) + + print("\n" + "=" * 72) + print("SUMMARY — fastest strategy per quadrant") + print("=" * 72) + print(f"{'quadrant':<18} {'files':>7} {'size':>10} {'mirror':<10} {'hydrate':<10}") + for profile in selected: + out = results[profile] + print( + f"{QUADRANT_LABEL[profile]:<18} " + f"{out['file_count']:>7} " + f"{out['total_gib']:>8.2f}GiB " + f"{out['mirror_winner']:<10} " + f"{out['hydrate_winner']:<10}" + ) + print( + "\nWhere 'tar' wins, per-file metadata dominated; where 'parallel' wins, " + "throughput did and threads hid the round-trips; where 'serial' ties, the " + "tree was too small to matter." + ) + + with open(args.out, "w") as fh: + json.dump(results, fh, indent=2) + print(f"\nwrote {args.out}") + + +if __name__ == "__main__": + main() diff --git a/05_data_workflows/05_volume_warm_cache/benchmark/quadrant/results.json b/05_data_workflows/05_volume_warm_cache/benchmark/quadrant/results.json new file mode 100644 index 0000000..93968bd --- /dev/null +++ b/05_data_workflows/05_volume_warm_cache/benchmark/quadrant/results.json @@ -0,0 +1,188 @@ +{ + "few_small": { + "profile": "few_small", + "file_count": 8, + "total_bytes": 33554432, + "total_gib": 0.031, + "trials": 2, + "workers": 16, + "tar_available": true, + "strategies": { + "serial": { + "mirror_s": 0.091, + "hydrate_s": 0.068, + "mirror_mibps": 351.4, + "hydrate_mibps": 468.8, + "mirror_stdev": 0.003, + "hydrate_stdev": 0.001 + }, + "parallel": { + "mirror_s": 0.049, + "hydrate_s": 0.024, + "mirror_mibps": 659.1, + "hydrate_mibps": 1325.0, + "mirror_stdev": 0.001, + "hydrate_stdev": 0.004 + }, + "tar": { + "mirror_s": 0.09, + "hydrate_s": 0.031, + "mirror_mibps": 354.6, + "hydrate_mibps": 1041.6, + "mirror_stdev": 0.0, + "hydrate_stdev": 0.001 + } + }, + "mirror_winner": "parallel", + "hydrate_winner": "parallel" + }, + "few_large": { + "profile": "few_large", + "file_count": 8, + "total_bytes": 2147483648, + "total_gib": 2.0, + "trials": 2, + "workers": 16, + "tar_available": true, + "strategies": { + "serial": { + "mirror_s": 2.669, + "hydrate_s": 1.564, + "mirror_mibps": 767.2, + "hydrate_mibps": 1309.5, + "mirror_stdev": 0.041, + "hydrate_stdev": 0.164 + }, + "parallel": { + "mirror_s": 0.594, + "hydrate_s": 0.788, + "mirror_mibps": 3445.4, + "hydrate_mibps": 2600.3, + "mirror_stdev": 0.009, + "hydrate_stdev": 0.019 + }, + "tar": { + "mirror_s": 5.318, + "hydrate_s": 1.48, + "mirror_mibps": 385.1, + "hydrate_mibps": 1383.9, + "mirror_stdev": 0.279, + "hydrate_stdev": 0.124 + } + }, + "mirror_winner": "parallel", + "hydrate_winner": "parallel" + }, + "many_medium": { + "profile": "many_medium", + "file_count": 3000, + "total_bytes": 3145728000, + "total_gib": 2.93, + "trials": 2, + "workers": 16, + "tar_available": true, + "strategies": { + "serial": { + "mirror_s": 31.479, + "hydrate_s": 14.613, + "mirror_mibps": 95.3, + "hydrate_mibps": 205.3, + "mirror_stdev": 0.29, + "hydrate_stdev": 0.046 + }, + "parallel": { + "mirror_s": 4.029, + "hydrate_s": 1.861, + "mirror_mibps": 744.7, + "hydrate_mibps": 1611.9, + "mirror_stdev": 0.024, + "hydrate_stdev": 0.005 + }, + "tar": { + "mirror_s": 9.359, + "hydrate_s": 2.146, + "mirror_mibps": 320.6, + "hydrate_mibps": 1398.2, + "mirror_stdev": 0.057, + "hydrate_stdev": 0.099 + } + }, + "mirror_winner": "parallel", + "hydrate_winner": "parallel" + }, + "mixed_hf": { + "profile": "mixed_hf", + "file_count": 8004, + "total_bytes": 1139277824, + "total_gib": 1.061, + "trials": 2, + "workers": 16, + "tar_available": true, + "strategies": { + "serial": { + "mirror_s": 92.037, + "hydrate_s": 27.71, + "mirror_mibps": 11.8, + "hydrate_mibps": 39.2, + "mirror_stdev": 0.936, + "hydrate_stdev": 0.024 + }, + "parallel": { + "mirror_s": 12.644, + "hydrate_s": 3.5, + "mirror_mibps": 85.9, + "hydrate_mibps": 310.4, + "mirror_stdev": 0.627, + "hydrate_stdev": 0.035 + }, + "tar": { + "mirror_s": 3.479, + "hydrate_s": 0.852, + "mirror_mibps": 312.3, + "hydrate_mibps": 1275.1, + "mirror_stdev": 0.134, + "hydrate_stdev": 0.014 + } + }, + "mirror_winner": "tar", + "hydrate_winner": "tar" + }, + "many_small": { + "profile": "many_small", + "file_count": 40000, + "total_bytes": 655360000, + "total_gib": 0.61, + "trials": 1, + "workers": 16, + "tar_available": true, + "strategies": { + "serial": { + "mirror_s": 428.725, + "hydrate_s": 161.794, + "mirror_mibps": 1.5, + "hydrate_mibps": 3.9, + "mirror_stdev": 0.0, + "hydrate_stdev": 0.0 + }, + "parallel": { + "mirror_s": 62.048, + "hydrate_s": 14.128, + "mirror_mibps": 10.1, + "hydrate_mibps": 44.2, + "mirror_stdev": 0.0, + "hydrate_stdev": 0.0 + }, + "tar": { + "mirror_s": 5.913, + "hydrate_s": 1.116, + "mirror_mibps": 105.7, + "hydrate_mibps": 559.9, + "mirror_stdev": 0.0, + "hydrate_stdev": 0.0 + } + }, + "mirror_winner": "tar", + "hydrate_winner": "tar", + "note": "trials=1; serial arm ran 428s (exec 1408s total) \u2014 captured via API status recovery after the driver's runsync timed out client-side." + } +} \ No newline at end of file