From 19e68f984d4d821d380eb9fcad9040f0a7471be1 Mon Sep 17 00:00:00 2001 From: Roni bhakta <77425964+ronibhakta1@users.noreply.github.com> Date: Tue, 21 Jul 2026 00:08:47 +0530 Subject: [PATCH] fix: optimize concurrency settings and update documentation for performance improvements --- app/config/settings.py | 7 ++++-- app/core/synthesizer.py | 23 ++++++++++++++---- app/providers/pocket_tts.py | 17 +++++++++---- docs/configuration.md | 8 +++---- docs/providers/pocket.md | 23 ++++++++++++++++-- docs/voices.md | 2 +- scripts/configure.sh | 48 +++++++++++++++++++++++++++++++++---- 7 files changed, 104 insertions(+), 24 deletions(-) diff --git a/app/config/settings.py b/app/config/settings.py index ef56abd..5288552 100644 --- a/app/config/settings.py +++ b/app/config/settings.py @@ -25,8 +25,11 @@ class Settings(BaseSettings): api_key_enabled: bool = False api_key: str = "" - # Concurrency - max_concurrent_syntheses: int = Field(default=2, ge=1) + # Concurrency. Safe hardware-blind floor: 1 = never oversubscribe CPU. Each + # synthesis uses ~2 cores (pocket-tts's generate→decode pipeline), so a 2-core box + # thrashes at 2 under mixed-language load. configure.sh derives a higher value from + # nproc/RAM for bigger hardware. + max_concurrent_syntheses: int = Field(default=1, ge=1) # Providers enabled_providers: str = "pocket" diff --git a/app/core/synthesizer.py b/app/core/synthesizer.py index 3cda899..178e218 100644 --- a/app/core/synthesizer.py +++ b/app/core/synthesizer.py @@ -1,5 +1,6 @@ import logging import struct +import time from app.api.errors import AppError, PayloadTooLarge, RequestValidationFailed, ServiceNotReady from app.config.settings import settings @@ -69,12 +70,11 @@ async def synthesize( out = request.output logger.info( - "synthesize voice=%s provider=%s format=%s chars=%d boundary=%s", + "synth start voice=%s provider=%s fmt=%s chars=%d", voice.identifier, provider.id, out.format.value, len(text), - request.boundary, ) params = SynthesisParams( text=text, @@ -91,6 +91,7 @@ async def synthesize( if breaker is not None and not breaker.allow(): raise ServiceNotReady(f"Provider '{provider.id}' is temporarily unavailable") + t0 = time.monotonic() try: result: AudioResult = await provider.synthesize(params) except AppError: @@ -100,12 +101,20 @@ async def synthesize( else: if breaker is not None: breaker.record_success() + gen_ms = round((time.monotonic() - t0) * 1000, 1) if out.format == AudioFormat.WAV: audio = _pcm_to_wav(result.pcm, result.sample_rate) - logger.info("generated wav pcm_bytes=%d output_bytes=%d", len(result.pcm), len(audio)) + logger.info( + "synth done voice=%s fmt=wav gen_ms=%.1f total_ms=%.1f bytes=%d", + voice.identifier, + gen_ms, + round((time.monotonic() - t0) * 1000, 1), + len(audio), + ) return (audio, "audio/wav", result.boundaries, boundaries_supported) + enc_t = time.monotonic() encoded = await ffmpeg_driver.encode( result.pcm, result.sample_rate, @@ -113,10 +122,14 @@ async def synthesize( ffmpeg_bin=settings.ffmpeg_bin, bitrate=out.bitrate, ) + enc_ms = round((time.monotonic() - enc_t) * 1000, 1) logger.info( - "generated %s pcm_bytes=%d output_bytes=%d", + "synth done voice=%s fmt=%s gen_ms=%.1f enc_ms=%.1f total_ms=%.1f bytes=%d", + voice.identifier, out.format.value, - len(result.pcm), + gen_ms, + enc_ms, + round((time.monotonic() - t0) * 1000, 1), len(encoded), ) return ( diff --git a/app/providers/pocket_tts.py b/app/providers/pocket_tts.py index 9e3a469..947e27a 100644 --- a/app/providers/pocket_tts.py +++ b/app/providers/pocket_tts.py @@ -23,13 +23,17 @@ _VOICES_PATH = Path(__file__).parent.parent / "data" / "voices" / "pocket" / "voices.json" -# BCP-47 prefix → pocket-tts language identifier +# BCP-47 prefix → pocket-tts language identifier. Default to the fast/small 6-layer model for +# every language that has one. French is the exception: pocket-tts ships only a 24-layer French +# (`french_24l`), no 6-layer variant. de/es/it/pt/en also have 24-layer (`_24l`) variants that are +# higher quality but ~3× larger and ~4× slower — not used here. Making this selectable is deferred +# (see issue); flip a value to its `_24l` form to opt a language into the heavier model. _LANG_MODEL: dict[str, str] = { "en": "english", "fr": "french_24l", "it": "italian", - "de": "german_24l", - "es": "spanish_24l", + "de": "german", + "es": "spanish", "pt": "portuguese", } @@ -59,10 +63,13 @@ async def load(self) -> None: self._ready = await anyio.to_thread.run_sync(self._load_sync) def _load_sync(self) -> bool: - import torch from pocket_tts import TTSModel - torch.set_num_threads(1) + # Do NOT set torch.set_num_threads() here. pocket-tts pins it to 1 itself on + # import (models/tts_model.py) by design: each model runs a 2-thread + # generate→decode pipeline that already saturates its documented "2 CPU cores" + # ceiling. Raising intra-op threads oversubscribes those pipeline threads and + # makes inference SLOWER, not faster. Scale throughput with WORKERS, not threads. enabled = self.active_languages() if not enabled: diff --git a/docs/configuration.md b/docs/configuration.md index bbb9931..e472f38 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -70,7 +70,7 @@ only ever *adds* cross-language support: |---|---|---| | `HF_TOKEN` | _(empty)_ | HuggingFace token. Optional — prevents rate-limiting on first-run model downloads. Shared across any provider that pulls models from HF | | `WORKERS` | `1` | Uvicorn worker processes. Each loads a full copy of every active language model | -| `MAX_CONCURRENT_SYNTHESES` | `2` | Max parallel CPU inference jobs per worker | +| `MAX_CONCURRENT_SYNTHESES` | `1` | Max parallel CPU inference jobs per worker. Each job uses ~2 cores; `configure.sh` derives this from `nproc`/RAM so cores ≈ workers × concurrency × 2 (no oversubscription) | | `CIRCUIT_BREAKER_ENABLED` | `true` | Trip a provider's circuit breaker after repeated `synthesize()` failures, returning `503` immediately instead of hammering a broken provider | | `CIRCUIT_BREAKER_FAILURE_THRESHOLD` | `5` | Consecutive failures before a provider's breaker opens | | `CIRCUIT_BREAKER_RECOVERY_SECONDS` | `30` | How long an open breaker waits before allowing one trial call | @@ -88,7 +88,7 @@ only ever *adds* cross-language support: | Variable | Default | Description | |---|---|---| -| `LANGUAGES` | _(empty)_ | Comma-separated BCP-47 language codes to **load as base models**. No hardcoded fallback — env-driven; unset means no base models load (no voices served). Size per language is *not* uniform: the 6-layer `en`/`it`/`pt` models are ~219 MB download / ~438 MB RAM; the 24-layer `fr`/`de`/`es` models are ~672 MB download / ~1344 MB RAM (loaded size ~2x the download, measured at startup). Supported: `en fr it de es pt`. This is the ceiling — nothing below can exceed it | +| `LANGUAGES` | _(empty)_ | Comma-separated BCP-47 language codes to **load as base models**. No hardcoded fallback — env-driven; unset means no base models load (no voices served). Size per language is *not* uniform: the 6-layer `en`/`it`/`de`/`es`/`pt` models are ~219 MB download / ~438 MB RAM; the 24-layer `fr` model is ~672 MB download / ~1344 MB RAM (loaded size ~2x the download, measured at startup). Supported: `en fr it de es pt`. This is the ceiling — nothing below can exceed it | | `VOICE_LANGUAGES` | _(empty)_ | Which voices get warmed against which of those loaded models, beyond each voice's own primary — see [Per-voice language overrides](#per-voice-language-overrides) | | `POCKET_DEFAULT_VOICE` | _(empty)_ | Default voice when none is specified. No hardcoded fallback — env-driven; empty means the setting is unset | @@ -144,8 +144,8 @@ are approximate. | Languages | Layers | Download (disk, once) | RAM (per worker) | |---|---|---|---| -| `en` `it` `pt` | 6 | ~219 MB | ~438 MB | -| `fr` `de` `es` | 24 (`_24l`) | ~672 MB | ~1344 MB | +| `en` `it` `de` `es` `pt` | 6 | ~219 MB | ~438 MB | +| `fr` | 24 (`_24l`) | ~672 MB | ~1344 MB | RAM ≈ 2× the download (measured at startup). This is the dominant cost. diff --git a/docs/providers/pocket.md b/docs/providers/pocket.md index 785ffad..c0f7662 100644 --- a/docs/providers/pocket.md +++ b/docs/providers/pocket.md @@ -51,5 +51,24 @@ re-encoding (the default, and fastest — no ffmpeg). The server can transcode o ## Constraints -- CPU only; torch ≥ 2.5 CPU build. `en`/`it`/`pt` are 6-layer models; `fr`/`de`/`es` are larger - 24-layer models (slower, higher quality). Throughput scales by worker processes. +- CPU only; torch ≥ 2.5 CPU build. Throughput scales by worker processes. +- **Model variant:** most languages ship both a fast/small **6-layer** model and a slower/bigger + higher-quality **24-layer** (`_24l`) one. We default to **6-layer** everywhere it exists. + French is the exception — pocket-tts ships only a 24-layer French, so `fr` is always 24-layer. + + | Lang | 6-layer | 24-layer | Used | + |---|---|---|---| + | `en` | `english` | — | 6-layer | + | `fr` | — | `french_24l` | **24-layer (only option)** | + | `it` | `italian` | `italian_24l` | 6-layer | + | `de` | `german` | `german_24l` | 6-layer | + | `es` | `spanish` | `spanish_24l` | 6-layer | + | `pt` | `portuguese` | `portuguese_24l` | 6-layer | + + (This is why you see `french_24l` but plain `spanish`/`german` — not an inconsistency; French + has no 6-layer.) The mapping lives in `_LANG_MODEL` (`app/providers/pocket_tts.py`). Making the + 6l/24l choice user-selectable per language is **deferred** pending need — see the follow-up + issue. +- Each model uses ~2 CPU cores via an internal generate→decode pipeline and pins torch to a + single intra-op thread itself (on import) — so we don't set `torch.set_num_threads()`. Give + the box more cores by running more workers, not more threads per model. diff --git a/docs/voices.md b/docs/voices.md index 2241e32..e938fc6 100644 --- a/docs/voices.md +++ b/docs/voices.md @@ -52,7 +52,7 @@ languages without repeating it per voice. PocketTTS pre-computes voice embeddings per voice per language (stored as `.safetensors` files under `kyutai/pocket-tts-without-voice-cloning`), sized by the target language's model: -~5–8 MB into the 6-layer `en`/`it`/`pt` models, ~24–33 MB into the 24-layer `fr`/`de`/`es`. The embedding is warmed once at +~5–8 MB into the 6-layer `en`/`it`/`de`/`es`/`pt` models, ~24–33 MB into the 24-layer `fr`. The embedding is warmed once at model-load time — no per-request cloning overhead — but only for the (voice, language) pairs `VOICE_LANGUAGES` and `LANGUAGES` actually call for. Unused language weights can be reclaimed from the Docker volume with `scripts/prune_weights.py` — see [Configuration → disk space](configuration.md). diff --git a/scripts/configure.sh b/scripts/configure.sh index 0ebed94..e8240a2 100755 --- a/scripts/configure.sh +++ b/scripts/configure.sh @@ -53,6 +53,36 @@ set_env() { _set_kv "$ENV_FILE" "$1" "$2"; } get_penv() { _get_kv "$POCKET_ENV_FILE" "$1"; } set_penv() { _set_kv "$POCKET_ENV_FILE" "$1" "$2"; } +# Hardware-derived defaults. pocket-tts uses ~2 CPU cores per synthesis (its +# generate→decode pipeline; docs: "Uses only 2 CPU cores") and each worker holds a +# full model copy (~2 GB for en+fr). So 1 worker ≈ 2 cores + ~2 GB RAM. Derive +# non-thrashing WORKERS / MAX_CONCURRENT_SYNTHESES from the box instead of shipping a +# static default that only fits one machine. Sets DETECTED_* and SUGGESTED_* globals. +_detect_hw() { + local mem_bytes w_by_core w_by_ram + if [[ "$(uname)" == "Darwin" ]]; then + DETECTED_CORES=$(sysctl -n hw.ncpu 2>/dev/null || echo 2) + mem_bytes=$(sysctl -n hw.memsize 2>/dev/null || echo 0) + DETECTED_RAM_GB=$(( mem_bytes / 1024 / 1024 / 1024 )) + else + DETECTED_CORES=$(nproc 2>/dev/null || echo 2) + DETECTED_RAM_GB=$(awk '/MemTotal/ {print int($2/1024/1024)}' /proc/meminfo 2>/dev/null || echo 0) + fi + [[ "$DETECTED_CORES" -ge 1 ]] || DETECTED_CORES=1 + [[ "$DETECTED_RAM_GB" -ge 1 ]] || DETECTED_RAM_GB=2 # unknown RAM → assume one worker + + w_by_core=$(( DETECTED_CORES / 2 )); [[ "$w_by_core" -ge 1 ]] || w_by_core=1 + w_by_ram=$(( DETECTED_RAM_GB / 2 )); [[ "$w_by_ram" -ge 1 ]] || w_by_ram=1 + SUGGESTED_WORKERS=$(( w_by_core < w_by_ram ? w_by_core : w_by_ram )) +} + +# Per-worker concurrent syntheses that fit the box's cores without oversubscribing: +# total 2-core jobs (cores/2) spread across the chosen worker count. Args: . +_concurrency_for() { + local c=$(( DETECTED_CORES / (2 * $1) )) + [[ "$c" -ge 1 ]] && echo "$c" || echo 1 +} + # ── Migrations (one-time, idempotent — safe to call every run) ───────────────── _migrate_pocket_env() { [[ -f "$ENV_FILE" ]] || return 0 @@ -382,15 +412,22 @@ do_first_setup() { hdr "Step 2/5 — Voice coverage" _set_coverage - hdr "Step 3/5 — Workers" - WORKERS=$(gum input --placeholder "1" \ - --header "Uvicorn worker processes (throughput; RAM scales with count — see docs)") || true - WORKERS="${WORKERS:-1}" + hdr "Step 3/5 — Workers & concurrency" + _detect_hw + ok "Detected: ${DETECTED_CORES} cores, ~${DETECTED_RAM_GB} GB RAM" + WORKERS=$(gum input --placeholder "$SUGGESTED_WORKERS" \ + --header "Uvicorn workers (each ≈2 cores + ~2 GB RAM; suggested $SUGGESTED_WORKERS)") || true + WORKERS="${WORKERS:-$SUGGESTED_WORKERS}" case "$WORKERS" in - ''|*[!0-9]*|0) warn "Invalid — using 1"; WORKERS=1 ;; + ''|*[!0-9]*|0) warn "Invalid — using $SUGGESTED_WORKERS"; WORKERS=$SUGGESTED_WORKERS ;; esac ok "Workers: $WORKERS" + # Derive per-worker concurrency for the ACTUAL worker count chosen, so cores stay + # ≈ workers × concurrency × 2 and never oversubscribe (which slows every request). + MAX_CONCURRENT=$(_concurrency_for "$WORKERS") + ok "Max concurrent syntheses/worker: $MAX_CONCURRENT (${DETECTED_CORES} cores ÷ 2 ÷ ${WORKERS} workers)" + hdr "Step 4/5 — HuggingFace Token" HF_TOKEN=$(gum input --password --placeholder "hf_..." \ --header "Get a free read-only token at: https://huggingface.co/settings/tokens") || true @@ -448,6 +485,7 @@ MAX_TEXT_LENGTH=2000 EOF chmod 600 "$ENV_FILE" set_env WORKERS "$WORKERS" + set_env MAX_CONCURRENT_SYNTHESES "$MAX_CONCURRENT" set_env HF_TOKEN "$HF_TOKEN" set_env DOMAIN "$DOMAIN_INPUT"