diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index 8eb342e..c6ba2a3 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -45,17 +45,19 @@ jobs: - name: Deploy to GCP VM uses: appleboy/ssh-action@v1 env: - # base64-encoded contents of .env and pocket-tts.env, stored as GitHub - # secrets (see docs/deployment.md). base64 avoids newline/quoting issues - # over SSH. This is the single source of truth for prod config — never - # edit the files on the VM by hand. + # base64-encoded contents of .env / pocket-tts.env / elevenlabs.env, stored as + # GitHub secrets (see docs/deployment.md). base64 avoids newline/quoting issues + # over SSH. This is the single source of truth for prod config — never edit the + # files on the VM by hand. ELEVENLABS_ENV_B64 is optional (only when ElevenLabs + # is enabled); an empty secret leaves elevenlabs.env absent, which is fine. DOTENV_B64: ${{ secrets.DOTENV_B64 }} POCKET_ENV_B64: ${{ secrets.POCKET_TTS_ENV_B64 }} + ELEVENLABS_ENV_B64: ${{ secrets.ELEVENLABS_ENV_B64 }} with: host: ${{ secrets.GCP_VM_HOST }} username: ${{ secrets.GCP_VM_USER }} key: ${{ secrets.GCP_SSH_PRIVATE_KEY }} - envs: DOTENV_B64,POCKET_ENV_B64 + envs: DOTENV_B64,POCKET_ENV_B64,ELEVENLABS_ENV_B64 script: | set -euo pipefail cd ~/speech-server @@ -64,6 +66,11 @@ jobs: printf '%s' "$DOTENV_B64" | base64 -d > .env printf '%s' "$POCKET_ENV_B64" | base64 -d > pocket-tts.env chmod 600 .env pocket-tts.env + # elevenlabs.env only when the secret is set (provider optional). + if [ -n "$ELEVENLABS_ENV_B64" ]; then + printf '%s' "$ELEVENLABS_ENV_B64" | base64 -d > elevenlabs.env + chmod 600 elevenlabs.env + fi docker compose pull app docker compose up -d --no-build app docker compose restart nginx diff --git a/.gitignore b/.gitignore index 7bf4350..ec758ef 100644 --- a/.gitignore +++ b/.gitignore @@ -150,6 +150,7 @@ activemq-data/ # Environments .env pocket-tts.env +elevenlabs.env .envrc .venv env/ diff --git a/README.md b/README.md index 499edcb..473ce0d 100644 --- a/README.md +++ b/README.md @@ -16,10 +16,10 @@ Designed to pair with [Readium Speech](https://github.com/readium/speech), Readi | | | |---|---| | **API** | `GET /service` · `GET /voices` · `POST /synthesize` | -| **Providers** | PocketTTS · ElevenLabs (planned) | +| **Providers** | [PocketTTS](docs/providers/pocket.md) (local) · [ElevenLabs](docs/providers/elevenlabs.md) (hosted) | | **Languages** | English · French · Italian · German · Spanish · Portuguese | | **Formats** | WAV (default) · MP3 · Opus | -| **Word boundaries** | Planned (ElevenLabs) | +| **Word boundaries** | ElevenLabs (word-level timing marks) | | **Deployment** | Docker · CPU-only · Single named volume for model weights | --- @@ -55,6 +55,7 @@ curl -s -X POST http://localhost:8000/synthesize \ - [API Reference](docs/API.md) - [Voices](docs/voices.md) - [Configuration](docs/configuration.md) +- Providers: [PocketTTS](docs/providers/pocket.md) · [ElevenLabs](docs/providers/elevenlabs.md) - [Development](docs/development.md) - [Deployment](docs/deployment.md) diff --git a/app/config/settings.py b/app/config/settings.py index 5288552..4028956 100644 --- a/app/config/settings.py +++ b/app/config/settings.py @@ -5,9 +5,9 @@ class Settings(BaseSettings): model_config = SettingsConfigDict( # .env: server/auth/concurrency/circuit-breaker — universal, provider-agnostic. - # pocket-tts.env: PocketTTS-scoped install config. Optional — later providers get - # their own file the same way; a missing file is silently skipped, not an error. - env_file=(".env", "pocket-tts.env"), + # pocket-tts.env / elevenlabs.env: provider-scoped config. Optional — each + # provider gets its own file the same way; a missing file is silently skipped. + env_file=(".env", "pocket-tts.env", "elevenlabs.env"), env_file_encoding="utf-8", case_sensitive=False, extra="ignore", @@ -50,6 +50,26 @@ def language_list(self) -> list[str]: # PocketTTS. Empty = no server-side default voice; a request must name one. pocket_default_voice: str = "" + # ElevenLabs (hosted HTTP provider). Only used when "elevenlabs" is in ENABLED_PROVIDERS. + elevenlabs_api_key: str = "" + elevenlabs_model_id: str = "eleven_multilingual_v2" + # Languages are PROVIDER-SCOPED: ElevenLabs has its own set in elevenlabs.env, independent of + # pocket's LANGUAGES. Empty = ElevenLabs serves no voices (like pocket's LANGUAGES unset). + elevenlabs_languages: str = "" + + @property + def elevenlabs_language_list(self) -> list[str]: + return [v.strip().lower() for v in self.elevenlabs_languages.split(",") if v.strip()] + + elevenlabs_base_url: str = "https://api.elevenlabs.io" # overridable for tests + # Per-day usage cap so users can't spam the ElevenLabs API and burn credits. Counts CHARACTERS + # sent, resets at UTC midnight. 0 = unlimited. Operators set ELEVENLABS_DAILY_CHAR_LIMIT in + # elevenlabs.env — pick a value per the model's rate (https://elevenlabs.io/pricing/api; + # flash/turbo bill half, so allow ~2× the characters for the same cost). Backed by a small JSON + # file shared across workers (see app/core/daily_limit.py) so the cap is host-wide. + elevenlabs_daily_char_limit: int = Field(default=0, ge=0) + elevenlabs_usage_file: str = "/tmp/elevenlabs_usage.json" # noqa: S108 — ephemeral counter; set a volume path to persist + # Per-voice cross-language install config. Generic across providers. Format: # comma-separated "originalName:lang" pairs, "-" prefix to remove instead of add # (e.g. "alba:fr,alba:de,-javert:es"). Additions still bounded by `languages` (never @@ -112,6 +132,10 @@ def validate_auth_and_providers(self) -> "Settings": providers = [p.strip() for p in self.enabled_providers.split(",")] if self.default_provider not in providers: raise ValueError(f"DEFAULT_PROVIDER '{self.default_provider}' not in ENABLED_PROVIDERS") + if "elevenlabs" in providers and not self.elevenlabs_api_key: + raise ValueError( + "ELEVENLABS_API_KEY must be set when 'elevenlabs' is in ENABLED_PROVIDERS" + ) if self.app_env == "production" and not self.domain: raise ValueError("DOMAIN must be set when APP_ENV=production") return self diff --git a/app/core/daily_limit.py b/app/core/daily_limit.py new file mode 100644 index 0000000..180ee3e --- /dev/null +++ b/app/core/daily_limit.py @@ -0,0 +1,58 @@ +"""Host-wide daily usage budget, backed by a small fcntl-locked JSON file. + +Shared across all uvicorn workers on the host and persists across restarts within the +same day, so a per-day cap can't be sidestepped by parallel workers or a quick restart. +Unix only (fcntl) — the server runs on Linux. `{"date": "YYYY-MM-DD", "used": N}`. +""" + +from __future__ import annotations + +import fcntl +import json +from datetime import UTC, datetime +from pathlib import Path + + +def reserve(path: str, amount: int, limit: int) -> tuple[bool, int]: + """Atomically reserve `amount` units against today's (UTC) budget. + + Returns (allowed, used_after_reservation). `limit <= 0` = unlimited (always allowed; + usage is NOT tracked, to avoid needless file I/O). When the reservation would push + usage over `limit`, nothing is added and `allowed` is False. + """ + if limit <= 0: + return (True, 0) + today = datetime.now(UTC).date().isoformat() + p = Path(path) + p.parent.mkdir(parents=True, exist_ok=True) + with open(p, "a+", encoding="utf-8") as f: + fcntl.flock(f, fcntl.LOCK_EX) + try: + f.seek(0) + raw = f.read() + data = json.loads(raw) if raw.strip() else {} + if data.get("date") != today: + data = {"date": today, "used": 0} + used = int(data.get("used", 0)) + if used + amount > limit: + return (False, used) + used += amount + f.seek(0) + f.truncate() + f.write(json.dumps({"date": today, "used": used})) + return (True, used) + finally: + fcntl.flock(f, fcntl.LOCK_UN) + + +if __name__ == "__main__": # self-check + import tempfile + + fp = str(Path(tempfile.mkdtemp()) / "u.json") + assert reserve(fp, 100, 250) == (True, 100) + assert reserve(fp, 100, 250) == (True, 200) + assert reserve(fp, 100, 250) == (False, 200) # would exceed → rejected, not added + assert reserve(fp, 50, 250) == (True, 250) # exact fit ok + assert reserve(fp, 1, 250)[0] is False # full + assert reserve(fp, 10_000, 0) == (True, 0) # limit 0 = unlimited, untracked + print("daily_limit self-check ok") diff --git a/app/core/synthesizer.py b/app/core/synthesizer.py index 178e218..ae19183 100644 --- a/app/core/synthesizer.py +++ b/app/core/synthesizer.py @@ -83,6 +83,8 @@ async def synthesize( voice_uri=voice.identifier, speed=out.speed, pitch=out.pitch, + audio_format=out.format, + bitrate=out.bitrate, prev_utterance=request.prev_utterance, next_utterance=request.next_utterance, ) @@ -94,8 +96,11 @@ async def synthesize( t0 = time.monotonic() try: result: AudioResult = await provider.synthesize(params) - except AppError: - if breaker is not None: + except AppError as exc: + # Only provider OUTAGES (5xx) trip the breaker. Client/quota errors — 429 rate + # limit (incl. the daily budget), 404 unsupported voice — are not provider failures + # and must not open the breaker for everyone. + if breaker is not None and exc.status_code >= 500: breaker.record_failure() raise else: @@ -103,6 +108,20 @@ async def synthesize( breaker.record_success() gen_ms = round((time.monotonic() - t0) * 1000, 1) + # Provider already encoded server-side (e.g. ElevenLabs native output) — + # return as-is, no local WAV/ffmpeg step. + if result.encoded is not None: + content_type = result.content_type or ffmpeg_driver.content_type(out.format.value) + logger.info( + "synth done voice=%s fmt=%s (native) gen_ms=%.1f total_ms=%.1f bytes=%d", + voice.identifier, + out.format.value, + gen_ms, + round((time.monotonic() - t0) * 1000, 1), + len(result.encoded), + ) + return (result.encoded, content_type, result.boundaries, boundaries_supported) + if out.format == AudioFormat.WAV: audio = _pcm_to_wav(result.pcm, result.sample_rate) logger.info( diff --git a/app/data/voices/elevenlabs/voices.json b/app/data/voices/elevenlabs/voices.json new file mode 100644 index 0000000..cb69a2d --- /dev/null +++ b/app/data/voices/elevenlabs/voices.json @@ -0,0 +1,47 @@ +[ + { + "name": "George", + "originalName": "george", + "identifier": "urn:readium:tts:elevenlabs:george", + "language": "en-GB", + "otherLanguages": ["ar", "bg", "cs", "da", "de", "el", "es", "fi", "fil", "fr", "hi", "hr", "id", "it", "ja", "ko", "ms", "nl", "pl", "pt", "ro", "ru", "sk", "sv", "ta", "tr", "uk", "zh"], + "gender": "male", + "quality": "high" + }, + { + "name": "Alice", + "originalName": "alice", + "identifier": "urn:readium:tts:elevenlabs:alice", + "language": "en-GB", + "otherLanguages": ["ar", "bg", "cs", "da", "de", "el", "es", "fi", "fil", "fr", "hi", "hr", "id", "it", "ja", "ko", "ms", "nl", "pl", "pt", "ro", "ru", "sk", "sv", "ta", "tr", "uk", "zh"], + "gender": "female", + "quality": "high" + }, + { + "name": "Brian", + "originalName": "brian", + "identifier": "urn:readium:tts:elevenlabs:brian", + "language": "en-US", + "otherLanguages": ["ar", "bg", "cs", "da", "de", "el", "es", "fi", "fil", "fr", "hi", "hr", "id", "it", "ja", "ko", "ms", "nl", "pl", "pt", "ro", "ru", "sk", "sv", "ta", "tr", "uk", "zh"], + "gender": "male", + "quality": "high" + }, + { + "name": "Sarah", + "originalName": "sarah", + "identifier": "urn:readium:tts:elevenlabs:sarah", + "language": "en-US", + "otherLanguages": ["ar", "bg", "cs", "da", "de", "el", "es", "fi", "fil", "fr", "hi", "hr", "id", "it", "ja", "ko", "ms", "nl", "pl", "pt", "ro", "ru", "sk", "sv", "ta", "tr", "uk", "zh"], + "gender": "female", + "quality": "high" + }, + { + "name": "River", + "originalName": "river", + "identifier": "urn:readium:tts:elevenlabs:river", + "language": "en-US", + "otherLanguages": ["ar", "bg", "cs", "da", "de", "el", "es", "fi", "fil", "fr", "hi", "hr", "id", "it", "ja", "ko", "ms", "nl", "pl", "pt", "ro", "ru", "sk", "sv", "ta", "tr", "uk", "zh"], + "gender": "female", + "quality": "high" + } +] diff --git a/app/main.py b/app/main.py index c021546..dcd76aa 100644 --- a/app/main.py +++ b/app/main.py @@ -27,6 +27,10 @@ def _build_registry() -> ProviderRegistry: enabled = {p.strip() for p in settings.enabled_providers.split(",")} if "pocket" in enabled: registry.register(PocketTTSProvider()) + if "elevenlabs" in enabled: + from app.providers.elevenlabs import ElevenLabsProvider + + registry.register(ElevenLabsProvider()) return registry @@ -107,7 +111,8 @@ def create_app() -> FastAPI: @app.get("/demo", include_in_schema=False) async def demo() -> FileResponse: - return FileResponse("app/static/demo.html") + # no-cache so the browser always revalidates — avoids serving a stale demo after edits. + return FileResponse("app/static/demo.html", headers={"Cache-Control": "no-cache"}) return app diff --git a/app/providers/elevenlabs.py b/app/providers/elevenlabs.py new file mode 100644 index 0000000..60cbe54 --- /dev/null +++ b/app/providers/elevenlabs.py @@ -0,0 +1,333 @@ +from __future__ import annotations + +import base64 +import json +import logging +from collections.abc import Sequence +from pathlib import Path +from re import sub as re_sub +from typing import ClassVar + +import httpx + +from app.api.errors import ( + ProviderError, + ProviderTimeout, + RateLimited, + VoiceLanguageUnsupported, +) +from app.config.settings import settings +from app.core.daily_limit import reserve +from app.domain.enums import AudioFormat, Quality +from app.drivers import ffmpeg as ffmpeg_driver +from app.providers.base import TTSProvider +from app.providers.voice_loading import VoiceEntry, build_voice, plan_install +from app.schemas.audio import AudioResult, SynthesisParams, TimingMark +from app.schemas.voice import Controls, Voice + +logger = logging.getLogger(__name__) + +_VOICES_PATH = Path(__file__).parent.parent / "data" / "voices" / "elevenlabs" / "voices.json" + +_TIMEOUT_SECONDS = 30.0 + +# ElevenLabs API voice_id per voices.json originalName — mapped HERE (in code), keeping voices.json +# a readable catalog (name/originalName like "george", not opaque ids). Add an entry when adding a +# voice. A voice whose originalName isn't in this map falls back to using originalName as the id. +_VOICE_IDS: dict[str, str] = { + "george": "JBFqnCBsd6RMkjVDRZzb", + "alice": "Xb7hH8MSUJpSbSDYk0k2", + "brian": "nPczCjzI2devNBz1zQrb", + "sarah": "EXAVITQu4vr4xnSDxMaL", + "river": "SAz9YHcvj6GT2YYXdXww", +} + +# ElevenLabs encodes server-side — request the requested format natively and skip our +# ffmpeg. Cost is per-character regardless of format (verified); only tier gates some +# variants (mp3_192=Creator+, pcm/wav_44100+=Pro+), so these are all free-tier-safe. +# Native mp3/opus at 44.1/48kHz also beats pcm_24000's 24kHz ceiling on non-Pro tiers. +_MP3_BITRATES = (32, 64, 96, 128) # 192 needs Creator tier — excluded +_OPUS_BITRATES = (32, 64, 96, 128, 192) + + +def _nearest(bitrate: int | None, allowed: tuple[int, ...], default: int) -> int: + """Largest allowed bitrate ≤ requested; default when unset/too low.""" + if bitrate is None: + return default + ok = [b for b in allowed if b <= bitrate] + return max(ok) if ok else min(allowed) + + +def _output_format(fmt: AudioFormat, bitrate: int | None) -> tuple[str, str, int]: + """(elevenlabs output_format, our content-type, sample_rate) for a requested format. + All variants are free-tier-safe (wav_44100/pcm_44100 would need Pro, so wav uses 24k).""" + if fmt == AudioFormat.MP3: + return f"mp3_44100_{_nearest(bitrate, _MP3_BITRATES, 128)}", "audio/mpeg", 44100 + if fmt == AudioFormat.OPUS: + return ( + f"opus_48000_{_nearest(bitrate, _OPUS_BITRATES, 128)}", + ffmpeg_driver.content_type("opus"), + 48000, + ) + return "wav_24000", "audio/wav", 24000 # AudioFormat.WAV + + +# Languages COMMON TO ALL ElevenLabs models (multilingual_v2 ∩ flash/turbo_v2.5 ∩ v3 = the 29 +# multilingual_v2 languages), so behaviour is model-independent regardless of ELEVENLABS_MODEL_ID. +# All are free-tier usable — free tier only paywalls Voice-Library VOICES, not languages. This is +# the ceiling; a voice may narrow its set via otherLanguages in voices.json, and what's actually +# SERVED is this ∩ ELEVENLABS_LANGUAGES. Deliberately not the full per-model catalog (v3=70+). +# ponytail: model-independent flat set; if we ever need a model's extra languages (v2.5 hu/no/vi, +# v3's 40+ more), reintroduce a per-model models.json keyed by model_id. +_SUPPORTED_LANGUAGES: frozenset[str] = frozenset( + { + "ar", + "bg", + "cs", + "da", + "de", + "el", + "en", + "es", + "fi", + "fil", + "fr", + "hi", + "hr", + "id", + "it", + "ja", + "ko", + "ms", + "nl", + "pl", + "pt", + "ro", + "ru", + "sk", + "sv", + "ta", + "tr", + "uk", + "zh", + } +) + + +def _strip_ssml(text: str) -> str: + return re_sub(r"<[^>]+>", "", text).strip() + + +def _alignment_to_marks( + characters: list[str], + starts: list[float], + ends: list[float], # noqa: ARG001 — kept for signature symmetry; word start is what boundary needs +) -> list[TimingMark]: + """ElevenLabs alignment is per-character; the Web Speech boundary event is + per-word. Walk the char arrays, split on whitespace, emit one mark per word. + charIndex/charLength index into "".join(characters) == the text we SENT. + # when params.ssml was stripped, offsets are into the stripped text, + # not the raw request text — acceptable for boundary highlighting.""" + marks: list[TimingMark] = [] + i = 0 + n = len(characters) + while i < n: + if characters[i].isspace(): + i += 1 + continue + start_idx = i + word_start = starts[i] + while i < n and not characters[i].isspace(): + i += 1 + marks.append( + TimingMark( + name="word", + charIndex=start_idx, + charLength=i - start_idx, + elapsedTime=word_start, + ) + ) + return marks + + +class ElevenLabsProvider(TTSProvider): + id = "elevenlabs" + default_quality: ClassVar[Quality] = Quality.HIGH + # v2 has no pitch and only partial SSML; speed maps to voice_settings.speed, + # boundary comes from the /with-timestamps alignment. + default_controls: ClassVar[Controls] = Controls( + pitch=False, speed=True, ssml=False, boundary=True + ) + + # active_languages() intersects this with ELEVENLABS_LANGUAGES. + supported_languages: ClassVar[frozenset[str]] = _SUPPORTED_LANGUAGES + + def __init__(self) -> None: + self._voices: list[Voice] = [] + self._voice_langs: dict[str, frozenset[str]] = {} # identifier → installed lang keys + self._voice_default_lang: dict[str, str] = {} # identifier → default lang (request omits) + self._voice_id: dict[str, str] = {} # identifier → ElevenLabs voice_id (entry.originalName) + + def active_languages(self) -> frozenset[str]: + """Languages are PROVIDER-SCOPED: intersect the supported set with this provider's own + ELEVENLABS_LANGUAGES (elevenlabs.env), NOT the global LANGUAGES pocket uses.""" + configured = {lang.split("-")[0].lower() for lang in settings.elevenlabs_language_list} + supported = {lang.split("-")[0].lower() for lang in self.supported_languages} + return frozenset(supported & configured) + + async def load(self) -> None: + # No network at startup: read the curated catalog and run the shared + # voices.json → installed-Voice pipeline (same as pocket, minus model loading). + enabled = self.active_languages() # supported set ∩ ELEVENLABS_LANGUAGES + if not enabled: + logger.warning("No languages in ELEVENLABS_LANGUAGES — ElevenLabs serves no voices") + return + + raw = json.loads(_VOICES_PATH.read_text()) + entries = [VoiceEntry.model_validate(v) for v in raw] + + add_map = settings.voice_language_add_map + remove_map = settings.voice_language_remove_map + + model_langs = self.supported_languages + for entry in entries: + key = entry.originalName.lower() + primary = entry.language.split("-")[0].lower() + # Cross-language: ElevenLabs voices are multilingual, so a voice serves the whole + # supported set (bounded by ELEVENLABS_LANGUAGES). A voice may narrow this by declaring + # its own otherLanguages in voices.json (e.g. a language-locked cloned voice). We ALSO + # curate native-sounding voices per language (primary set accordingly) so each language + # has a natural voice, not only English voices cross-speaking it. + if entry.otherLanguages: + declared = {p.split("-")[0].lower() for p in entry.otherLanguages} + entry.otherLanguages = sorted((declared & model_langs) - {primary}) + else: + entry.otherLanguages = sorted(model_langs - {primary}) + plan = plan_install( + entry, + enabled, + True, + add_map.get(key, frozenset()), + remove_map.get(key, frozenset()), + ) + if plan is None: + continue + default_lang, installed = plan + other_langs = installed - frozenset({primary}) + voice = build_voice( + entry, self.id, other_langs, self.default_quality, self.default_controls + ) + self._voices.append(voice) + self._voice_default_lang[voice.identifier] = default_lang + self._voice_langs[voice.identifier] = installed + # ElevenLabs API voice_id from the internal map; fall back to originalName. + self._voice_id[voice.identifier] = _VOICE_IDS.get( + entry.originalName.lower(), entry.originalName + ) + + logger.info("ElevenLabs ready — %d voices", len(self._voices)) + + async def _all_voices(self) -> Sequence[Voice]: + return self._voices + + def _resolve_lang_key(self, voice_uri: str, requested_language: str | None) -> str | None: + """Warmed language to synthesize in. A requested language the voice isn't + served in returns None — never silently fall back to a different language.""" + installed = self._voice_langs.get(voice_uri, frozenset()) + if requested_language: + requested = requested_language.split("-")[0].lower() + return requested if requested in installed else None + return self._voice_default_lang[voice_uri] + + async def synthesize(self, params: SynthesisParams) -> AudioResult: + # Neutral "unsupported" responses — never reveal config state, only that it + # isn't served (mirrors pocket). + if params.voice_uri not in self._voice_langs: + raise VoiceLanguageUnsupported(f"Voice '{params.voice_uri}' is not supported.") + + lang_key = self._resolve_lang_key(params.voice_uri, params.language) + if lang_key is None: + raise VoiceLanguageUnsupported( + f"Voice '{params.voice_uri}' is not supported for language '{params.language}'." + ) + + text = _strip_ssml(params.text) if params.ssml else params.text + if not text: + raise ProviderError("Text is empty after stripping SSML tags") + + # Daily budget: reserve the characters we're about to send. Set the limit per your model's + # rate (see https://elevenlabs.io/pricing/api — flash/turbo bill half). Reserve BEFORE the + # call so parallel requests can't overshoot; unlimited when limit=0. + allowed, used = reserve( + settings.elevenlabs_usage_file, len(text), settings.elevenlabs_daily_char_limit + ) + if not allowed: + raise RateLimited( + f"Daily ElevenLabs limit reached " + f"({used}/{settings.elevenlabs_daily_char_limit} characters today) — resets at " + "00:00 UTC." + ) + + voice_id = self._voice_id[params.voice_uri] + url = f"{settings.elevenlabs_base_url}/v1/text-to-speech/{voice_id}/with-timestamps" + body = { + "text": text, + "model_id": settings.elevenlabs_model_id, + "language_code": lang_key, + "voice_settings": {"speed": params.speed}, + } + headers = {"xi-api-key": settings.elevenlabs_api_key} + el_format, content_type, sample_rate = _output_format(params.audio_format, params.bitrate) + params_q = {"output_format": el_format} + + # async httpx directly — this is network I/O, NOT the CPU-bound work + # run_inference()/the semaphore exist for. Per-request client; a shared pooled + # client is the upgrade path if connection setup cost ever shows up in latency. + try: + async with httpx.AsyncClient(timeout=_TIMEOUT_SECONDS) as client: + resp = await client.post(url, json=body, headers=headers, params=params_q) + except httpx.TimeoutException as exc: + raise ProviderTimeout(f"ElevenLabs request timed out: {exc}") from exc + except httpx.HTTPError as exc: + raise ProviderError(f"ElevenLabs request failed: {exc}") from exc + + if resp.status_code != 200: + self._raise_for_status(resp, params.voice_uri, lang_key) + + data = resp.json() + encoded = base64.b64decode(data["audio_base64"]) # already in el_format — no ffmpeg + alignment = data.get("alignment") or {} + marks = _alignment_to_marks( + alignment.get("characters", []), + alignment.get("character_start_times_seconds", []), + alignment.get("character_end_times_seconds", []), + ) + return AudioResult( + sample_rate=sample_rate, + boundaries=marks, + encoded=encoded, + content_type=content_type, + ) + + @staticmethod + def _raise_for_status(resp: httpx.Response, voice_uri: str, lang_key: str) -> None: + code = resp.status_code + if code == 429: + raise RateLimited("ElevenLabs rate limit exceeded") + if code == 402: + # Voice needs a paid plan — free tier can't use Voice Library voices via the API. + raise ProviderError( + f"ElevenLabs voice '{voice_uri}' requires a paid plan " + "(free tier can't use Voice Library voices via the API)" + ) + if code in (401, 403): + raise ProviderError("ElevenLabs authentication failed — check ELEVENLABS_API_KEY") + if code in (404, 422): + # Bad voice_id or unsupported language — stay neutral like pocket. + raise VoiceLanguageUnsupported( + f"Voice '{voice_uri}' is not supported for language '{lang_key}'." + ) + raise ProviderError(f"ElevenLabs returned {code}: {resp.text[:200]}") + + async def health(self) -> bool: + return bool(self._voices and settings.elevenlabs_api_key) diff --git a/app/schemas/audio.py b/app/schemas/audio.py index 2d058c6..0f63c6c 100644 --- a/app/schemas/audio.py +++ b/app/schemas/audio.py @@ -2,6 +2,8 @@ from pydantic import BaseModel +from app.domain.enums import AudioFormat + class TimingMark(BaseModel): """Mirrors the Web Speech API SpeechSynthesisEvent boundary fields. @@ -20,11 +22,21 @@ class SynthesisParams(BaseModel): voice_uri: str speed: float pitch: float | None + # Requested container/codec + bitrate. A provider that encodes server-side + # (e.g. ElevenLabs) uses these to fetch already-encoded audio; PCM providers + # (pocket/fake) ignore them and the Synthesizer encodes downstream. + audio_format: AudioFormat = AudioFormat.WAV + bitrate: int | None = None prev_utterance: str | None = None next_utterance: str | None = None class AudioResult(BaseModel): - pcm: bytes + pcm: bytes = b"" sample_rate: int boundaries: list[TimingMark] = [] # empty when provider doesn't support word timing + # Pre-encoded audio from a provider that encodes server-side. When set, the + # Synthesizer returns it as-is (with content_type) and skips WAV/ffmpeg. When + # None, `pcm` is authoritative and the Synthesizer encodes it. + encoded: bytes | None = None + content_type: str | None = None diff --git a/app/static/demo.html b/app/static/demo.html index e65dfb4..1be64f4 100644 --- a/app/static/demo.html +++ b/app/static/demo.html @@ -3,296 +3,286 @@ -Speech Server Demo +Readium Speech Server -TTS API Demo -

Readium Speech Server

-

Talks to GET /service, GET /voices and POST /synthesize on this same origin.

- -
-
-
- - -
-
- - -
-
+
+
+ + POST /synthesize · GET /voices · GET /service +
- - +

Readium speech server demo

+

A live console for the Readium Speech Server. Write a line, pick a voice, and listen, the waveform is the real audio the API returns.

- - +
+ + +
+
+ + +
+
+ + +
+
+
+
- - +
+ + + + + + +
-
Synthesizing…
- + -
- Developer tools — advanced request fields -

Fields the selected voice doesn't support are disabled.

-
-
- - -
-
- - -
+
+
+ + + 0:00
-
-
- - +
+
+ +
+ Advanced options +
+
+
+
+
-
- - +
+
-
- - -
-
- - +
+ +
-
-

Response

-
- - - 
(nothing yet)
+
+ API request +
+ curl + +
+ 

+    
response

+    
Nothing spoken yet.

+  

 

 
-

-  

-    Equivalent curl
-    
-  

-

-

+