Support bilibili URL ingestion with automatic titles and pin direct dependencies to a verified compatible set

Author: ArietidsZ

pyproject.toml

@@ -5,15 +5,16 @@ description = "Intelligent video summarization and RSS feed generation using Qwe
 requires-python = ">=3.11"
 license = {text = "Apache-2.0"}
 dependencies = [
-    "qwen-asr[vllm]",
-    "vllm>=0.8.5",
-    "torch",
-    "fastapi>=0.115",
-    "uvicorn[standard]>=0.34",
-    "asyncpg>=0.30",
-    "httpx>=0.28",
-    "feedparser>=6.0",
-    "click>=8.1",
+    "qwen-asr[vllm]==0.0.6",
+    "vllm==0.14.0",
+    "torch==2.9.1",
+    "fastapi==0.134.0",
+    "uvicorn[standard]==0.41.0",
+    "asyncpg==0.31.0",
+    "httpx==0.28.1",
+    "feedparser==6.0.12",
+    "click==8.3.1",
+    "yt-dlp==2026.2.21",
 ]
 
 [project.scripts]

vra/media.py

@@ -1,12 +1,35 @@
 from __future__ import annotations
 
 import asyncio
+from dataclasses import dataclass
+import sys
 import uuid
 from pathlib import Path
+from urllib.parse import unquote, urlparse
 
 import httpx
 
 
+_DIRECT_MEDIA_EXTENSIONS = {
+    ".aac",
+    ".flac",
+    ".m4a",
+    ".mka",
+    ".mp3",
+    ".mp4",
+    ".ogg",
+    ".opus",
+    ".wav",
+    ".webm",
+}
+
+
+@dataclass(slots=True)
+class PreparedAudio:
+    audio_path: Path
+    title: str | None = None
+
+
 async def ensure_storage_dirs(storage_dir: str) -> None:
     base = Path(storage_dir)
     (base / "raw").mkdir(parents=True, exist_ok=True)
@@ -24,12 +47,17 @@ async def download_to_file(client: httpx.AsyncClient, url: str, dest: Path) -> N
 
 async def extract_audio_ffmpeg(input_path: Path, output_path: Path) -> None:
     proc = await asyncio.create_subprocess_exec(
-        "ffmpeg", "-y",
-        "-i", str(input_path),
+        "ffmpeg",
+        "-y",
+        "-i",
+        str(input_path),
         "-vn",
-        "-acodec", "pcm_s16le",
-        "-ar", "16000",
-        "-ac", "1",
+        "-acodec",
+        "pcm_s16le",
+        "-ar",
+        "16000",
+        "-ac",
+        "1",
         str(output_path),
         stdout=asyncio.subprocess.DEVNULL,
         stderr=asyncio.subprocess.PIPE,
@@ -43,24 +71,125 @@ def _is_url(value: str) -> bool:
     return value.startswith("http://") or value.startswith("https://")
 
 
+def _url_suffix(url: str) -> str:
+    return Path(urlparse(url).path).suffix.lower()
+
+
+def _looks_like_direct_media_url(url: str) -> bool:
+    return _url_suffix(url) in _DIRECT_MEDIA_EXTENSIONS
+
+
+def _title_from_url(url: str) -> str | None:
+    stem = Path(unquote(urlparse(url).path)).stem.strip()
+    return stem or None
+
+
+def _title_from_path(path: Path) -> str | None:
+    stem = path.stem.strip()
+    return stem or None
+
+
+async def download_with_ytdlp(
+    url: str,
+    raw_dir: Path,
+    file_id: uuid.UUID,
+) -> tuple[Path, str | None]:
+    raw_dir.mkdir(parents=True, exist_ok=True)
+    output_template = str(raw_dir / f"{file_id}.%(ext)s")
+
+    proc = await asyncio.create_subprocess_exec(
+        sys.executable,
+        "-m",
+        "yt_dlp",
+        "--no-playlist",
+        "--no-warnings",
+        "-f",
+        "bestaudio/best",
+        "--output",
+        output_template,
+        "--print",
+        "title:%(title)s",
+        "--print",
+        "after_move:filepath:%(filepath)s",
+        url,
+        stdout=asyncio.subprocess.PIPE,
+        stderr=asyncio.subprocess.PIPE,
+    )
+    stdout, stderr = await proc.communicate()
+    if proc.returncode != 0:
+        err = stderr.decode(errors="replace").strip()
+        raise RuntimeError(f"yt-dlp failed: {err}")
+
+    out_text = stdout.decode(errors="replace")
+    downloaded_path: Path | None = None
+    title: str | None = None
+    for raw_line in out_text.splitlines():
+        line = raw_line.strip()
+        if not line:
+            continue
+        if line.startswith("title:"):
+            value = line.removeprefix("title:").strip()
+            if value:
+                title = value
+            continue
+        if line.startswith("filepath:"):
+            value = line.removeprefix("filepath:").strip()
+            p = Path(value)
+            if p.exists():
+                downloaded_path = p
+            continue
+
+        p = Path(line)
+        if p.exists():
+            downloaded_path = p
+
+    if downloaded_path:
+        return downloaded_path, title
+
+    matches = sorted(
+        raw_dir.glob(f"{file_id}.*"), key=lambda p: p.stat().st_mtime, reverse=True
+    )
+    if matches:
+        return matches[0], title
+
+    raise RuntimeError("yt-dlp completed but no media file was found")
+
+
 async def prepare_audio(
     client: httpx.AsyncClient,
     source: str,
     storage_dir: str,
-) -> Path:
+) -> PreparedAudio:
     await ensure_storage_dirs(storage_dir)
     base = Path(storage_dir)
     file_id = uuid.uuid4()
+    inferred_title: str | None = None
 
     if _is_url(source):
-        raw_path = base / "raw" / str(file_id)
-        await download_to_file(client, source, raw_path)
-        input_path = raw_path
+        input_path: Path | None = None
+
+        if _looks_like_direct_media_url(source):
+            raw_path = base / "raw" / f"{file_id}{_url_suffix(source)}"
+            try:
+                await download_to_file(client, source, raw_path)
+                input_path = raw_path
+                inferred_title = _title_from_url(source)
+            except Exception:
+                input_path = None
+
+        if input_path is None:
+            input_path, inferred_title = await download_with_ytdlp(
+                source, base / "raw", file_id
+            )
+
+        if not inferred_title:
+            inferred_title = _title_from_url(source)
     else:
         input_path = Path(source)
         if not input_path.exists():
             raise FileNotFoundError(f"Source file not found: {source}")
+        inferred_title = _title_from_path(input_path)
 
     output_path = base / "audio" / f"{file_id}.wav"
     await extract_audio_ffmpeg(input_path, output_path)
-    return output_path
+    return PreparedAudio(audio_path=output_path, title=inferred_title)

vra/pipeline.py

@@ -91,7 +91,9 @@ async def ingest_feed(
             if not source_url:
                 continue
 
-            video_id = await self._db.upsert_video(feed_id, guid, title, source_url, pub_dt)
+            video_id = await self._db.upsert_video(
+                feed_id, guid, title, source_url, pub_dt
+            )
             report.item_count += 1
 
             if process:
@@ -101,11 +103,15 @@ async def ingest_feed(
 
         return report
 
-    async def process_source(self, source_url: str, title: str | None = None) -> ProcessReport:
+    async def process_source(
+        self, source_url: str, title: str | None = None
+    ) -> ProcessReport:
         video_id = await self._db.upsert_video(None, None, title, source_url, None)
         return await self._process_with_video(video_id, source_url, title)
 
-    async def rss_feed(self, title: str, link: str, description: str, limit: int = 20) -> str:
+    async def rss_feed(
+        self, title: str, link: str, description: str, limit: int = 20
+    ) -> str:
         records = await self._db.latest_summaries(limit)
         return render_feed(title, link, description, records)
 
@@ -119,22 +125,32 @@ async def _process_with_video(
         source_url: str,
         title: str | None,
     ) -> ProcessReport:
-        audio_path = await prepare_audio(self._client, source_url, self._config.storage_dir)
-        audio_str = str(audio_path)
+        prepared = await prepare_audio(
+            self._client, source_url, self._config.storage_dir
+        )
+        resolved_title = title or prepared.title
+
+        if resolved_title and resolved_title != title:
+            await self._db.upsert_video(None, None, resolved_title, source_url, None)
+
+        audio_str = str(prepared.audio_path)
 
         tr = await asyncio.to_thread(self._transcriber.transcribe, audio_str)
         sr = await asyncio.to_thread(self._summarizer.summarize, tr.text)
 
         await self._db.insert_transcript(video_id, tr)
         await self._db.insert_summary(video_id, sr)
 
-        return ProcessReport(source_url=source_url, title=title, transcription=tr, summary=sr)
+        return ProcessReport(
+            source_url=source_url, title=resolved_title, transcription=tr, summary=sr
+        )
 
 
 # ------------------------------------------------------------------
 # Helpers
 # ------------------------------------------------------------------
 
+
 def _pick_source_url(entry) -> str | None:
     enclosures = entry.get("enclosures", [])
     if enclosures:
@@ -147,6 +163,7 @@ def _pick_source_url(entry) -> str | None:
 
 def _struct_to_dt(st):
     from datetime import datetime, timezone
+
     try:
         return datetime(*st[:6], tzinfo=timezone.utc)
     except Exception: