diff --git a/.env.example b/.env.example index 9897c2b5..e0aeabcb 100644 --- a/.env.example +++ b/.env.example @@ -78,7 +78,9 @@ CREDENTIAL_ENCRYPTION_KEY= # --- LLM Tracing (Langfuse, opt-in) -------------------------- # Erfordert: docker compose --profile langfuse up -d -# und: uv add langfuse (in pyproject.toml optional-dependencies) +# Server >= 4.59.0 fuer Python-SDK v5 erforderlich +# Lokal: uv sync --extra tracing (auch bei uv run --extra tracing verwenden) +# Docker: Core mit --build-arg INSTALL_TRACING=true bauen # 1. Langfuse UI aufrufen (http://localhost:3000), Account erstellen # 2. Project erstellen, API-Keys kopieren #LANGFUSE_HOST=http://langfuse:3000 diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 964c2c39..e8ae58e8 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -135,6 +135,12 @@ jobs: LOG_LEVEL: DEBUG run: uv run --extra dev pytest -m "not llm_eval" --cov --cov-report=term-missing --cov-report=xml + - name: Install optional tracing dependencies + run: uv sync --frozen --extra dev --extra tracing + + - name: Test Langfuse integration + run: uv run --frozen --extra dev --extra tracing pytest tests/test_tracing.py -v + - name: Upload coverage artifact if: always() uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7 diff --git a/.secrets.baseline b/.secrets.baseline index a8966aed..e44780d3 100644 --- a/.secrets.baseline +++ b/.secrets.baseline @@ -139,7 +139,7 @@ "filename": "docker/Dockerfile.niles", "hashed_secret": "acd583725c1b1afca04c22eb7a8ed3434034eb91", "is_verified": false, - "line_number": 25, + "line_number": 35, "is_secret": false }, { @@ -147,7 +147,7 @@ "filename": "docker/Dockerfile.niles", "hashed_secret": "1a0908b3ef3e59f870c94dc4c4e4a4a4708aa378", "is_verified": false, - "line_number": 43, + "line_number": 53, "is_secret": false }, { @@ -155,7 +155,7 @@ "filename": "docker/Dockerfile.niles", "hashed_secret": "0c961dd775de1f886e35c547aa874dcc6bb4417e", "is_verified": false, - "line_number": 46, + "line_number": 56, "is_secret": false } ], @@ -620,5 +620,5 @@ } ] }, - "generated_at": "2026-06-16T07:43:34Z" + "generated_at": "2026-10-27T12:58:05Z" } diff --git a/docker/Dockerfile.niles b/docker/Dockerfile.niles index e7573a8b..a452a116 100644 --- a/docker/Dockerfile.niles +++ b/docker/Dockerfile.niles @@ -8,16 +8,26 @@ ENV UV_COMPILE_BYTECODE=1 UV_LINK_MODE=copy UV_PROJECT_ENVIRONMENT=/app/.venv WORKDIR /build +ARG INSTALL_TRACING=false + # Install dependencies first (cached layer, only invalidated when pyproject.toml/uv.lock change) COPY pyproject.toml uv.lock ./ RUN --mount=type=cache,target=/root/.cache/uv \ - uv sync --frozen --no-dev --no-install-project + if [ "$INSTALL_TRACING" = "true" ]; then \ + uv sync --frozen --no-dev --no-install-project --extra tracing; \ + else \ + uv sync --frozen --no-dev --no-install-project; \ + fi # Copy source code, install project, and build Tailwind CSS COPY tailwind.config.js . COPY src/ ./src/ RUN --mount=type=cache,target=/root/.cache/uv \ - uv sync --frozen --no-dev --no-editable + if [ "$INSTALL_TRACING" = "true" ]; then \ + uv sync --frozen --no-dev --no-editable --extra tracing; \ + else \ + uv sync --frozen --no-dev --no-editable; \ + fi RUN python -c "\ import urllib.request, hashlib; \ urllib.request.urlretrieve('https://github.com/tailwindlabs/tailwindcss/releases/download/v3.4.17/tailwindcss-linux-x64', '/usr/local/bin/tailwindcss'); \ diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml index 1315d7bc..aa270aa4 100644 --- a/docker/docker-compose.yml +++ b/docker/docker-compose.yml @@ -213,7 +213,7 @@ services: # Langfuse (LLM Tracing, opt-in) — activate with: docker compose --profile langfuse up -d # Access: http://localhost:3000 | Credentials: set via LANGFUSE_* env vars langfuse: - image: langfuse/langfuse:4@sha256:7dcc1419da9062c968a428bc63a4ff0e03d4cc84113125f999754a9037912425 + image: langfuse/langfuse:4.59.0@sha256:48ecae32cd74d3a567588602c144de0ec626daf32dc5f2e831a03cebd0bb2d34 container_name: niles_langfuse restart: unless-stopped security_opt: diff --git a/docs/Deployment.md b/docs/Deployment.md index 2f57d02e..9bd152a7 100644 --- a/docs/Deployment.md +++ b/docs/Deployment.md @@ -892,6 +892,13 @@ docker compose -f docker/docker-compose.yml logs -f niles_core Langfuse ist ein optionaler Self-hosted LLM-Tracing-Service, der jeden LLM-Aufruf (Prompt, Response, Latenz, Token-Verbrauch, Tool-Calls) aufzeichnet. +Das Python-SDK v5 benoetigt **Langfuse Server >= 4.59.0**. Das Compose-Image +ist auf diese Version mit Digest gepinnt. Vor einem Upgrade eines bestehenden +Servers Datenbanken und weitere persistente Langfuse-Daten sichern und den +[offiziellen Upgrade-Leitfaden](https://langfuse.com/self-hosting/upgrade) +beachten. Den Server zuerst aktualisieren; fuer aeltere Server SDK v4 verwenden. +Der Image-Pin aktualisiert keinen bereits laufenden Container automatisch. + ```bash # Vorbereitung: Secrets generieren openssl rand -hex 32 # → LANGFUSE_NEXTAUTH_SECRET @@ -923,11 +930,47 @@ LANGFUSE_PUBLIC_KEY=pk-lf-... LANGFUSE_SECRET_KEY=sk-lf-... ``` -Langfuse-Paket installieren: +Langfuse-Paket fuer lokalen Betrieb installieren (bei `uv run` ebenfalls +`--extra tracing` angeben, damit das optionale Paket installiert bleibt): +```bash +uv sync --frozen --extra tracing +uv run --extra tracing uvicorn niles.main:app --host 127.0.0.1 --port 8000 +``` + +Das Standard-Docker-Image enthaelt das optionale SDK nicht. Fuer Tracing das +Niles-Image mit dem Build-Argument bauen und den Core neu erstellen: + ```bash -uv add langfuse +docker compose --env-file .env -f docker/docker-compose.yml build --build-arg INSTALL_TRACING=true niles_core +docker compose --env-file .env -f docker/docker-compose.yml up -d --no-deps niles_core ``` +### Upgrade pruefen + +SDK-Regressionstests mit synthetischen Daten, ohne laufendes LLM oder Langfuse: + +```bash +uv sync --frozen --extra dev --extra tracing +uv run --frozen --extra dev --extra tracing pytest tests/test_tracing.py -v +``` + +Diese Tests pruefen den echten SDK-Wrapper fuer normale und gestreamte Antworten +einschliesslich Modell, Ein-/Ausgabe und Token-Verbrauch mit einem In-Memory- +Exporter. Sie ersetzen keinen Test gegen den eigenen Langfuse-Server: + +1. Server-Version im Langfuse-UI pruefen: mindestens 4.59.0. +2. Niles mit den eigenen Langfuse-Zugangsdaten starten. +3. Je eine synthetische Chat-Anfrage ohne Streaming und ueber den Web-Chat mit + Streaming ausfuehren, z. B. "Antworte nur mit Hallo.". +4. Im eigenen Langfuse-Projekt auf beide Generations warten und Modell, + Ein-/Ausgabe und Token-Verbrauch pruefen, soweit das LLM diese liefert. +5. Bei fehlenden Traces die Niles-Logs auf Exportfehler pruefen. Eine erfolgreiche + Chat-Antwort allein bestaetigt keinen erfolgreichen Trace-Export. + +Ohne vollstaendige `LANGFUSE_HOST`, `LANGFUSE_PUBLIC_KEY` und +`LANGFUSE_SECRET_KEY` bleibt der normale lokale Client aktiv. Traces enthalten +Chat-Inhalte; nur einen bewusst konfigurierten eigenen Server verwenden. + ### Token Budget Alert (Prometheus) Niles exportiert Token-Metriken unter `/metrics` (Prometheus-Format): diff --git a/docs/Niles-Core-Spec.md b/docs/Niles-Core-Spec.md index e58ae5f6..9d792f36 100644 --- a/docs/Niles-Core-Spec.md +++ b/docs/Niles-Core-Spec.md @@ -30,7 +30,7 @@ Niles is a local, private AI butler on a Mac Mini M4. It receives events from va | Vikunja 1.1.0 | 3456 | `https://vikunja.example.local` | Todo/task management | | signal-cli-rest-api | 8080 | Not exposed | Signal gateway (optional) | | SearXNG | 8080 | Not exposed | Meta search engine (optional, profile `search`) | -| Langfuse 3 | 3000 | `127.0.0.1:3000` | LLM tracing (optional, profile `langfuse`) | +| Langfuse >= 4.59.0 | 3000 | `127.0.0.1:3000` | LLM tracing (optional, profile `langfuse`, Python SDK v5) | **Network architecture:** All Docker services communicate internally via HTTP on `niles_network`. External access through homelab-gateway (Caddy reverse proxy on `proxy` network, HTTPS with self-signed certs, subdomain-based routing). PostgreSQL and internal service ports are not exposed. signal-cli-rest-api runs as a Docker service alongside other containers (activated via `feature_signal` in Settings UI). @@ -1476,7 +1476,7 @@ Multi-stage build (Builder → gws-downloader → Runtime): | `vikunja` | `vikunja/vikunja:1.1.0` | niles_network, proxy | Todo/task management | | `niles_signal_api` | `bbernhard/signal-cli-rest-api:1771797934-ci` | niles_network | Signal gateway (signal-cli v0.13.24) | | `niles_searxng` | `searxng/searxng:2025.5.10-1b787ed35` | niles_network | Meta search engine (profile `search`) | -| `niles_langfuse` | `langfuse/langfuse:3` | niles_network | LLM tracing (opt-in, profile `langfuse`) | +| `niles_langfuse` | `langfuse/langfuse:4.59.0` (digest-pinned) | niles_network | LLM tracing (opt-in, profile `langfuse`, Python SDK v5) | HTTPS termination is handled by `gateway-caddy` in the separate [homelab-gateway](../../homelab-gateway) repo. diff --git a/pyproject.toml b/pyproject.toml index ed23c3b6..9a1fcfdd 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -83,7 +83,7 @@ dependencies = [ [project.optional-dependencies] tracing = [ - "langfuse>=4.7.1,<5", + "langfuse>=5.0.4,<6", ] dev = [ "pytest>=9.0.0", diff --git a/src/niles/agent/core.py b/src/niles/agent/core.py index c88effa7..c29c49e2 100644 --- a/src/niles/agent/core.py +++ b/src/niles/agent/core.py @@ -96,7 +96,7 @@ def __init__( self._OpenAI = _TracedOpenAI logger.info("LLM tracing enabled via Langfuse (%s)", config.langfuse_host) except ImportError: - logger.warning("langfuse package not installed; LLM tracing disabled. Install: uv add langfuse") + logger.warning("langfuse package not installed; LLM tracing disabled. Install: uv sync --extra tracing") self.llm_timeout = config.llm_timeout self.llm = self._OpenAI(base_url=config.llm_base_url, api_key="not-needed", timeout=self.llm_timeout) self.model = config.llm_model diff --git a/tests/test_tracing.py b/tests/test_tracing.py new file mode 100644 index 00000000..2c2bd1ef --- /dev/null +++ b/tests/test_tracing.py @@ -0,0 +1,171 @@ +"""Langfuse SDK compatibility tests with synthetic LLM responses.""" + +import json +from unittest.mock import AsyncMock, patch + +import httpx +import pytest +from openai import AsyncOpenAI + +from niles.agent.core import NilesAgent +from tests.helpers import make_test_settings + + +def _make_agent(**overrides): + settings = make_test_settings( + llm_base_url="http://llm.test/v1", + llm_model="test-model", + langfuse_host=overrides.get("langfuse_host", ""), + langfuse_public_key=overrides.get("langfuse_public_key", ""), + langfuse_secret_key=overrides.get("langfuse_secret_key", ""), + ) + history = AsyncMock() + history.get_recent.return_value = [] + memory = AsyncMock() + memory.list_all.return_value = [] + with patch("niles.agent.core.load_system_prompt", return_value="Synthetic test assistant."): + return NilesAgent( + config=settings, + contacts=AsyncMock(), + whatsapp=AsyncMock(), + memory=memory, + history=history, + ) + + +@pytest.mark.parametrize( + "config", + [ + {}, + {"langfuse_host": "http://langfuse.test"}, + {"langfuse_host": "http://langfuse.test", "langfuse_public_key": "pk-lf-test"}, + ], +) +async def test_tracing_requires_complete_configuration(config): + with patch("builtins.__import__", wraps=__import__) as imports: + agent = _make_agent(**config) + try: + assert agent._OpenAI is AsyncOpenAI + assert not any(call.args[0].startswith("langfuse") for call in imports.call_args_list) + finally: + await agent.llm.close() + + +async def test_missing_tracing_package_warns_and_keeps_local_client(caplog, monkeypatch): + original_import = __import__ + monkeypatch.setenv("LANGFUSE_HOST", "http://langfuse.test") + monkeypatch.setenv("LANGFUSE_PUBLIC_KEY", "pk-lf-test") + monkeypatch.setenv("LANGFUSE_SECRET_KEY", "sk-lf-test") + + def without_langfuse(name, *args, **kwargs): + if name.startswith("langfuse"): + raise ModuleNotFoundError("Synthetic missing optional package") + return original_import(name, *args, **kwargs) + + with patch("builtins.__import__", side_effect=without_langfuse): + agent = _make_agent( + langfuse_host="http://langfuse.test", + langfuse_public_key="pk-lf-test", + langfuse_secret_key="sk-lf-test", # pragma: allowlist secret + ) + try: + assert agent._OpenAI is AsyncOpenAI + assert "LLM tracing disabled" in caplog.text + assert "uv sync --extra tracing" in caplog.text + finally: + await agent.llm.close() + + +@pytest.fixture(scope="module") +def tracing_client(): + langfuse = pytest.importorskip("langfuse") + from opentelemetry.sdk.trace import TracerProvider + from opentelemetry.sdk.trace.export.in_memory_span_exporter import InMemorySpanExporter + + exporter = InMemorySpanExporter() + client = langfuse.Langfuse( + public_key="pk-lf-test", + secret_key="sk-lf-test", # pragma: allowlist secret + base_url="http://langfuse.test", + tracer_provider=TracerProvider(), + span_exporter=exporter, + ) + yield client, exporter + client.shutdown() + + +@pytest.mark.parametrize("streaming", [False, True]) +async def test_real_sdk_records_agent_generation(tracing_client, monkeypatch, streaming): + client, exporter = tracing_client + client.flush() + exporter.clear() + requests = [] + + async def handle_request(transport, request): + assert request.url == "http://llm.test/v1/chat/completions" + body = json.loads(request.content) + requests.append(body) + completion = { + "id": "chatcmpl-test", + "object": "chat.completion", + "created": 1, + "model": "test-model", + "choices": [{"index": 0, "message": {"role": "assistant", "content": "Hello."}, "finish_reason": "stop"}], + "usage": {"prompt_tokens": 12, "completion_tokens": 3, "total_tokens": 15}, + } + if not body.get("stream", False): + return httpx.Response(200, json=completion) + chunks = [ + { + **{key: completion[key] for key in ("id", "created", "model")}, + "object": "chat.completion.chunk", + "choices": [{"index": 0, "delta": {"role": "assistant", "content": "Hello."}, "finish_reason": None}], + }, + { + **{key: completion[key] for key in ("id", "created", "model")}, + "object": "chat.completion.chunk", + "choices": [{"index": 0, "delta": {}, "finish_reason": "stop"}], + }, + { + **{key: completion[key] for key in ("id", "created", "model")}, + "object": "chat.completion.chunk", + "choices": [], + "usage": completion["usage"], + }, + ] + data = "".join(f"data: {json.dumps(chunk)}\n\n" for chunk in chunks) + "data: [DONE]\n\n" + return httpx.Response(200, headers={"content-type": "text/event-stream"}, content=data) + + monkeypatch.setattr(httpx.AsyncHTTPTransport, "handle_async_request", handle_request) + monkeypatch.setenv("LANGFUSE_HOST", "http://langfuse.test") + monkeypatch.setenv("LANGFUSE_PUBLIC_KEY", "pk-lf-test") + monkeypatch.setenv("LANGFUSE_SECRET_KEY", "sk-lf-test") + agent = _make_agent( + langfuse_host="http://langfuse.test", + langfuse_public_key="pk-lf-test", + langfuse_secret_key="sk-lf-test", # pragma: allowlist secret + ) + event = {"type": "web", "from": "web-user-1", "content": "Say hello."} + try: + if streaming: + events = [item async for item in agent.process_event_stream(event)] + assert events[-1] == {"type": "done"} + assert "".join(item["text"] for item in events if item["type"] == "chunk") == "Hello." + else: + assert await agent.process_event(event) == "Hello." + agent.history.add_message.assert_any_await("web-user-1", "assistant", "Hello.") + assert len(requests) == 1 + assert requests[0].get("stream", False) is streaming + client.flush() + spans = exporter.get_finished_spans() + assert len(spans) == 1 + attributes = spans[0].attributes + assert attributes["langfuse.observation.type"] == "generation" + assert attributes["langfuse.observation.model.name"] == "test-model" + assert "Say hello." in attributes["langfuse.observation.input"] + assert "Hello." in attributes["langfuse.observation.output"] + usage = json.loads(attributes["langfuse.observation.usage_details"]) + assert usage["prompt_tokens"] == 12 + assert usage["completion_tokens"] == 3 + finally: + await agent.llm.close() diff --git a/uv.lock b/uv.lock index 54ff8214..fec9801c 100644 --- a/uv.lock +++ b/uv.lock @@ -786,7 +786,7 @@ wheels = [ [[package]] name = "langfuse" -version = "4.7.1" +version = "5.0.4" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "backoff" }, @@ -796,11 +796,12 @@ dependencies = [ { name = "opentelemetry-sdk" }, { name = "packaging" }, { name = "pydantic" }, + { name = "typing-extensions" }, { name = "wrapt" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/2c/74/a6f1a99893ee6d1a69439ae7eb92f8fe8806103492dc26531d5942dbd3bf/langfuse-4.7.1.tar.gz", hash = "sha256:f9e262eceedb353b191c1da1f8452d1e8ebf52297ca20e160cda0206608e3a40", size = 320620, upload-time = "2026-05-29T18:06:22.435Z" } +sdist = { url = "https://files.pythonhosted.org/packages/75/b3/6694c5edf351a01f85aae48a867fd34979865062cda808630aae86a99768/langfuse-5.0.4.tar.gz", hash = "sha256:f9f13a7a0bf0aad4d75287411cece3172c717d9440e73b4b84255ffffb549c31", size = 367328, upload-time = "2026-10-21T07:36:07.896Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/9f/9a/bd3368f46b6c72ee2068b80536826b02ae86df53eff1c79941344503098f/langfuse-4.7.1-py3-none-any.whl", hash = "sha256:a4e59c81ad5e5b16a65d3849f4923ebc3ad6e67ec803ada83d50c0cb66149490", size = 562571, upload-time = "2026-05-29T18:06:20.517Z" }, + { url = "https://files.pythonhosted.org/packages/42/f4/de90215c074ddde1849a60a939f75f9ceaf656c019501b5cac7cdb63b2fc/langfuse-5.0.4-py3-none-any.whl", hash = "sha256:6ab4ddcbc4b33efbd9c7f5feb405bfb0060949e7a170f7410f8fcc52984887a7", size = 637544, upload-time = "2026-10-21T07:36:06.457Z" }, ] [[package]] @@ -1150,7 +1151,7 @@ requires-dist = [ { name = "itsdangerous", specifier = ">=2.0" }, { name = "jinja2", specifier = ">=3.1.0" }, { name = "json-repair", specifier = ">=0.58.0" }, - { name = "langfuse", marker = "extra == 'tracing'", specifier = ">=4.7.1,<5" }, + { name = "langfuse", marker = "extra == 'tracing'", specifier = ">=5.0.4,<6" }, { name = "lxml-html-clean", specifier = ">=0.4.5" }, { name = "mcp", specifier = ">=2.3.0,<3" }, { name = "mypy", marker = "extra == 'dev'", specifier = ">=1.14.0" },