From 52c0eafa96c28892bece805336d7e8382431537e Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Tue, 27 Oct 2026 12:48:07 +0000 Subject: [PATCH 1/2] fix(deps): update dependency langfuse to v5 --- pyproject.toml | 2 +- uv.lock | 9 +++++---- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index ed23c3b6..9a1fcfdd 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -83,7 +83,7 @@ dependencies = [ [project.optional-dependencies] tracing = [ - "langfuse>=4.7.1,<5", + "langfuse>=5.0.4,<6", ] dev = [ "pytest>=9.0.0", diff --git a/uv.lock b/uv.lock index 54ff8214..fec9801c 100644 --- a/uv.lock +++ b/uv.lock @@ -786,7 +786,7 @@ wheels = [ [[package]] name = "langfuse" -version = "4.7.1" +version = "5.0.4" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "backoff" }, @@ -796,11 +796,12 @@ dependencies = [ { name = "opentelemetry-sdk" }, { name = "packaging" }, { name = "pydantic" }, + { name = "typing-extensions" }, { name = "wrapt" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/2c/74/a6f1a99893ee6d1a69439ae7eb92f8fe8806103492dc26531d5942dbd3bf/langfuse-4.7.1.tar.gz", hash = "sha256:f9e262eceedb353b191c1da1f8452d1e8ebf52297ca20e160cda0206608e3a40", size = 320620, upload-time = "2026-05-29T18:06:22.435Z" } +sdist = { url = "https://files.pythonhosted.org/packages/75/b3/6694c5edf351a01f85aae48a867fd34979865062cda808630aae86a99768/langfuse-5.0.4.tar.gz", hash = "sha256:f9f13a7a0bf0aad4d75287411cece3172c717d9440e73b4b84255ffffb549c31", size = 367328, upload-time = "2026-10-21T07:36:07.896Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/9f/9a/bd3368f46b6c72ee2068b80536826b02ae86df53eff1c79941344503098f/langfuse-4.7.1-py3-none-any.whl", hash = "sha256:a4e59c81ad5e5b16a65d3849f4923ebc3ad6e67ec803ada83d50c0cb66149490", size = 562571, upload-time = "2026-05-29T18:06:20.517Z" }, + { url = "https://files.pythonhosted.org/packages/42/f4/de90215c074ddde1849a60a939f75f9ceaf656c019501b5cac7cdb63b2fc/langfuse-5.0.4-py3-none-any.whl", hash = "sha256:6ab4ddcbc4b33efbd9c7f5feb405bfb0060949e7a170f7410f8fcc52984887a7", size = 637544, upload-time = "2026-10-21T07:36:06.457Z" }, ] [[package]] @@ -1150,7 +1151,7 @@ requires-dist = [ { name = "itsdangerous", specifier = ">=2.0" }, { name = "jinja2", specifier = ">=3.1.0" }, { name = "json-repair", specifier = ">=0.58.0" }, - { name = "langfuse", marker = "extra == 'tracing'", specifier = ">=4.7.1,<5" }, + { name = "langfuse", marker = "extra == 'tracing'", specifier = ">=5.0.4,<6" }, { name = "lxml-html-clean", specifier = ">=0.4.5" }, { name = "mcp", specifier = ">=2.3.0,<3" }, { name = "mypy", marker = "extra == 'dev'", specifier = ">=1.14.0" }, From d67ff75bb96d55146745f42045b6a7a3b822f23e Mon Sep 17 00:00:00 2001 From: Gerald Fruhmann Date: Tue, 27 Oct 2026 14:00:00 +0100 Subject: [PATCH 2/2] fix(tracing): validate Langfuse v5 and require compatible server Pin Langfuse Server 4.59.0, exercise the real SDK for normal and streamed agent replies, and cover the optional tracing extra in CI. Add an opt-in Docker build and document upgrade and export checks. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .env.example | 4 +- .github/workflows/ci.yml | 6 ++ .secrets.baseline | 8 +- docker/Dockerfile.niles | 14 +++- docker/docker-compose.yml | 2 +- docs/Deployment.md | 47 ++++++++++- docs/Niles-Core-Spec.md | 4 +- src/niles/agent/core.py | 2 +- tests/test_tracing.py | 171 ++++++++++++++++++++++++++++++++++++++ 9 files changed, 245 insertions(+), 13 deletions(-) create mode 100644 tests/test_tracing.py diff --git a/.env.example b/.env.example index 9897c2b5..e0aeabcb 100644 --- a/.env.example +++ b/.env.example @@ -78,7 +78,9 @@ CREDENTIAL_ENCRYPTION_KEY= # --- LLM Tracing (Langfuse, opt-in) -------------------------- # Erfordert: docker compose --profile langfuse up -d -# und: uv add langfuse (in pyproject.toml optional-dependencies) +# Server >= 4.59.0 fuer Python-SDK v5 erforderlich +# Lokal: uv sync --extra tracing (auch bei uv run --extra tracing verwenden) +# Docker: Core mit --build-arg INSTALL_TRACING=true bauen # 1. Langfuse UI aufrufen (http://localhost:3000), Account erstellen # 2. Project erstellen, API-Keys kopieren #LANGFUSE_HOST=http://langfuse:3000 diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 964c2c39..e8ae58e8 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -135,6 +135,12 @@ jobs: LOG_LEVEL: DEBUG run: uv run --extra dev pytest -m "not llm_eval" --cov --cov-report=term-missing --cov-report=xml + - name: Install optional tracing dependencies + run: uv sync --frozen --extra dev --extra tracing + + - name: Test Langfuse integration + run: uv run --frozen --extra dev --extra tracing pytest tests/test_tracing.py -v + - name: Upload coverage artifact if: always() uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7 diff --git a/.secrets.baseline b/.secrets.baseline index a8966aed..e44780d3 100644 --- a/.secrets.baseline +++ b/.secrets.baseline @@ -139,7 +139,7 @@ "filename": "docker/Dockerfile.niles", "hashed_secret": "acd583725c1b1afca04c22eb7a8ed3434034eb91", "is_verified": false, - "line_number": 25, + "line_number": 35, "is_secret": false }, { @@ -147,7 +147,7 @@ "filename": "docker/Dockerfile.niles", "hashed_secret": "1a0908b3ef3e59f870c94dc4c4e4a4a4708aa378", "is_verified": false, - "line_number": 43, + "line_number": 53, "is_secret": false }, { @@ -155,7 +155,7 @@ "filename": "docker/Dockerfile.niles", "hashed_secret": "0c961dd775de1f886e35c547aa874dcc6bb4417e", "is_verified": false, - "line_number": 46, + "line_number": 56, "is_secret": false } ], @@ -620,5 +620,5 @@ } ] }, - "generated_at": "2026-06-16T07:43:34Z" + "generated_at": "2026-10-27T12:58:05Z" } diff --git a/docker/Dockerfile.niles b/docker/Dockerfile.niles index e7573a8b..a452a116 100644 --- a/docker/Dockerfile.niles +++ b/docker/Dockerfile.niles @@ -8,16 +8,26 @@ ENV UV_COMPILE_BYTECODE=1 UV_LINK_MODE=copy UV_PROJECT_ENVIRONMENT=/app/.venv WORKDIR /build +ARG INSTALL_TRACING=false + # Install dependencies first (cached layer, only invalidated when pyproject.toml/uv.lock change) COPY pyproject.toml uv.lock ./ RUN --mount=type=cache,target=/root/.cache/uv \ - uv sync --frozen --no-dev --no-install-project + if [ "$INSTALL_TRACING" = "true" ]; then \ + uv sync --frozen --no-dev --no-install-project --extra tracing; \ + else \ + uv sync --frozen --no-dev --no-install-project; \ + fi # Copy source code, install project, and build Tailwind CSS COPY tailwind.config.js . COPY src/ ./src/ RUN --mount=type=cache,target=/root/.cache/uv \ - uv sync --frozen --no-dev --no-editable + if [ "$INSTALL_TRACING" = "true" ]; then \ + uv sync --frozen --no-dev --no-editable --extra tracing; \ + else \ + uv sync --frozen --no-dev --no-editable; \ + fi RUN python -c "\ import urllib.request, hashlib; \ urllib.request.urlretrieve('https://github.com/tailwindlabs/tailwindcss/releases/download/v3.4.17/tailwindcss-linux-x64', '/usr/local/bin/tailwindcss'); \ diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml index 1315d7bc..aa270aa4 100644 --- a/docker/docker-compose.yml +++ b/docker/docker-compose.yml @@ -213,7 +213,7 @@ services: # Langfuse (LLM Tracing, opt-in) — activate with: docker compose --profile langfuse up -d # Access: http://localhost:3000 | Credentials: set via LANGFUSE_* env vars langfuse: - image: langfuse/langfuse:4@sha256:7dcc1419da9062c968a428bc63a4ff0e03d4cc84113125f999754a9037912425 + image: langfuse/langfuse:4.59.0@sha256:48ecae32cd74d3a567588602c144de0ec626daf32dc5f2e831a03cebd0bb2d34 container_name: niles_langfuse restart: unless-stopped security_opt: diff --git a/docs/Deployment.md b/docs/Deployment.md index 2f57d02e..9bd152a7 100644 --- a/docs/Deployment.md +++ b/docs/Deployment.md @@ -892,6 +892,13 @@ docker compose -f docker/docker-compose.yml logs -f niles_core Langfuse ist ein optionaler Self-hosted LLM-Tracing-Service, der jeden LLM-Aufruf (Prompt, Response, Latenz, Token-Verbrauch, Tool-Calls) aufzeichnet. +Das Python-SDK v5 benoetigt **Langfuse Server >= 4.59.0**. Das Compose-Image +ist auf diese Version mit Digest gepinnt. Vor einem Upgrade eines bestehenden +Servers Datenbanken und weitere persistente Langfuse-Daten sichern und den +[offiziellen Upgrade-Leitfaden](https://langfuse.com/self-hosting/upgrade) +beachten. Den Server zuerst aktualisieren; fuer aeltere Server SDK v4 verwenden. +Der Image-Pin aktualisiert keinen bereits laufenden Container automatisch. + ```bash # Vorbereitung: Secrets generieren openssl rand -hex 32 # → LANGFUSE_NEXTAUTH_SECRET @@ -923,11 +930,47 @@ LANGFUSE_PUBLIC_KEY=pk-lf-... LANGFUSE_SECRET_KEY=sk-lf-... ``` -Langfuse-Paket installieren: +Langfuse-Paket fuer lokalen Betrieb installieren (bei `uv run` ebenfalls +`--extra tracing` angeben, damit das optionale Paket installiert bleibt): +```bash +uv sync --frozen --extra tracing +uv run --extra tracing uvicorn niles.main:app --host 127.0.0.1 --port 8000 +``` + +Das Standard-Docker-Image enthaelt das optionale SDK nicht. Fuer Tracing das +Niles-Image mit dem Build-Argument bauen und den Core neu erstellen: + ```bash -uv add langfuse +docker compose --env-file .env -f docker/docker-compose.yml build --build-arg INSTALL_TRACING=true niles_core +docker compose --env-file .env -f docker/docker-compose.yml up -d --no-deps niles_core ``` +### Upgrade pruefen + +SDK-Regressionstests mit synthetischen Daten, ohne laufendes LLM oder Langfuse: + +```bash +uv sync --frozen --extra dev --extra tracing +uv run --frozen --extra dev --extra tracing pytest tests/test_tracing.py -v +``` + +Diese Tests pruefen den echten SDK-Wrapper fuer normale und gestreamte Antworten +einschliesslich Modell, Ein-/Ausgabe und Token-Verbrauch mit einem In-Memory- +Exporter. Sie ersetzen keinen Test gegen den eigenen Langfuse-Server: + +1. Server-Version im Langfuse-UI pruefen: mindestens 4.59.0. +2. Niles mit den eigenen Langfuse-Zugangsdaten starten. +3. Je eine synthetische Chat-Anfrage ohne Streaming und ueber den Web-Chat mit + Streaming ausfuehren, z. B. "Antworte nur mit Hallo.". +4. Im eigenen Langfuse-Projekt auf beide Generations warten und Modell, + Ein-/Ausgabe und Token-Verbrauch pruefen, soweit das LLM diese liefert. +5. Bei fehlenden Traces die Niles-Logs auf Exportfehler pruefen. Eine erfolgreiche + Chat-Antwort allein bestaetigt keinen erfolgreichen Trace-Export. + +Ohne vollstaendige `LANGFUSE_HOST`, `LANGFUSE_PUBLIC_KEY` und +`LANGFUSE_SECRET_KEY` bleibt der normale lokale Client aktiv. Traces enthalten +Chat-Inhalte; nur einen bewusst konfigurierten eigenen Server verwenden. + ### Token Budget Alert (Prometheus) Niles exportiert Token-Metriken unter `/metrics` (Prometheus-Format): diff --git a/docs/Niles-Core-Spec.md b/docs/Niles-Core-Spec.md index e58ae5f6..9d792f36 100644 --- a/docs/Niles-Core-Spec.md +++ b/docs/Niles-Core-Spec.md @@ -30,7 +30,7 @@ Niles is a local, private AI butler on a Mac Mini M4. It receives events from va | Vikunja 1.1.0 | 3456 | `https://vikunja.example.local` | Todo/task management | | signal-cli-rest-api | 8080 | Not exposed | Signal gateway (optional) | | SearXNG | 8080 | Not exposed | Meta search engine (optional, profile `search`) | -| Langfuse 3 | 3000 | `127.0.0.1:3000` | LLM tracing (optional, profile `langfuse`) | +| Langfuse >= 4.59.0 | 3000 | `127.0.0.1:3000` | LLM tracing (optional, profile `langfuse`, Python SDK v5) | **Network architecture:** All Docker services communicate internally via HTTP on `niles_network`. External access through homelab-gateway (Caddy reverse proxy on `proxy` network, HTTPS with self-signed certs, subdomain-based routing). PostgreSQL and internal service ports are not exposed. signal-cli-rest-api runs as a Docker service alongside other containers (activated via `feature_signal` in Settings UI). @@ -1476,7 +1476,7 @@ Multi-stage build (Builder → gws-downloader → Runtime): | `vikunja` | `vikunja/vikunja:1.1.0` | niles_network, proxy | Todo/task management | | `niles_signal_api` | `bbernhard/signal-cli-rest-api:1771797934-ci` | niles_network | Signal gateway (signal-cli v0.13.24) | | `niles_searxng` | `searxng/searxng:2025.5.10-1b787ed35` | niles_network | Meta search engine (profile `search`) | -| `niles_langfuse` | `langfuse/langfuse:3` | niles_network | LLM tracing (opt-in, profile `langfuse`) | +| `niles_langfuse` | `langfuse/langfuse:4.59.0` (digest-pinned) | niles_network | LLM tracing (opt-in, profile `langfuse`, Python SDK v5) | HTTPS termination is handled by `gateway-caddy` in the separate [homelab-gateway](../../homelab-gateway) repo. diff --git a/src/niles/agent/core.py b/src/niles/agent/core.py index c88effa7..c29c49e2 100644 --- a/src/niles/agent/core.py +++ b/src/niles/agent/core.py @@ -96,7 +96,7 @@ def __init__( self._OpenAI = _TracedOpenAI logger.info("LLM tracing enabled via Langfuse (%s)", config.langfuse_host) except ImportError: - logger.warning("langfuse package not installed; LLM tracing disabled. Install: uv add langfuse") + logger.warning("langfuse package not installed; LLM tracing disabled. Install: uv sync --extra tracing") self.llm_timeout = config.llm_timeout self.llm = self._OpenAI(base_url=config.llm_base_url, api_key="not-needed", timeout=self.llm_timeout) self.model = config.llm_model diff --git a/tests/test_tracing.py b/tests/test_tracing.py new file mode 100644 index 00000000..2c2bd1ef --- /dev/null +++ b/tests/test_tracing.py @@ -0,0 +1,171 @@ +"""Langfuse SDK compatibility tests with synthetic LLM responses.""" + +import json +from unittest.mock import AsyncMock, patch + +import httpx +import pytest +from openai import AsyncOpenAI + +from niles.agent.core import NilesAgent +from tests.helpers import make_test_settings + + +def _make_agent(**overrides): + settings = make_test_settings( + llm_base_url="http://llm.test/v1", + llm_model="test-model", + langfuse_host=overrides.get("langfuse_host", ""), + langfuse_public_key=overrides.get("langfuse_public_key", ""), + langfuse_secret_key=overrides.get("langfuse_secret_key", ""), + ) + history = AsyncMock() + history.get_recent.return_value = [] + memory = AsyncMock() + memory.list_all.return_value = [] + with patch("niles.agent.core.load_system_prompt", return_value="Synthetic test assistant."): + return NilesAgent( + config=settings, + contacts=AsyncMock(), + whatsapp=AsyncMock(), + memory=memory, + history=history, + ) + + +@pytest.mark.parametrize( + "config", + [ + {}, + {"langfuse_host": "http://langfuse.test"}, + {"langfuse_host": "http://langfuse.test", "langfuse_public_key": "pk-lf-test"}, + ], +) +async def test_tracing_requires_complete_configuration(config): + with patch("builtins.__import__", wraps=__import__) as imports: + agent = _make_agent(**config) + try: + assert agent._OpenAI is AsyncOpenAI + assert not any(call.args[0].startswith("langfuse") for call in imports.call_args_list) + finally: + await agent.llm.close() + + +async def test_missing_tracing_package_warns_and_keeps_local_client(caplog, monkeypatch): + original_import = __import__ + monkeypatch.setenv("LANGFUSE_HOST", "http://langfuse.test") + monkeypatch.setenv("LANGFUSE_PUBLIC_KEY", "pk-lf-test") + monkeypatch.setenv("LANGFUSE_SECRET_KEY", "sk-lf-test") + + def without_langfuse(name, *args, **kwargs): + if name.startswith("langfuse"): + raise ModuleNotFoundError("Synthetic missing optional package") + return original_import(name, *args, **kwargs) + + with patch("builtins.__import__", side_effect=without_langfuse): + agent = _make_agent( + langfuse_host="http://langfuse.test", + langfuse_public_key="pk-lf-test", + langfuse_secret_key="sk-lf-test", # pragma: allowlist secret + ) + try: + assert agent._OpenAI is AsyncOpenAI + assert "LLM tracing disabled" in caplog.text + assert "uv sync --extra tracing" in caplog.text + finally: + await agent.llm.close() + + +@pytest.fixture(scope="module") +def tracing_client(): + langfuse = pytest.importorskip("langfuse") + from opentelemetry.sdk.trace import TracerProvider + from opentelemetry.sdk.trace.export.in_memory_span_exporter import InMemorySpanExporter + + exporter = InMemorySpanExporter() + client = langfuse.Langfuse( + public_key="pk-lf-test", + secret_key="sk-lf-test", # pragma: allowlist secret + base_url="http://langfuse.test", + tracer_provider=TracerProvider(), + span_exporter=exporter, + ) + yield client, exporter + client.shutdown() + + +@pytest.mark.parametrize("streaming", [False, True]) +async def test_real_sdk_records_agent_generation(tracing_client, monkeypatch, streaming): + client, exporter = tracing_client + client.flush() + exporter.clear() + requests = [] + + async def handle_request(transport, request): + assert request.url == "http://llm.test/v1/chat/completions" + body = json.loads(request.content) + requests.append(body) + completion = { + "id": "chatcmpl-test", + "object": "chat.completion", + "created": 1, + "model": "test-model", + "choices": [{"index": 0, "message": {"role": "assistant", "content": "Hello."}, "finish_reason": "stop"}], + "usage": {"prompt_tokens": 12, "completion_tokens": 3, "total_tokens": 15}, + } + if not body.get("stream", False): + return httpx.Response(200, json=completion) + chunks = [ + { + **{key: completion[key] for key in ("id", "created", "model")}, + "object": "chat.completion.chunk", + "choices": [{"index": 0, "delta": {"role": "assistant", "content": "Hello."}, "finish_reason": None}], + }, + { + **{key: completion[key] for key in ("id", "created", "model")}, + "object": "chat.completion.chunk", + "choices": [{"index": 0, "delta": {}, "finish_reason": "stop"}], + }, + { + **{key: completion[key] for key in ("id", "created", "model")}, + "object": "chat.completion.chunk", + "choices": [], + "usage": completion["usage"], + }, + ] + data = "".join(f"data: {json.dumps(chunk)}\n\n" for chunk in chunks) + "data: [DONE]\n\n" + return httpx.Response(200, headers={"content-type": "text/event-stream"}, content=data) + + monkeypatch.setattr(httpx.AsyncHTTPTransport, "handle_async_request", handle_request) + monkeypatch.setenv("LANGFUSE_HOST", "http://langfuse.test") + monkeypatch.setenv("LANGFUSE_PUBLIC_KEY", "pk-lf-test") + monkeypatch.setenv("LANGFUSE_SECRET_KEY", "sk-lf-test") + agent = _make_agent( + langfuse_host="http://langfuse.test", + langfuse_public_key="pk-lf-test", + langfuse_secret_key="sk-lf-test", # pragma: allowlist secret + ) + event = {"type": "web", "from": "web-user-1", "content": "Say hello."} + try: + if streaming: + events = [item async for item in agent.process_event_stream(event)] + assert events[-1] == {"type": "done"} + assert "".join(item["text"] for item in events if item["type"] == "chunk") == "Hello." + else: + assert await agent.process_event(event) == "Hello." + agent.history.add_message.assert_any_await("web-user-1", "assistant", "Hello.") + assert len(requests) == 1 + assert requests[0].get("stream", False) is streaming + client.flush() + spans = exporter.get_finished_spans() + assert len(spans) == 1 + attributes = spans[0].attributes + assert attributes["langfuse.observation.type"] == "generation" + assert attributes["langfuse.observation.model.name"] == "test-model" + assert "Say hello." in attributes["langfuse.observation.input"] + assert "Hello." in attributes["langfuse.observation.output"] + usage = json.loads(attributes["langfuse.observation.usage_details"]) + assert usage["prompt_tokens"] == 12 + assert usage["completion_tokens"] == 3 + finally: + await agent.llm.close()