From ba24699d4b1d03cfc22d857814ef32da6e1c689c Mon Sep 17 00:00:00 2001 From: Coding-Dev-Tools Date: Mon, 20 Jul 2026 00:39:30 -0400 Subject: [PATCH 01/11] feat(commercial): add commercial v1 ga code and test cases --- .env.example | 113 +- .github/workflows/ci.yml | 71 +- .github/workflows/commercial-backup.yml | 48 + .../workflows/commercial-restore-drill.yml | 110 + .github/workflows/production-synthetics.yml | 53 + .github/workflows/release.yml | 111 +- .gitignore | 4 + CHANGELOG.md | 71 +- Dockerfile | 9 +- NOTICE | 9 +- README.md | 134 +- SECURITY.md | 8 +- _measure_d3.mjs | 51 + _measure_galaxy.mjs | 51 + _smoke.mjs | 0 _smoke2.mjs | 0 deploy/railway-template.json | 61 + docs/AGENT_CONNECT.md | 20 +- docs/COMMERCIAL_OPERATIONS.md | 248 ++ docs/HOSTING_RAILWAY.md | 352 ++- docs/RAILWAY_TEMPLATE.md | 116 +- docs/SYNC.md | 51 +- engraphis/__init__.py | 2 +- engraphis/app.py | 49 +- engraphis/autosync.py | 6 +- engraphis/backends/embedder_api.py | 14 +- engraphis/backends/extractor.py | 20 +- engraphis/backends/graph_extractor.py | 41 +- engraphis/backends/sync_folder.py | 5 +- engraphis/backends/sync_relay.py | 171 +- engraphis/billing.py | 391 +++- engraphis/cloud_license.py | 179 +- engraphis/commercial.py | 290 +++ engraphis/commercial_manifest.json | 63 + engraphis/config.py | 128 +- engraphis/core/engine.py | 19 +- engraphis/core/graph_scene.py | 1706 ++++++++++++++ engraphis/core/schema.py | 130 +- engraphis/core/store.py | 716 +++++- engraphis/core/sync.py | 28 +- engraphis/dashboard_app.py | 127 +- engraphis/email_outbox.py | 443 ++++ engraphis/engines/intelligence.py | 12 +- engraphis/engines/thoughts.py | 15 +- engraphis/http_security.py | 64 +- engraphis/inspector/__init__.py | 5 +- engraphis/inspector/app.py | 35 +- engraphis/inspector/auth.py | 332 ++- engraphis/inspector/cloud_mount.py | 21 +- engraphis/inspector/license_cloud.py | 948 ++++++-- engraphis/inspector/license_compat_proxy.py | 140 ++ engraphis/inspector/license_registry.py | 361 ++- engraphis/inspector/sync_relay.py | 120 +- engraphis/inspector/webhooks.py | 665 ++++-- engraphis/licensing.py | 85 +- engraphis/llm/client.py | 68 +- engraphis/observability.py | 86 + engraphis/resend_events.py | 101 + engraphis/routes/memory.py | 6 +- engraphis/routes/v2_api.py | 734 +++++- engraphis/routes/v2_team.py | 322 ++- engraphis/service.py | 2024 ++++++++++++++++- engraphis/static/dashboard.css | 746 ++++++ engraphis/static/dashboard.js | 1440 ++++++++++++ engraphis/static/index.html | 2022 ++-------------- engraphis/vendor_app.py | 159 ++ pyproject.toml | 7 +- railway.json | 4 +- scripts/check_commercial_manifest.py | 174 ++ scripts/commercial_backup.py | 685 ++++++ scripts/externalize_dashboard_assets.py | 158 ++ scripts/license_admin.py | 360 ++- scripts/start_dashboard.py | 16 +- scripts/start_server.py | 19 +- scripts/sync.py | 74 +- skills/engraphis-memory/SKILL.md | 14 + tests/conftest.py | 31 + tests/e2e/commercial.spec.js | 222 ++ tests/graph_scene_fixture.json | 204 ++ tests/team_client.py | 37 + tests/test_agent_connect.py | 171 +- tests/test_agent_connect_mcp.py | 2 +- tests/test_autosync.py | 5 + tests/test_billing.py | 702 +++++- tests/test_cloud_license.py | 528 ++++- tests/test_commercial_backup.py | 187 ++ tests/test_commercial_ga.py | 718 ++++++ tests/test_config.py | 20 +- tests/test_core_store.py | 52 +- tests/test_dashboard_auth_placement.py | 54 +- tests/test_dashboard_dream_ui.py | 16 +- tests/test_dashboard_v2.py | 358 ++- tests/test_extractor.py | 4 + tests/test_graph_explorer_v2.py | 1671 ++++++++++++++ tests/test_graph_extractor.py | 17 + tests/test_graph_scene_contract.py | 64 + tests/test_http_security.py | 46 +- tests/test_inspector_pro.py | 35 + tests/test_license_compat_proxy.py | 92 + tests/test_licensing.py | 237 +- tests/test_llm_config.py | 55 + tests/test_llm_dashboard.py | 188 ++ tests/test_observability.py | 58 + tests/test_packaging.py | 19 +- tests/test_password_reset.py | 86 +- tests/test_provider_error_redaction.py | 260 +++ tests/test_release_infrastructure.py | 94 + tests/test_security_hardening_2026_07_18.py | 12 +- tests/test_start_dashboard.py | 41 + tests/test_sync.py | 28 +- tests/test_sync_cli.py | 92 +- tests/test_sync_dashboard.py | 27 + tests/test_sync_relay.py | 126 +- tests/test_team_audit.py | 269 ++- tests/test_vendor_admin_token.py | 10 +- tests/test_webhooks.py | 42 +- tests/test_workspace_ops.py | 126 + tests/vendor_relay.py | 55 + 118 files changed, 21748 insertions(+), 3554 deletions(-) create mode 100644 .github/workflows/commercial-backup.yml create mode 100644 .github/workflows/commercial-restore-drill.yml create mode 100644 .github/workflows/production-synthetics.yml create mode 100644 _measure_d3.mjs create mode 100644 _measure_galaxy.mjs create mode 100644 _smoke.mjs create mode 100644 _smoke2.mjs create mode 100644 deploy/railway-template.json create mode 100644 docs/COMMERCIAL_OPERATIONS.md create mode 100644 engraphis/commercial.py create mode 100644 engraphis/commercial_manifest.json create mode 100644 engraphis/core/graph_scene.py create mode 100644 engraphis/email_outbox.py create mode 100644 engraphis/inspector/license_compat_proxy.py create mode 100644 engraphis/observability.py create mode 100644 engraphis/resend_events.py create mode 100644 engraphis/static/dashboard.css create mode 100644 engraphis/static/dashboard.js create mode 100644 engraphis/vendor_app.py create mode 100644 scripts/check_commercial_manifest.py create mode 100644 scripts/commercial_backup.py create mode 100644 scripts/externalize_dashboard_assets.py create mode 100644 tests/e2e/commercial.spec.js create mode 100644 tests/graph_scene_fixture.json create mode 100644 tests/team_client.py create mode 100644 tests/test_commercial_backup.py create mode 100644 tests/test_commercial_ga.py create mode 100644 tests/test_graph_explorer_v2.py create mode 100644 tests/test_graph_scene_contract.py create mode 100644 tests/test_license_compat_proxy.py create mode 100644 tests/test_llm_config.py create mode 100644 tests/test_llm_dashboard.py create mode 100644 tests/test_observability.py create mode 100644 tests/test_provider_error_redaction.py create mode 100644 tests/test_release_infrastructure.py create mode 100644 tests/vendor_relay.py diff --git a/.env.example b/.env.example index 0e1c3f8..52498e6 100644 --- a/.env.example +++ b/.env.example @@ -6,22 +6,32 @@ # ── Server ────────────────────────────────────────────────────────────────── ENGRAPHIS_HOST=127.0.0.1 ENGRAPHIS_PORT=8700 +# customer: dashboard/memory/sync only; vendor: issuance/billing/email only; +# combined: development compatibility mode (never use for either production service). +ENGRAPHIS_SERVICE_MODE=combined # Clients point at the server via this base URL (derived from host/port): # export ENGRAPHIS_BASE_URL="http://127.0.0.1:8700" -# Optional bearer token for the REST API. If set, protected /api routes require -# the header Authorization: Bearer . Hosted deployments must set a strong value -# before hosted trial or first-admin setup; enter it in the corresponding setup field. -# Leave empty only for local, single-user, loopback-bound use. +# Optional shared service/operations bearer. If set, supported protected routes accept +# Authorization: Bearer ; managed backup/readiness automation should use a strong, +# independently revocable value. It does not prove hosted deployment ownership and is not +# entered during trial or first-admin setup (use ENGRAPHIS_DEPLOYMENT_TOKEN there). ENGRAPHIS_API_TOKEN= +# Hosted deployment ownership secret. Railway customer templates require a unique value +# of at least 32 characters. It starts a deployment-bound trial and authorizes the first +# remote admin setup; it is not an agent token or a vendor credential. +ENGRAPHIS_DEPLOYMENT_TOKEN= + # VENDOR RELAY ONLY — token authorizing vendor-wide license administration on the # shared relay (/license/v1 revoke, keys-by-email, deactivate, device listing). # Keep it a SEPARATE secret from ENGRAPHIS_API_TOKEN: the API token is handed to # scripts/agents as a per-instance service credential and must never be able to # revoke customers' licenses. REQUIRED for those routes as of 0.9.8 — the old # fallback to ENGRAPHIS_API_TOKEN was removed (it made the separation above -# meaningless). Unset ⇒ the vendor admin routes are DISABLED, not open. +# meaningless). Unset ⇒ the vendor admin routes are DISABLED, not open. Production +# fails closed unless this is 32-4096 non-whitespace characters (no control ASCII). Generate it: +# python -c "import secrets; print(secrets.token_urlsafe(48))" ENGRAPHIS_VENDOR_ADMIN_TOKEN= # ── Storage ───────────────────────────────────────────────────────────────── @@ -52,8 +62,13 @@ ENGRAPHIS_EMBED_MODEL=sentence-transformers/all-MiniLM-L6-v2 # ── v2 write-path fact extraction (optional) ───────────────────────────────── # "none" (default): store text as given. "chunk": deterministic offline chunks. # "llm": free-form fact extraction. "llm_structured": schema-validated typed facts, -# entities, relations, keywords, and confidence using the LLM settings below. +# entities, relations, and keywords using the LLM settings below. ENGRAPHIS_EXTRACTOR=none +# A successful dashboard connection test turns llm_structured on automatically. The +# Settings On/Off button updates the live engine immediately and persists this flag plus +# ENGRAPHIS_EXTRACTOR only when the project .env is writable. Explicit deployment +# environment variables remain authoritative after restart. +ENGRAPHIS_LLM_AUTO_EXTRACT=1 # ── Knowledge-graph extraction (powers the dashboard Graph tab) ────────────── # "regex" (default): dependency-free heuristic NER runs on every ingest so the Graph tab @@ -61,6 +76,8 @@ ENGRAPHIS_EXTRACTOR=none # Validated entity/relation metadata from "llm_structured" still feeds the graph # automatically. Existing memories are backfilled when a workspace graph first opens. ENGRAPHIS_GRAPH_EXTRACTOR=regex +# Analytical Galaxy is the default; set 0 for the one-release legacy ForceGraph fallback. +ENGRAPHIS_GRAPH_UI_V2=1 # Optional host-LLM retention supervision. "none" preserves the deterministic write path; # "llm" sends a bounded excerpt to the configured provider for an advisory @@ -97,9 +114,8 @@ ENGRAPHIS_RETENTION_SUPERVISOR=none # Referrer-Policy, Permissions-Policy, and (over HTTPS only) HSTS. Both variables # REPLACE the built-in value wholesale; set either to an EMPTY string to omit that # header entirely, which is what you want when a fronting proxy sets its own. -# The default CSP is same-origin with 'unsafe-inline' (the dashboard is one HTML file -# with inline script/style); it needs no third-party allowances because d3/marked/ -# DOMPurify are vendored under /static. Widen it only if you add an external origin. +# The default CSP is strict same-origin and contains no unsafe-inline; dashboard CSS, +# JavaScript, and vendored libraries are served under /static. # Quote the values — both contain characters a shell would otherwise split on. # ENGRAPHIS_CSP="default-src 'self'; frame-ancestors 'none'" # replace the policy # ENGRAPHIS_CSP="" # send no CSP at all @@ -145,12 +161,11 @@ ENGRAPHIS_DECAY_HALFLIFE_DAYS=7 # Team license to add seats beyond the first admin; first visit creates the admin. # ENGRAPHIS_TEAM_MODE=0 -# Public URL of your dashboard, included as a sign-in link in the "you've been -# added to a team" notification email sent when an admin adds a member. Without -# it, the email just tells the new member to ask their admin for the address. +# Canonical HTTPS URL of this dashboard. Deployment claims bind this origin, and pending +# invitation emails contain a one-time password-creation link at this origin. # ENGRAPHIS_DASHBOARD_URL=https://dash.yourcompany.com -# "Add member" invite emails: if THIS instance has its own ENGRAPHIS_RESEND_API_KEY +# Pending-invitation emails: if THIS instance has its own ENGRAPHIS_RESEND_API_KEY # or ENGRAPHIS_SMTP_* configured (see the email-delivery section below), it sends # invites directly from your own address. Otherwise it automatically falls back to # relaying the invite through the VENDOR's mail provider at your Team license's signed @@ -163,6 +178,11 @@ ENGRAPHIS_DECAY_HALFLIFE_DAYS=7 # enough for a real team onboarding, low enough to bound abuse of a free key. # ENGRAPHIS_TEAM_INVITE_DAILY_CAP=10 +# Vendor-side hosted password-reset relay limits. Resets are authenticated by an active +# Pro/Team key, origin-bound to the customer deployment, and queued in the durable outbox. +# ENGRAPHIS_PASSWORD_RESET_DAILY_CAP=20 +# ENGRAPHIS_PASSWORD_RESET_RECIPIENT_HOURLY_CAP=5 + # Where 402 responses / upgrade banners send users. Set to your real checkout at launch. # ENGRAPHIS_PRO_UPGRADE_URL=https://buy.polar.sh/ # ENGRAPHIS_TEAM_UPGRADE_URL=https://buy.polar.sh/ @@ -185,31 +205,46 @@ ENGRAPHIS_DECAY_HALFLIFE_DAYS=7 # ENGRAPHIS_AUTOSYNC_LOOP=1 # Override where the auto-sync policy is stored (default: next to ENGRAPHIS_DB_PATH). # ENGRAPHIS_AUTOSYNC_STATE=/data/.engraphis/autosync.json +# Local client credential for the managed relay. Prefer saving it through the dashboard; +# if supplied here, use only a per-user token from /api/auth/token, never a license key. +# ENGRAPHIS_SYNC_TOKEN=engr_ut_replace_me +# ENGRAPHIS_SYNC_READ_ONLY=0 +# Temporary customer migration switch; leave off when no legacy customers exist. +# ENGRAPHIS_LEGACY_TEAM_KEY_SYNC=0 # ── Commercial / vendor (ONLY if you SELL Engraphis — not needed to self-host) ── # Server-side of cloud enforcement + fulfillment. Set these on your vendor host. # # Vendor Ed25519 signing seed — signs issued license KEYS and short-lived LEASES. # 64-char hex inline (how Railway/Fly set it) OR a path to a file with that hex. +# A file must be a regular file no larger than 1 KiB and, on POSIX, owner-only +# (chmod 600); non-regular or group/world-readable targets are refused. # MUST correspond to the pinned public key in engraphis/licensing.py (_VENDOR_PUBKEY_HEX), # else every key you issue fails to verify on clients. Verify with: -# python -m scripts.license_admin verify-key +# python -m scripts.license_admin verify '' # ENGRAPHIS_VENDOR_SIGNING_KEY=<64-char-hex-seed or /path/to/vendor_signing.key> # # Registry + relay + registrations DB (issued keys, revocations, sync bundles, device # seats). Put it on a PERSISTENT volume or revoked keys un-revoke on restart. # ENGRAPHIS_RELAY_DB=/data/.engraphis/relay.db +# Durable Polar delivery/order claims and Team seat baselines. This must also live on the +# persistent vendor volume. Vendor-mode backup fails closed if this file or the relay DB is +# absent, so initialize the real stores before enabling the daily backup workflow. +# ENGRAPHIS_WEBHOOK_STATE=/data/.engraphis/polar-webhooks.db # # Lease lifetime (hours). Lower = faster revocation takes effect, more phone-home. # Default 24 (license_registry.LEASE_TTL_HOURS_DEFAULT), floored at 5 minutes. # ENGRAPHIS_LEASE_TTL_HOURS=24 # -# REQUIRED to offer self-serve trials. The public https:// base URL of THIS relay, used +# REQUIRED on the vendor control plane. The public https:// base URL of THIS service, used # to build the emailed magic-link. There is deliberately no fallback: deriving it from # the request would mean deriving it from the client-supplied Host header, which let an # attacker have a victim emailed a confirmation link pointing at attacker-controlled # infrastructure (fixed 2026-07-18). Unset ⇒ /license/v1/start-trial answers 503. -# ENGRAPHIS_RELAY_PUBLIC_URL=https://team.engraphis.com +# ENGRAPHIS_RELAY_PUBLIC_URL=https://license.engraphis.com +# Development-only escape hatch for the pre-GA key-copy trial route. Never enable on the +# production vendor service; readiness and onboarding use /license/v1/trial-claims. +# ENGRAPHIS_ENABLE_LEGACY_TRIAL_FLOW=0 # # Per-account relay storage ceilings. Defaults 2 GiB / 64 workspaces. The per-workspace # caps alone bound nothing — workspace ids are caller-supplied — and the relay DB shares @@ -218,19 +253,30 @@ ENGRAPHIS_DECAY_HALFLIFE_DAYS=7 # ENGRAPHIS_RELAY_MAX_ACCOUNT_BYTES=2147483648 # ENGRAPHIS_RELAY_MAX_WORKSPACES_PER_ACCOUNT=64 # -# Per-IP burst cap on the unauthenticated key-verifying routes: /license/v1/register and -# /license/v1/team-invite share ONE budget (per minute, per process), so alternating +# Per-IP burst cap on the unauthenticated key-verifying routes: /license/v1/register, +# /license/v1/team-invite, and /license/v1/password-reset share ONE budget (per minute, +# per process), so alternating # between them buys no extra work. Guards the ~3 ms pure-Python Ed25519 verify from being # used as an unauthenticated CPU DoS. 0 disables. Default 60. # ENGRAPHIS_REGISTER_RATE_PER_MINUTE=60 # # Polar billing webhook (auto-fulfill: order.paid -> signed key -> email). +# Use Polar's generated value. Raw secrets, or the decoded bytes of legacy whsec_ +# values, must contain at least 16 bytes of key material or readiness fails closed. # POLAR_WEBHOOK_SECRET=whsec_xxxxxxxxxxxxxxxxxxxxxxxx -# POLAR_ORGANIZATION_ID= # optional: verify the webhook's org matches +# POLAR_ORGANIZATION_ID= # required in vendor production +# These values must match engraphis/commercial_manifest.json exactly. Unknown products +# and wrong-organization events are rejected rather than heuristically fulfilled. +# POLAR_PRO_MONTHLY_PRODUCT_ID=6349d29a-b4c0-4b8e-b9e8-071705351c9c +# POLAR_PRO_ANNUAL_PRODUCT_ID=5e10d5d2-607a-4e9c-ad1c-88cb48c37e11 +# POLAR_TEAM_MONTHLY_PRODUCT_ID=6444a48a-7a61-4c8e-8045-07a930f8f381 +# POLAR_TEAM_ANNUAL_PRODUCT_ID=929d926a-2981-4ad7-95bd-f52dabf0794e # # Outbound email (license-key delivery AND team invite notifications). Resend # API (preferred) OR SMTP. # ENGRAPHIS_RESEND_API_KEY=re_xxxxxxxxxxxx +# Use Resend's generated value; its raw/decoded signing key must be at least 16 bytes. +# RESEND_WEBHOOK_SECRET=whsec_xxxxxxxxxxxxxxxxxxxxxxxx # ENGRAPHIS_SMTP_FROM=licenses@yourdomain.com # ENGRAPHIS_SMTP_HOST= # alternative to Resend # ENGRAPHIS_SMTP_PORT=587 @@ -239,13 +285,36 @@ ENGRAPHIS_DECAY_HALFLIFE_DAYS=7 # # ── Client side (end-user machines) ───────────────────────────────────────── # Paid keys require a revocable, machine-bound lease from the managed service. The -# built-in default is https://team.engraphis.com; override only for a different vendor +# built-in default is https://license.engraphis.com; override only for a different vendor # deployment. Free-tier use remains local and does not require the service. -# ENGRAPHIS_CLOUD_URL=https://team.engraphis.com +# ENGRAPHIS_CLOUD_URL=https://license.engraphis.com # Server-side license enforcement (vendor/fulfillment server only): when set, every key # minted by the Polar webhook carries a signed enforce:"cloud" claim + this URL, and the # client requires a live lease from it (revocable, seat-counted, useless offline). # Set this to the canonical managed-service URL for newly issued keys. Older keys carrying # the retired Railway hostname are migrated by the client without changing their signature. -# ENGRAPHIS_KEY_CLOUD_URL=https://team.engraphis.com +# ENGRAPHIS_KEY_CLOUD_URL=https://license.engraphis.com + +# ── Encrypted commercial backups ──────────────────────────────────────────── +# Exact 32-byte key as 64 hex characters, supplied only through the production secret +# store. Mount off-volume storage and schedule scripts/commercial_backup.py daily. +# ENGRAPHIS_BACKUP_KEY= +# ENGRAPHIS_BACKUP_OUTPUT_DIR=/backup/engraphis +# ENGRAPHIS_BACKUP_STATUS_FILE=/data/.engraphis/backup-status.json +# ENGRAPHIS_BACKUP_RETENTION_DAYS=30 +# ENGRAPHIS_BACKUP_MAX_AGE_SECONDS=93600 +# ENGRAPHIS_CUSTOMER_MIN_FREE_BYTES=268435456 +# ENGRAPHIS_REJECTED_LEASE_ALERT_THRESHOLD=50 +# ENGRAPHIS_REJECTED_LEASE_ALERT_WINDOW_SECONDS=3600 +# Control-plane readiness fails when pending/retrying mail is too old, a message exhausts +# retries, or the 24-hour bounce/complaint rate is too high after the minimum sample size. +# ENGRAPHIS_EMAIL_MAX_BACKLOG_AGE_SECONDS=900 +# ENGRAPHIS_EMAIL_MAX_BOUNCE_RATE=0.05 +# ENGRAPHIS_EMAIL_BOUNCE_MIN_SAMPLE=20 +# ENGRAPHIS_JSON_LOGS=1 +# Customer deployments with local Resend/SMTP retry durable email every 30 seconds. +# ENGRAPHIS_EMAIL_OUTBOX_LOOP=1 +# Example (destination must be a separate mounted device in production): +# python -m scripts.commercial_backup backup --output-dir /backup/engraphis \ +# --marker /data/.engraphis/backup-status.json --retention-days 30 diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 77481a7..42309fb 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -5,6 +5,9 @@ on: branches: [main] pull_request: +permissions: + contents: read + jobs: test: name: test + lint (full offline stack) @@ -14,8 +17,8 @@ jobs: matrix: python-version: ["3.10", "3.11", "3.12"] steps: - - uses: actions/checkout@v7 - - uses: actions/setup-python@v6 + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7 + - uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6 with: python-version: ${{ matrix.python-version }} - name: Install (core + server/mcp/code extras; no torch — the offline gate) @@ -24,6 +27,10 @@ jobs: pip install -e ".[test]" - name: Lint (ruff) run: ruff check . + - name: Commercial manifest + strict-CSP drift gate + run: | + python scripts/check_commercial_manifest.py + python scripts/externalize_dashboard_assets.py - name: Unit tests (full suite — extras-gated tests included) run: python -m pytest tests/ -q - name: Retrieval eval gate @@ -37,8 +44,8 @@ jobs: name: core floor (numpy-only, Python 3.9) runs-on: ubuntu-latest steps: - - uses: actions/checkout@v7 - - uses: actions/setup-python@v6 + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7 + - uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6 with: python-version: "3.9" - name: Install (numpy-only core — the minimum supported runtime) @@ -52,16 +59,39 @@ jobs: - name: Ablation run: python -m eval.ablation + browser-accessibility: + name: browser accessibility smoke + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7 + - uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6 + with: + python-version: "3.11" + - uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0 + with: + node-version: "24" + cache: npm + - name: Install dashboard and browser test dependencies + run: | + python -m pip install --upgrade pip + pip install -e ".[test]" + npm ci + npx playwright install --with-deps chromium + - name: Build and verify deterministic galaxy assets + run: npm run test:galaxy + - name: Playwright desktop/mobile, keyboard, CSP, console, and axe checks + run: npm run test:e2e + docker-gate: # Keeps CI fast: the docker job always runs on push to main, but on PRs only - # when image-relevant paths changed (Dockerfile, engraphis/**, scripts/**, + # when image/deployment paths changed (Dockerfile, entrypoint, Railway, source, # pyproject.toml, or this workflow). No third-party actions — plain git diff. name: docker smoke — path gate runs-on: ubuntu-latest outputs: run: ${{ steps.decide.outputs.run }} steps: - - uses: actions/checkout@v7 + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7 with: fetch-depth: 0 - id: decide @@ -69,7 +99,7 @@ jobs: if [ "${{ github.event_name }}" != "pull_request" ]; then echo "run=true" >> "$GITHUB_OUTPUT" elif git diff --name-only "${{ github.event.pull_request.base.sha }}" "${{ github.sha }}" \ - | grep -qE '^(Dockerfile|\.dockerignore|engraphis/|scripts/|pyproject\.toml|\.github/workflows/ci\.yml)'; then + | grep -qE '^(Dockerfile|docker-entrypoint\.sh|docker-compose\.yml|railway\.json|deploy/|\.dockerignore|engraphis/|scripts/|pyproject\.toml|\.github/workflows/ci\.yml)'; then echo "run=true" >> "$GITHUB_OUTPUT" else echo "run=false" >> "$GITHUB_OUTPUT" @@ -81,15 +111,26 @@ jobs: needs: docker-gate if: needs.docker-gate.outputs.run == 'true' steps: - - uses: actions/checkout@v7 + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7 - name: Build image run: docker build -t engraphis:ci . + - name: Verify production image OCR runtime + run: >- + docker run --rm --entrypoint sh engraphis:ci -c + 'python -c "import PIL, pytesseract" && command -v tesseract >/dev/null && + tesseract --version | head -n 1' + - name: Audit the exact production image dependency set + run: >- + docker run --rm --entrypoint sh engraphis:ci -c + 'python -m pip install --no-cache-dir pip-audit && + python -m pip_audit --local' - name: Run container (offline deterministic embedder — no model downloads) run: | docker run -d --name engraphis -p 8700:8700 \ -e ENGRAPHIS_EMBED_MODEL= \ -e ENGRAPHIS_LOOP_INTERVAL=0 \ -e ENGRAPHIS_HOST=0.0.0.0 \ + -e ENGRAPHIS_SERVICE_MODE=customer \ engraphis:ci - name: Wait for /api/health, then check /api/ready run: | @@ -113,13 +154,17 @@ jobs: name: build + install wheel runs-on: ubuntu-latest steps: - - uses: actions/checkout@v7 - - uses: actions/setup-python@v6 + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7 + - uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6 with: python-version: "3.11" - name: Build sdist + wheel and verify a clean install run: | - python -m pip install --upgrade pip build + python -m pip install --upgrade pip build pip-audit python -m build - pip install dist/*.whl - python -c "import engraphis; print('wheel import OK')" + python -m venv .audit-venv + .audit-venv/bin/python -m pip install --upgrade "pip>=26.1.2" "setuptools>=83" + .audit-venv/bin/python -m pip install dist/*.whl + AUDIT_SITE=$(.audit-venv/bin/python -c "import site; print(site.getsitepackages()[0])") + python -m pip_audit --path "$AUDIT_SITE" + .audit-venv/bin/python -c "import engraphis; print('wheel import OK')" diff --git a/.github/workflows/commercial-backup.yml b/.github/workflows/commercial-backup.yml new file mode 100644 index 0000000..e8da17b --- /dev/null +++ b/.github/workflows/commercial-backup.yml @@ -0,0 +1,48 @@ +name: commercial encrypted backups + +on: + schedule: + - cron: "41 3 * * *" + workflow_dispatch: + +permissions: + contents: read + +jobs: + backup: + runs-on: ubuntu-latest + timeout-minutes: 15 + env: + CUSTOMER_URL: ${{ secrets.ENGRAPHIS_CUSTOMER_URL }} + # Use the permanent, independently revocable service credential. The deployment + # token is only for ownership proof and first-admin setup. + CUSTOMER_TOKEN: ${{ secrets.ENGRAPHIS_CUSTOMER_OPS_TOKEN }} + LICENSE_URL: ${{ secrets.ENGRAPHIS_LICENSE_URL }} + VENDOR_TOKEN: ${{ secrets.ENGRAPHIS_VENDOR_ADMIN_TOKEN }} + steps: + - name: Require production backup targets + run: | + test -n "$CUSTOMER_URL" + test -n "$CUSTOMER_TOKEN" + test -n "$LICENSE_URL" + test -n "$VENDOR_TOKEN" + case "$CUSTOMER_URL" in https://*) ;; *) echo "customer URL must use HTTPS"; exit 1;; esac + case "$LICENSE_URL" in https://*) ;; *) echo "license URL must use HTTPS"; exit 1;; esac + - name: Back up managed customer service + run: | + printf 'Authorization: Bearer %s\n' "$CUSTOMER_TOKEN" | + curl --fail --silent --show-error --max-time 300 --proto '=https' \ + -X POST --header @- --url "${CUSTOMER_URL%/}/api/ops/backup" + - name: Back up license control plane + run: | + printf 'Authorization: Bearer %s\n' "$VENDOR_TOKEN" | + curl --fail --silent --show-error --max-time 300 --proto '=https' \ + -X POST --header @- --url "${LICENSE_URL%/}/ops/backup" + - name: Require fresh readiness after backup + run: | + printf 'Authorization: Bearer %s\n' "$CUSTOMER_TOKEN" | + curl --fail --silent --show-error --max-time 20 --proto '=https' \ + --header @- --url "${CUSTOMER_URL%/}/api/ops/ready" + printf 'Authorization: Bearer %s\n' "$VENDOR_TOKEN" | + curl --fail --silent --show-error --max-time 20 --proto '=https' \ + --header @- --url "${LICENSE_URL%/}/ops/ready" diff --git a/.github/workflows/commercial-restore-drill.yml b/.github/workflows/commercial-restore-drill.yml new file mode 100644 index 0000000..36cc866 --- /dev/null +++ b/.github/workflows/commercial-restore-drill.yml @@ -0,0 +1,110 @@ +name: commercial encrypted restore drill + +on: + schedule: + - cron: "19 4 1 * *" + workflow_dispatch: + +permissions: + contents: read + +jobs: + restore-drill: + runs-on: ubuntu-latest + timeout-minutes: 15 + steps: + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7 + - uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6 + with: + python-version: "3.11" + - name: Install backup encryption dependency + run: python -m pip install "cryptography>=48.0.1" + - name: Create isolated vendor-state fixtures + shell: bash + run: | + python - <<'PY' + import os + import secrets + import sqlite3 + import uuid + from pathlib import Path + + root = Path(os.environ["RUNNER_TEMP"]) / ( + "engraphis-commercial-restore-" + uuid.uuid4().hex) + live = root / "live" + live.mkdir(parents=True) + relay = live / "relay.db" + webhooks = live / "polar-webhooks.db" + + with sqlite3.connect(relay) as conn: + conn.execute( + "CREATE TABLE issued_licenses(" + "key_id TEXT PRIMARY KEY, subscription_id TEXT, order_id TEXT)") + conn.execute( + "INSERT INTO issued_licenses VALUES " + "('drill-license', 'drill-subscription', 'drill-order')") + with sqlite3.connect(webhooks) as conn: + conn.execute( + "CREATE TABLE processed(webhook_id TEXT PRIMARY KEY, state TEXT)") + conn.execute( + "INSERT INTO processed VALUES ('drill-delivery', 'fulfilled')") + conn.execute( + "CREATE TABLE subscription_seats(" + "subscription_id TEXT PRIMARY KEY, seats INTEGER, event_ts REAL)") + conn.execute( + "INSERT INTO subscription_seats VALUES ('drill-subscription', 5, 1.0)") + + values = { + "DRILL_ROOT": root, + "ENGRAPHIS_SERVICE_MODE": "vendor", + "ENGRAPHIS_DB_PATH": live / "memory.db", + "ENGRAPHIS_RELAY_DB": relay, + "ENGRAPHIS_WEBHOOK_STATE": webhooks, + "ENGRAPHIS_BACKUP_KEY": secrets.token_hex(32), + } + with Path(os.environ["GITHUB_ENV"]).open("a", encoding="utf-8") as handle: + for name, value in values.items(): + handle.write(f"{name}={value}\n") + PY + - name: Create and restore an encrypted backup + shell: bash + run: | + artifact="$(python -m scripts.commercial_backup backup \ + --output-dir "$DRILL_ROOT/artifacts" \ + --marker "$DRILL_ROOT/backup-status.json" \ + --retention-days 2 --allow-same-device)" + test -f "$artifact" + python -m scripts.commercial_backup verify "$artifact" + python -m scripts.commercial_backup restore "$artifact" \ + --output-dir "$DRILL_ROOT/restored" + echo "DRILL_ARTIFACT=$artifact" >> "$GITHUB_ENV" + - name: Verify restored order and seat state + shell: bash + run: | + python - <<'PY' + import os + import sqlite3 + from pathlib import Path + + restored = Path(os.environ["DRILL_ROOT"]) / "restored" + with sqlite3.connect(restored / "relay.db") as conn: + assert conn.execute( + "SELECT key_id FROM issued_licenses WHERE order_id='drill-order'" + ).fetchone() == ("drill-license",) + with sqlite3.connect(restored / "webhooks.db") as conn: + assert conn.execute( + "SELECT state FROM processed WHERE webhook_id='drill-delivery'" + ).fetchone() == ("fulfilled",) + assert conn.execute( + "SELECT seats FROM subscription_seats " + "WHERE subscription_id='drill-subscription'" + ).fetchone() == (5,) + PY + - name: Prove restore never overwrites an existing directory + shell: bash + run: | + if python -m scripts.commercial_backup restore "$DRILL_ARTIFACT" \ + --output-dir "$DRILL_ROOT/restored"; then + echo "restore unexpectedly overwrote existing staging data" + exit 1 + fi diff --git a/.github/workflows/production-synthetics.yml b/.github/workflows/production-synthetics.yml new file mode 100644 index 0000000..c2d4936 --- /dev/null +++ b/.github/workflows/production-synthetics.yml @@ -0,0 +1,53 @@ +name: commercial production synthetics + +on: + schedule: + - cron: "17 * * * *" + workflow_dispatch: + +permissions: + contents: read + +jobs: + readiness: + runs-on: ubuntu-latest + timeout-minutes: 5 + env: + CUSTOMER_URL: ${{ secrets.ENGRAPHIS_CUSTOMER_URL }} + # Recurring monitoring uses a permanent, independently revocable service + # credential, never the one-time deployment ownership token. + CUSTOMER_TOKEN: ${{ secrets.ENGRAPHIS_CUSTOMER_OPS_TOKEN }} + LICENSE_URL: ${{ secrets.ENGRAPHIS_LICENSE_URL }} + OPS_TOKEN: ${{ secrets.ENGRAPHIS_VENDOR_ADMIN_TOKEN }} + steps: + - name: Require monitored endpoints + run: | + test -n "$CUSTOMER_URL" + test -n "$CUSTOMER_TOKEN" + test -n "$LICENSE_URL" + test -n "$OPS_TOKEN" + case "$CUSTOMER_URL" in https://*) ;; *) echo "customer URL must use HTTPS"; exit 1;; esac + case "$LICENSE_URL" in https://*) ;; *) echo "license URL must use HTTPS"; exit 1;; esac + - name: Customer readiness + run: >- + curl --fail --silent --show-error --max-time 20 --proto '=https' + --url "${CUSTOMER_URL%/}/api/ready" + - name: Authenticated customer storage readiness + run: | + printf 'Authorization: Bearer %s\n' "$CUSTOMER_TOKEN" | + curl --fail --silent --show-error --max-time 20 --proto '=https' \ + --header @- --url "${CUSTOMER_URL%/}/api/ops/ready" + - name: Control-plane readiness + run: >- + curl --fail --silent --show-error --max-time 20 --proto '=https' + --url "${LICENSE_URL%/}/api/ready" + - name: Authenticated dependency detail + run: | + printf 'Authorization: Bearer %s\n' "$OPS_TOKEN" | + curl --fail --silent --show-error --max-time 20 --proto '=https' \ + --header @- --url "${LICENSE_URL%/}/ops/ready" + - name: Non-mutating trial dependency synthetic + run: | + printf 'Authorization: Bearer %s\n' "$OPS_TOKEN" | + curl --fail --silent --show-error --max-time 20 --proto '=https' \ + --header @- --url "${LICENSE_URL%/}/ops/synthetic/trial" diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index fd2c2e5..8d4532b 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -11,6 +11,9 @@ on: required: false type: string +permissions: + contents: read + jobs: build: name: Build distributions @@ -22,14 +25,18 @@ jobs: steps: - name: Check out source uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7 + with: + fetch-depth: 0 - name: Set up Python uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6 with: python-version: "3.11" - - name: Install release gate - run: python -m pip install --upgrade "pip>=26.1.2" "setuptools>=83" build twine pip-audit ".[test]" + - name: Install release gate and the production dependency set + run: >- + python -m pip install --upgrade "pip>=26.1.2" "setuptools>=83" + build twine pip-audit ".[all,test]" - name: Require tag and package version to match if: github.event_name == 'push' @@ -40,8 +47,17 @@ jobs: test "$GITHUB_REF_NAME" = "v$actual" test "$expected" = "$actual" + - name: Require release tag commit to be on protected main + if: github.event_name == 'push' + shell: bash + run: | + git fetch --no-tags origin main:refs/remotes/origin/main + git merge-base --is-ancestor "$GITHUB_SHA" origin/main + - name: Full release gate run: | + python scripts/check_commercial_manifest.py + python scripts/externalize_dashboard_assets.py ruff check . python -m pytest tests/ -q python -m eval.harness --dataset eval/datasets/sample.jsonl --k 5 @@ -61,9 +77,96 @@ jobs: name: python-package-distributions path: dist/ + python-matrix: + name: Python ${{ matrix.python-version }} release gate + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + python-version: ["3.9", "3.10", "3.11", "3.12"] + steps: + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7 + - uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6 + with: + python-version: ${{ matrix.python-version }} + - name: Install version-appropriate gate + shell: bash + run: | + python -m pip install --upgrade pip + if [ "${{ matrix.python-version }}" = "3.9" ]; then + python -m pip install numpy pytest ruff + else + python -m pip install -e ".[test]" + fi + - name: Unit, lint, and retrieval gates + run: | + ruff check . + python -m pytest tests/ -q + python -m eval.harness --dataset eval/datasets/sample.jsonl --k 5 + python -m eval.harness --dataset eval/datasets/codemem.jsonl --k 5 + python -m eval.ablation + + browser-accessibility: + name: Browser accessibility release gate + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7 + - uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6 + with: + python-version: "3.11" + - uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0 + with: + node-version: "24" + cache: npm + - name: Install browser gate + run: | + python -m pip install --upgrade pip + python -m pip install -e ".[test]" + npm ci + npx playwright install --with-deps chromium + - name: Playwright desktop/mobile, keyboard, CSP, console, and axe checks + run: npm run test:e2e + + docker-smoke: + name: Production image release gate + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7 + - name: Build production image + run: docker build -t engraphis:release . + - name: Verify production image OCR runtime + run: >- + docker run --rm --entrypoint sh engraphis:release -c + 'python -c "import PIL, pytesseract" && command -v tesseract >/dev/null && + tesseract --version | head -n 1' + - name: Audit production image dependencies + run: >- + docker run --rm --entrypoint sh engraphis:release -c + 'python -m pip install --no-cache-dir pip-audit && + python -m pip_audit --local' + - name: Run customer-mode readiness smoke + shell: bash + run: | + docker run -d --name engraphis-release -p 8700:8700 \ + -e ENGRAPHIS_EMBED_MODEL= \ + -e ENGRAPHIS_LOOP_INTERVAL=0 \ + -e ENGRAPHIS_HOST=0.0.0.0 \ + engraphis:release + for i in $(seq 1 60); do + if curl -fsS http://127.0.0.1:8700/api/ready; then + exit 0 + fi + sleep 1 + done + docker logs engraphis-release + exit 1 + - name: Teardown + if: always() + run: docker rm -f engraphis-release || true + publish: name: Publish to PyPI - needs: build + needs: [build, python-matrix, browser-accessibility, docker-smoke] # Manual dispatch is intentionally build/check-only. Publication requires a pushed # semver tag, whose value was matched to pyproject.toml in the build job above. if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v') @@ -80,7 +183,7 @@ jobs: path: dist/ - name: Publish distributions to PyPI - uses: pypa/gh-action-pypi-publish@cef221092ed1bacb1cc03d23a2d87d1d172e277b # release/v1 + uses: pypa/gh-action-pypi-publish@cef221092ed1bacb1cc03d23a2d87d1d172e277b # v1.14.0 github-release: name: Publish GitHub Release diff --git a/.gitignore b/.gitignore index c33d6b4..e5f2428 100644 --- a/.gitignore +++ b/.gitignore @@ -25,6 +25,10 @@ build/ /engraphis-[0-9]*/ .pytest_cache/ .ruff_cache/ +node_modules/ +.playwright/ +playwright-report/ +test-results/ models_cache/ .secrets/ internal/ diff --git a/CHANGELOG.md b/CHANGELOG.md index b01a0ff..9d0b450 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,14 +7,79 @@ All notable changes to Engraphis are documented here. Format loosely follows ### Added +- A search-first Analytical Galaxy Graph Explorer backed by canonical, evidence-weighted + scenes; deterministic hierarchical system gravity; compact edge backbones and aggregate + bridges; exact-ID evidence/history inspection; strongest-evidence paths; synchronized + accessible tables; responsive list-first mobile behavior; saved/shareable scene state; and + local PNG, JSON, and CSV exports. +- Additive schema-v4 canonical identity and bi-temporal edge-support records, deterministic + `/api/graph/scene`, `/api/graph/suggest`, `/api/graph/entities/{canonical_id}`, and + `/api/graph/path` contracts with repository, memory-type, evidence-time, historical, and + weak-co-occurrence filtering, plus a persisted graph-index job with dry-run, progress, + cancellation, generation state, bounded error reporting, audit records, and tamper-evident + receipts. +- Galaxy Explorer: a new Simple view (the default) renders a clean glowing galactic core with + soft bloom, plus an Advanced view toggle that keeps the fully decorated explorer available. + Default galaxy rotation is now ~10x slower for readability, and a new Freeze control + pauses/resumes the live rotation. + +### Changed + +- Graph rendering now uses pinned, locally bundled Sigma 3.0.3 and Graphology 0.26 assets, + a CSP-safe TypeScript worker, evidence-derived rest lengths, and bounded level-of-detail. + The legacy API and ForceGraph renderer remain available for one compatibility release via + `ENGRAPHIS_GRAPH_UI_V2=0`; the analytical explorer is the validated default. +- Graph GET requests are strictly read-only. While an explicit mutating index job runs, graph + reads return a rebuilding conflict instead of mixing old and partially derived metrics. + +## [1.0.0] - 2026-07-19 + +First commercial GA release for Pro and Team. + +### Added + - The Knowledge Graph can color nodes by deterministic communities, entity type, or connection count. The selected mode persists and the legend follows the active scale. +- Vendor signer rotation now has a dry-run-first bulk reissue command that requires exact + active-registry coverage, preserves every signed entitlement claim, keeps source keys + valid during dual verification, records an atomic migration audit, and enforces the + 30-day grace period before source-key retirement. +- Isolated `customer` and `vendor` production service modes, with an authenticated, + secret-free commercial readiness surface. +- Deployment-bound Pro/Team trial claims with scanner-safe confirmation, automatic + server-to-server activation, replay/abuse protection, and recovery after a closed tab. +- Atomic 72-hour Team invitations with resend, revoke, seat reservations, recipient-chosen + passwords, and immediate session/token invalidation for disabled users. +- Hashed, scoped, 90-day per-user tokens for agent and sync access, including viewer + read-only sync. +- Durable transactional-email outbox, verified Resend delivery events, redacted operations + state, bounded retries, and bounce/backlog readiness gates. +- Canonical commercial product manifest, exact Polar product mapping, Railway customer + template descriptor, encrypted off-volume backup/restore tooling, and production + synthetics. + +### Changed + +- Paid licenses default to `https://license.engraphis.com`; the former license path is a + 90-day compatibility surface. +- The dashboard is auth-first when hosted, uses accessible dialogs instead of browser + prompts, and serves external same-origin assets under a CSP with no `unsafe-inline`. +- The three-day no-card application trial is the only trial mechanism; Polar is paid + monthly/annual checkout only. ### Fixed -- GitHub Release publication now supplies explicit repository context to `gh` in jobs - without a checkout. A workflow-dispatch repair path can reuse the tagged run's - validated distribution artifact if PyPI succeeds before release creation fails. +- GitHub Release publication supplies explicit repository context to `gh` in jobs without + a checkout. A workflow-dispatch repair path can reuse the tagged run's validated + distribution artifact if PyPI succeeds before release creation fails. + +### Security + +- The browser and invitation emails never receive the account-wide Team license key. +- Trial invitation URLs are restricted to the dashboard origin bound into the initiating + deployment claim. +- Production issuance rejects unknown Polar products and wrong organizations; signer + readiness remains fail-closed until the offline rotation ceremony is approved. ## [0.9.9] - 2026-07-18 diff --git a/Dockerfile b/Dockerfile index b103cf2..6e5dce2 100644 --- a/Dockerfile +++ b/Dockerfile @@ -10,6 +10,7 @@ FROM python:3.11-slim AS base ENV PYTHONUNBUFFERED=1 \ PYTHONDONTWRITEBYTECODE=1 \ PIP_NO_CACHE_DIR=1 \ + ENGRAPHIS_SERVICE_MODE=customer \ ENGRAPHIS_PORT=8700 \ ENGRAPHIS_DB_PATH=/data/engraphis.db \ # Cache the sentence-transformers model on the persistent /data volume so it downloads @@ -26,7 +27,7 @@ WORKDIR /app # gosu lets the entrypoint drop from root to the non-root app user after fixing volume # permissions (see docker-entrypoint.sh). Installed here for good layer caching. RUN apt-get update \ - && apt-get install -y --no-install-recommends gosu \ + && apt-get install -y --no-install-recommends gosu tesseract-ocr \ && rm -rf /var/lib/apt/lists/* # Install dependencies first for better layer caching. @@ -47,15 +48,15 @@ COPY docker-entrypoint.sh /usr/local/bin/docker-entrypoint.sh RUN chmod +x /usr/local/bin/docker-entrypoint.sh EXPOSE 8700 -# /api/health is served by BOTH entrypoints (engraphis-server AND engraphis-dashboard), -# so this check is correct regardless of which one a service runs. +# /api/ready verifies that the configured customer service can actually serve traffic; +# Railway uses the same endpoint, so a process-only health signal cannot mask a bad mode. # start-period is generous: the first cold boot downloads the embedding model (cached to # the /data volume via HF_HOME thereafter). The entrypoint's default bind (`::` dual-stack # where available, else 0.0.0.0) answers loopback IPv4 probes like this one AND platform # IPv6 routing; the check also honors $PORT if the platform overrides it — matching # scripts/start_dashboard.py, which prefers $PORT over ENGRAPHIS_PORT for the bind. HEALTHCHECK --interval=30s --timeout=5s --start-period=300s --retries=3 \ - CMD python -c "import os,urllib.request,sys; p=os.environ.get('PORT') or os.environ.get('ENGRAPHIS_PORT','8700'); sys.exit(0 if urllib.request.urlopen('http://127.0.0.1:%s/api/health' % p).status==200 else 1)" + CMD python -c "import os,urllib.request,sys; p=os.environ.get('PORT') or os.environ.get('ENGRAPHIS_PORT','8700'); sys.exit(0 if urllib.request.urlopen('http://127.0.0.1:%s/api/ready' % p).status==200 else 1)" # The entrypoint fixes volume ownership then drops to the non-root `engraphis` user before # running the CMD (or any Railway/compose start-command override, which becomes its args). diff --git a/NOTICE b/NOTICE index dd3aff0..4342ce5 100644 --- a/NOTICE +++ b/NOTICE @@ -10,8 +10,7 @@ You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 "Engraphis" and the Engraphis logo are trademarks of the Engraphis project. -Use of the trademarks is subject to the Engraphis Trademark Policy; the -Apache-2.0 license does not grant trademark rights (see LICENSE, section 6). +The Apache-2.0 license does not grant trademark rights (see LICENSE, section 6). Third-party browser assets distributed with this product: @@ -19,7 +18,11 @@ Third-party browser assets distributed with this product: engraphis/static/vendor/d3.LICENSE. - Marked 12.0.2, Copyright MarkedJS, Christopher Jeffrey, and Markdown's contributors, MIT and BSD-style licenses. See engraphis/static/vendor/marked.LICENSE. -- force-graph, Copyright 2018 Vasco Asturiano, MIT license. See +- force-graph 1.51.4, Copyright 2018 Vasco Asturiano, MIT license. See engraphis/static/vendor/force-graph.LICENSE. +- Sigma.js 3.0.3, Graphology 0.26.0, graphology-utils, and events, distributed + under their MIT licenses. The exact dependency hashes and combined license + texts are shipped in engraphis/static/vendor/galaxy-dependencies.json and + engraphis/static/vendor/galaxy-vendor.LICENSE.txt. - DOMPurify 3.4.11, Copyright Cure53 and contributors, distributed under the Apache License 2.0 option stated in its embedded license header; see LICENSE. diff --git a/README.md b/README.md index 3d9156d..b5c0c0e 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # Engraphis -[![Version](https://img.shields.io/badge/version-0.9.9-blue.svg)](https://github.com/Coding-Dev-Tools/engraphis) +[![Version](https://img.shields.io/badge/version-1.0.0-blue.svg)](https://github.com/Coding-Dev-Tools/engraphis) [![License](https://img.shields.io/badge/license-Apache--2.0-green.svg)](https://github.com/Coding-Dev-Tools/engraphis/blob/main/LICENSE) [![Buy Me a Coffee](https://img.shields.io/badge/Buy%20Me%20a%20Coffee-support-yellow?style=for-the-badge&logo=buy-me-a-coffee)](https://buymeacoffee.com/Jaixii) @@ -22,11 +22,11 @@ https://discord.com/invite/Wfr2ejBmY --- ->Open-Source users: Remember to Update regularly! Improvements and fixes twice a day. Invite your friends! +> Open-source users: update regularly for the latest fixes and improvements. > -> **Beta:** the **Team** layer (multi-user dashboard, seats, roles, audit log, team invite -> emails, cloud sync relay) is **early-access beta** — expect rough edges and breaking -> changes before it stabilizes. The single-user engine, dashboard, and MCP server are stable. +> **Version 1.0:** the core engine, dashboard, MCP server, Pro features, and Team layer +> are generally available. Team includes multi-user authentication, roles, seat management, +> invitation and password-reset flows, audit history, and scoped cloud-sync tokens. ## The WebUI — one command, local-first @@ -59,8 +59,8 @@ a color palette and layout preset; or change the colors used for each type of no | **Consolidate** | Run a consolidation sweep on demand — see what got distilled and what got pruned | | **Automation** *(Pro)* | Scheduled consolidation + retention policies on autopilot — plus **auto-dreaming**: a background consolidation + cross-cluster inference loop that fires when the store has accumulated enough new memories *and* gone idle. Configurable from the dashboard (cadence, dream trigger, idle threshold, inference toggle) or the `GET/POST /api/automation` API, and via `scripts/auto_maintain` for cron / Task Scheduler | | **Workspaces** | Create, rename, describe, copy, merge, and delete workspaces; import files & folders; drag-and-drop upload | -| **Team** *(beta)* | Multi-user access with PBKDF2 logins, password reset, admin / member / viewer roles, seat management, and team audit log (Team) — **early-access beta** | -| **Settings** | License activation (Pro/Team), cloud sync, LLM provider setup/test, Agent Connect token management, appearance, and engine/store info | +| **Team** *(Team plan)* | Multi-user access with PBKDF2 logins, password reset, admin / member / viewer roles, seat management, scoped agent/sync tokens, and team audit history | +| **Settings** | License activation (Pro/Team), cloud sync, LLM provider setup/test, a live structured-extraction switch and activity viewer, Agent Connect token management, appearance, and engine/store info | The dashboard is powered by the v2 engine — the same `MemoryService` that backs the MCP server and the Python library. What you see in the UI is what your agents get. @@ -119,9 +119,43 @@ or structured consolidation. - **Scoped** — `workspace → repo → session` hierarchy. - **Encryption at rest** — optional SQLCipher (AES-256) whole-database encryption via `ENGRAPHIS_DB_KEY`. No plaintext fallback when a key is set. - **Cloud sync** — cross-device and cross-team memory sync with deterministic CRDT merge (folder transport for self-hosting, managed relay for zero-setup). One-click "Sync now" or automatic cadence in the dashboard. -- **Import & ingest** — local documents/code/DOCX plus optional PDF OCR, image OCR, +- **Import & ingest** — local documents/code/DOCX plus optional PDF text extraction, image OCR, audio/video transcription, and live PostgreSQL schema introspection. +### Connect an LLM and inspect exactly what it changed + +The memory engine, embeddings, conflict resolution, and normal recall remain local and do not +need an LLM. Connecting one adds optional structured extraction, cited prose synthesis, +structured consolidation, and retention supervision. + +Open **Settings → Connect an LLM**, configure the provider/model/key in `.env`, restart, and +click **Test connection**. When that live test succeeds, Engraphis automatically turns +`llm_structured` extraction on unless you previously chose **Turn extraction off**. The adjacent +button changes the live engine immediately and persists both the extractor mode and your +automatic-extraction preference to the project `.env` when it is writable; no restart is required +for the running dashboard. Explicit deployment environment variables remain authoritative after +a restart. Turning extraction off does not disconnect the provider, so explicitly requested +synthesis or consolidation can still use it. + +Structured extraction applies to `engraphis_ingest` and to file/folder imports where **Derive +discrete facts with the configured extractor** is explicitly selected. It does not silently send +ordinary `engraphis_remember` writes, existing memories, or every imported file to the provider. +For each successful source, the validated output becomes one or more individually recallable +memories with typed facts, keywords, entities, and relations. A provider/schema failure falls +back to deterministic local chunking so the source write is not lost. + +Click **View LLM memory activity** to open a workspace-scoped window listing memories the LLM +extracted, structurally consolidated, or retention-classified. Extraction entries show the +provider/model when recorded, fact position within the source batch, extracted entities and +relations, and a link to the resulting memory. The activity API and window expose stored outcomes +only—never the API key, prompt, original provider payload, or raw response. Older structured +memories created before provider/model activity metadata was introduced still appear as legacy +structured-extraction entries. + +> Privacy boundary: text sent through structured extraction leaves the local process and is +> handled under the selected provider/model's data terms. Keep extraction off for material that +> must remain entirely local, or use the offline `chunk` extractor instead. + --- ## Why it wins @@ -143,20 +177,23 @@ or structured consolidation. ## Host on Railway (Pro solo or Team) -The dashboard ships as a Docker image that defaults to the v2 dashboard (multi-user auth, -roles, seats, cloud-license revocation). Deploy one instance on Railway and access your -memories from any browser. Two supported paths: +The official template runs the shared image in `customer` mode, mounts `/data`, checks +`/api/ready`, and generates a 48-character deployment token. Deploy one instance and use the +hosted wizard: verify deployment ownership → choose Pro or Team → confirm email → automatic +activation → create the first admin. The signed key never appears in the browser and the +service does not redeploy during activation. - **Pro solo** — a Pro member deploys a single-admin cloud instance: browser dashboard (analytics, automation, export) + a self-hosted sync relay. Activate the same Pro key on each local instance, set `ENGRAPHIS_RELAY_URL` on both the hosted service and local - instances to **your Railway deployment URL**, then enable auto-sync (or run **Sync now**). - Keep `ENGRAPHIS_CLOUD_URL=https://team.engraphis.com` on the hosted service so trials, - revocable leases, and fallback invite delivery continue to use the managed issuer. - One admin, no member seats. + instances to **your Railway deployment URL**, then connect with an expiring scoped sync + token and enable auto-sync (or run **Sync now**). Keep + `ENGRAPHIS_CLOUD_URL=https://license.engraphis.com` for trials and license leases. One + admin, no member seats. - **Team admin** — a Team administrator deploys one instance and invites members (email + - password + role). Members sign in at your URL and connect their agents over HTTP/MCP — - no local install for members. + role). The recipient chooses their own password from a 72-hour invitation. Members sign + in at your URL and create scoped agent/sync tokens; member invitations never contain the + purchaser's account-wide Team license key. See [`docs/HOSTING_RAILWAY.md`](docs/HOSTING_RAILWAY.md) for the 5-minute guide covering both paths (volume, custom domain, activate Pro/Team, create the first admin, invite @@ -164,23 +201,23 @@ members, and connect agents). **→ [Deploy on Railway (5-minute guide)](docs/HOSTING_RAILWAY.md)** -> Deploying from this repo's `Dockerfile` takes about five minutes: create the service, -> attach a persistent `/data` volume (so activated keys + memories survive redeploys), -> and set `ENGRAPHIS_FORWARDED_ALLOW_IPS=*`, `ENGRAPHIS_DASHBOARD_URL`, and a strong -> `ENGRAPHIS_API_TOKEN` for hosted bootstrap. `railway.json` -> in this repo already supplies the build and healthcheck config. The hosting guide walks -> through each step with the exact values. +> Until the public Railway template code is published, deploy from this repository and +> apply [`deploy/railway-template.json`](deploy/railway-template.json) exactly: persistent +> `/data`, customer service mode, generated public-domain references, and a unique +> `ENGRAPHIS_DEPLOYMENT_TOKEN`. `railway.json` supplies the build and `/api/ready` check. > > *(A one-click "Deploy on Railway" button previously sat here pointing at > `railway.app/new?template=`. Railway ignores that parameter — > `railway.json` is per-service build config, not a publishable template — so the button -> only ever landed people on a generic project chooser. It was removed on 2026-07-18 -> rather than left looking functional. `docs/RAILWAY_TEMPLATE.md` contains the spec to -> publish a real template; once published, its button can go back here.)* - -Hosted **Agent Connect** tokens are per-user, shown only once, and stored only as SHA-256 -digests. Roles are rechecked on every HTTP/MCP call; disabling a member or resetting their -password permanently revokes existing agent tokens. The hosted `/mcp` endpoint exposes the same +> only ever landed people on a generic project chooser. It remains removed until the +> source descriptor is published through Railway and passes the logged-out acceptance +> suite. `docs/RAILWAY_TEMPLATE.md` is the publication runbook.)* + +Hosted **Agent Connect** tokens are per-user and shown only once; the server stores only +SHA-256 digests. A local sync device necessarily retains its raw bearer in an owner-only +credential file so it can authenticate future rounds. Roles are rechecked on every HTTP/MCP +call; disabling a member or resetting their password permanently revokes existing agent tokens. +The hosted `/mcp` endpoint exposes the same 28-tool service as local `engraphis-mcp`. See [the Agent Connect guide](docs/AGENT_CONNECT.md). ## Install @@ -196,6 +233,10 @@ pip install "engraphis[encryption]" # SQLCipher encryption-at-rest extra pip install engraphis # core library — numpy only, fully offline ``` +The official Docker image includes the local Tesseract executable for image OCR. Outside +Docker, the `documents` extra installs its Python bindings; install Tesseract through your +operating system as well if you enable image OCR. + The NumPy-only core library supports Python 3.9+. Current patched releases of the WebUI stack, MCP SDK, and image parser require Python 3.10+, so use Python 3.10 or newer for the `server`, `mcp`, `documents`, or `all` installation paths. @@ -355,15 +396,14 @@ pinned. The full multi-predecessor chain remains visible through inspection, Why The core engine, single-user dashboard, standalone MCP server, and governance tools are free and Apache-2.0, permanently. Paid Pro/Team keys are **server-authoritative**: the vendor signature is checked locally, then the key must hold a current machine-bound lease -from the configured/vendor relay. Revoked, expired, or seat-exceeded keys fail closed; +from the configured license service. Revoked, expired, or seat-exceeded keys fail closed; an unexpired lease provides bounded grace for transient network failures. **Pro is $10/mo ($100/yr), Team is $20/seat/mo ($200/seat/yr)**, and the dashboard offers a **3-day server-issued Pro or Team trial** after email confirmation — no card required. -> **Team is early-access beta.** Multi-user logins, seats, roles, the team audit log, -> team invite emails, and the cloud-sync relay are all in active development — expect -> rough edges and breaking changes. Pro (single-user paid features) is stable. Free is -> stable. +Pro and Team are GA in v1.0.0. Cloud sync is opt-in and transported over HTTPS; Engraphis +does not advertise end-to-end encryption. Paid entitlements require online lease renewal, +while the Free core remains fully local and offline-capable. | | Free (available now) | Pro — $10/mo or $100/yr | Team — $20/seat/mo or $200/seat/yr | |---|---|---|---| @@ -377,9 +417,10 @@ server-issued Pro or Team trial** after email confirmation — no card required. | Automated maintenance: scheduled consolidation + retention policies + **auto-dreaming** | | ✓ | ✓ | | Signed compliance export (checksummed bi-temporal bundle) | | ✓ | ✓ | | Priority support | | ✓ | ✓ | -| Multi-user dashboard: logins, roles, seat management *(beta)* | | | ✓ | -| Team audit log + CSV export *(beta)* | | | ✓ | -| Team invite emails (vendor relay, zero email setup) *(beta)* | | | ✓ | +| Multi-user dashboard: invitations, logins, roles, seat management | | | ✓ | +| Team audit log + CSV export | | | ✓ | +| 72-hour pending invitations (resend/revoke) | | | ✓ | +| Scoped, expiring per-user agent and sync tokens | | | ✓ | --- @@ -451,8 +492,9 @@ tier, across a group — without giving up local-first ownership. It ships two t - **Folder transport** — any shared directory (Dropbox, iCloud, Syncthing, a git repo, a mounted drive). Zero infrastructure. -- **Managed relay** — HTTPS against the Engraphis relay, authenticated by your license key. - One-click in the dashboard or `python -m scripts.sync --relay`. +- **Managed relay** — HTTPS against the customer service, authenticated by an expiring, + revocable per-user token. One-click in the dashboard or + `python -m scripts.sync --relay --relay-token `; viewers use `--read-only`. Sync is a **state-based CRDT**: deterministic merge, no conflict copies, no data loss. Every field resolves by a commutative, idempotent rule so `merge(A, B) == merge(B, A)`. @@ -544,8 +586,9 @@ Drag-and-drop or server-side import, role-gated and bounded: the bundled eval: `python -m eval.chunking_eval --dataset eval/datasets/longdoc.jsonl --k 5` (whole-file vs. chunked, same recall pipeline, offline). - **Structured LLM extraction** — `ENGRAPHIS_EXTRACTOR=llm_structured` validates typed - facts, entities, relations, keywords, and confidence before storage. Its preserved - entity/relation metadata feeds the knowledge graph automatically. + facts, entities, relations, and keywords before storage. Its preserved entity/relation + metadata feeds the knowledge graph automatically. A successful dashboard connection test + enables this mode by default; the Settings switch can disable or re-enable it immediately. Files imported through the dashboard or `import_folder()` are marked **untrusted** by default; MCP ingest remains an authenticated agent write. @@ -596,7 +639,9 @@ All via environment (or `.env`): | `ENGRAPHIS_DB_PATH` | Source: `/engraphis.db`; installed: platform user-data directory | SQLite database file. Installed defaults are `%LOCALAPPDATA%\engraphis\engraphis.db` (Windows), `~/Library/Application Support/engraphis/engraphis.db` (macOS), and `$XDG_DATA_HOME/engraphis/engraphis.db` or `~/.local/share/engraphis/engraphis.db` (Linux). The environment variable overrides every default. | | `ENGRAPHIS_HOST` | `127.0.0.1` | Server bind address | | `ENGRAPHIS_PORT` | `8700` | Dashboard port | -| `ENGRAPHIS_API_TOKEN` | — | Protects REST API routes with `Authorization: Bearer ` and proves deployment ownership during hosted trial and remote first-admin setup; leave unset only for loopback-only use | +| `ENGRAPHIS_SERVICE_MODE` | `combined` | `customer` for hosted dashboards, `vendor` for the isolated license control plane, and `combined` for development only | +| `ENGRAPHIS_API_TOKEN` | — | Optional service-wide REST bearer credential; per-user tokens are preferred for hosted agent access | +| `ENGRAPHIS_DEPLOYMENT_TOKEN` | — | Secret ownership proof required by hosted trial activation and remote first-admin setup | | `ENGRAPHIS_CORS_ORIGINS` | loopback on `ENGRAPHIS_PORT` | Comma-separated REST CORS allow-list; defaults to `127.0.0.1` and `localhost` on the configured port | | `ENGRAPHIS_WORKSPACES` | — | Optional comma-separated server-side workspace allow-list | | `ENGRAPHIS_DB_KEY` | — | Encrypt the database at rest (SQLCipher). Or use `ENGRAPHIS_DB_KEY_FILE` | @@ -614,6 +659,7 @@ All via environment (or `.env`): | `ENGRAPHIS_LLM_MODEL` | `gpt-4o-mini` | Model name (provider-specific) | | `ENGRAPHIS_LLM_API_KEY` | — | API key for chat/synthesis, `llm` / `llm_structured` extraction, and structured consolidation | | `ENGRAPHIS_LLM_BASE_URL` | — | Base URL for openrouter / custom OpenAI-compatible endpoints | +| `ENGRAPHIS_LLM_AUTO_EXTRACT` | `1` | After a successful live connection test, automatically switch the running engine to `llm_structured`; the dashboard's extraction Off button persists `0`, and its On button restores `1` | | `ENGRAPHIS_LICENSE_KEY` | — | Pro/Team key (or `~/.engraphis/license.key`) | | `ENGRAPHIS_TEAM_MODE` | `1` | Mount hosted auth/team plumbing; any active Pro/Team license activates the login wall and first-admin setup, and existing users keep the wall active after lapse. Set `0` to disable hosted user auth for single-user mode | | `ENGRAPHIS_DASHBOARD_URL` | — | Canonical public dashboard URL used in invites, reset links, redirects, and the hosted MCP Host/Origin allow-list | @@ -622,7 +668,7 @@ All via environment (or `.env`): | `ENGRAPHIS_FORWARDED_ALLOW_IPS` | *(none)* | Proxies trusted for forwarded client/TLS headers (`*` only when the service is reachable exclusively through that proxy) | | `ENGRAPHIS_LOCAL_TRUSTED_PEERS` | *(none)* | Exact peers/CIDRs treated as local without forwarding headers; intended for the shipped loopback-published Compose bridge, not public deployments | | `ENGRAPHIS_RELAY_URL` | `https://team.engraphis.com` | Sync relay target (Pro/Team); set to a customer deployment for self-hosted sync | -| `ENGRAPHIS_CLOUD_URL` | signed key issuer, then relay URL | License/trial/invite service override; keep `https://team.engraphis.com` when `ENGRAPHIS_RELAY_URL` points at a customer-operated sync relay | +| `ENGRAPHIS_CLOUD_URL` | `https://license.engraphis.com` | License/trial/invite control plane; keep separate from a customer-operated `ENGRAPHIS_RELAY_URL` | | `ENGRAPHIS_AUTOSYNC_LOOP` | `1` | Kill switch for the in-process auto-sync loop (0 = off) | See `.env.example` for the full list including commercial/vendor, email delivery, and diff --git a/SECURITY.md b/SECURITY.md index ad389dd..8d79d65 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -40,7 +40,8 @@ DOMPurify at all render sites. Verified against payloads with `onerror` handlers - **Vendor admin separation** (`ENGRAPHIS_VENDOR_ADMIN_TOKEN`): vendor-wide license administration on the relay (`/license/v1` revoke/keys/deactivate) uses its own secret, so the per-instance service token can never revoke other customers' keys. - If the vendor token is unset, those routes fail closed; there is no fallback. + It must be 32–4096 printable ASCII characters. If it is absent or weaker, those + routes fail closed; there is no fallback. - **Login throttling**: per-email lockout plus a per-source-IP failure window (cross-email credential-stuffing); lockouts are typed and mapped to HTTP 429. Behind a reverse proxy, set `ENGRAPHIS_FORWARDED_ALLOW_IPS` to the proxy address so @@ -142,8 +143,9 @@ second issuer. mean deriving it from the client-supplied `Host` header, which allowed an attacker to have a victim emailed a vendor-signed confirmation link pointing at attacker infrastructure (fixed 2026-07-18). Unset ⇒ `/license/v1/start-trial` answers 503. -- `ENGRAPHIS_VENDOR_ADMIN_TOKEN` is **required** for the vendor admin routes and no - longer falls back to `ENGRAPHIS_API_TOKEN`. Unset ⇒ those routes are disabled. +- `ENGRAPHIS_VENDOR_ADMIN_TOKEN` is **required** for the vendor admin routes, must be + 32–4096 printable ASCII characters, and no longer falls back to + `ENGRAPHIS_API_TOKEN`. Absent/invalid ⇒ those routes are disabled. - Relay storage is capped per account (`ENGRAPHIS_RELAY_MAX_ACCOUNT_BYTES`, `ENGRAPHIS_RELAY_MAX_WORKSPACES_PER_ACCOUNT`), not only per workspace: workspace ids are caller-supplied, and the relay DB shares a volume with the revocation registry. diff --git a/_measure_d3.mjs b/_measure_d3.mjs new file mode 100644 index 0000000..c153578 --- /dev/null +++ b/_measure_d3.mjs @@ -0,0 +1,51 @@ +import { readFile } from "node:fs/promises"; +import { chromium } from "playwright"; +const root = "/sessions/relaxed-stoic-hamilton/mnt/engraphis/engraphis/static"; +const d3src = await readFile(`${root}/vendor/d3.min.js`, "utf8"); +const browser = await chromium.launch({ headless: true }); +const page = await browser.newPage(); +await page.addScriptTag({ content: d3src }); +const res = await page.evaluate(() => { + // synthetic galaxy: 10 communities, hub + 7 members each, intra links + ring cross-links + const nodes = [], links = []; + const C = 10, M = 8; + for (let c = 0; c < C; c++) for (let m = 0; m < M; m++) { + nodes.push({ id: `n${c}_${m}`, community: c, degree: m===0?M:1, val: m===0?9:2, radius: m===0?6:3, + x: (Math.random()-.5)*800, y: (Math.random()-.5)*800 }); + if (m>0) links.push({ source: `n${c}_0`, target: `n${c}_${m}` }); + } + for (let c=1;c{ns=n}; return f; } + + const repel=42, link=20; + function run(cfg){ + // fresh positions + nodes.forEach(n=>{n.x=(Math.random()-.5)*800;n.y=(Math.random()-.5)*800;n.vx=0;n.vy=0;}); + const sim=window.d3.forceSimulation(nodes) + .force('charge',window.d3.forceManyBody().strength(-repel)) + .force('link',window.d3.forceLink(links).id(d=>d.id).distance(link)) + .force('x',window.d3.forceX(0).strength(cfg.center)) + .force('y',window.d3.forceY(0).strength(cfg.center)) + .force('cluster',clusterForce(cfg.cluster)) + .force('collide',window.d3.forceCollide(n=>n.radius+1.5)) + .stop(); + for(let i=0;i<400;i++) sim.tick(); + const ds=nodes.map(n=>Math.hypot(n.x,n.y)).sort((a,b)=>a-b); + return { mean:+(ds.reduce((a,b)=>a+b,0)/ds.length).toFixed(1), max:+ds.at(-1).toFixed(1) }; + } + const out={}; + for(const grav of [10,26,40]){ + const g=grav/100; + out[grav]={ OLD:run({center:.012, cluster:Math.max(.06,g*1.7)}), + NEW:run({center:Math.max(.02,g), cluster:Math.max(.04,g*0.55)}) }; + } + return out; +}); +await browser.close(); +console.log("gravity | OLD mean(max) [center=.012 fixed] | NEW mean(max) [center=g]"); +for(const [grav,o] of Object.entries(res)) + console.log(` ${String(grav).padStart(2)} | OLD ${String(o.OLD.mean).padStart(6)} (${o.OLD.max}) | NEW ${String(o.NEW.mean).padStart(6)} (${o.NEW.max})`); diff --git a/_measure_galaxy.mjs b/_measure_galaxy.mjs new file mode 100644 index 0000000..211e30f --- /dev/null +++ b/_measure_galaxy.mjs @@ -0,0 +1,51 @@ +import { readFile } from "node:fs/promises"; +import { chromium } from "playwright"; +const staticDir = "/sessions/relaxed-stoic-hamilton/mnt/engraphis/engraphis/static"; +const source = await readFile(`${staticDir}/galaxy-explorer.js`, "utf8"); +const systemCount = 12, members = 8; +const nodes = [], communities = [], bridges = [], edges = []; +for (let s = 0; s < systemCount; s += 1) { + const sysId = `sys_${s}`, anchorId = `n_${s}_0`; + communities.push({ id: sysId, anchor_id: anchorId, mass: 40 - s, radius: 36, member_count: members }); + for (let m = 0; m < members; m += 1) { + nodes.push({ canonical_id: `n_${s}_${m}`, label: `${s}.${m}`, community_id: sysId, + anchor_role: s === 0 && m === 0 ? "global" : m === 0 ? "community" : "none", + mass_score: m === 0 ? 0.9 : 0.2, gravity_mass: m === 0 ? 9 : 2, visual_radius: m === 0 ? 8 : 3 }); + if (m > 0) edges.push({ id: `e_${s}_${m}`, source: anchorId, target: `n_${s}_${m}`, relation: "supports", layer: "semantic", strength: 0.8, support_count: 2, visible_by_default: true }); + } + if (s > 0) bridges.push({ id: `b_${s}`, source_community: `sys_${s-1}`, target_community: sysId, strength: 0.5, physics_strength: 0.5, support_count: 4, edge_count: 2 }); +} +const scene = { meta: { level: "complete", complete_scene: true, scene_hash: "measure", layout_seed: 7 }, nodes, edges, communities, community_bridges: bridges }; +const browser = await chromium.launch({ headless: true }); +const page = await browser.newPage({ viewport: { width: 1000, height: 700 } }); +await page.addStyleTag({ path: `${staticDir}/galaxy-explorer.css` }); +const out = await page.evaluate(async ({ bundle, scene }) => { + const url = URL.createObjectURL(new Blob([bundle], { type: "text/javascript" })); + const galaxy = await import(url); + const container = document.createElement("main"); + container.style.width = "1000px"; container.style.height = "700px"; + document.body.appendChild(container); + const controller = galaxy.createGalaxyGraph(container, { forceSynchronous: true }); + const layout = await controller.setScene(scene); + const blackId = layout.blackHoleId; + const wait = (ms) => new Promise((r) => setTimeout(r, ms)); + const meanDist = () => { + const c = controller.getNodeGraphPosition(blackId); + const ds = scene.nodes.filter((n) => n.canonical_id !== blackId) + .map((n) => { const p = controller.getNodeGraphPosition(n.canonical_id); return Math.hypot(p.x - c.x, p.y - c.y); }); + ds.sort((a, b) => a - b); + return { mean: ds.reduce((a, b) => a + b, 0) / ds.length, max: ds.at(-1), median: ds[Math.floor(ds.length/2)] }; + }; + const run = async (controls) => { await controller.relayout(scene, controls); controller.setPhysicsEnabled(true); await wait(3000); controller.setPhysicsEnabled(false); return meanDist(); }; + const gravity = {}; + for (const p of [0.65, 1.0, 1.35]) gravity[p] = await run({ systemPull: p, orbitalCompactness: 1, linkCohesion: 1, layoutPreset: "compact" }); + const presets = {}; + for (const preset of ["compact", "spacious", "islands", "radial"]) presets[preset] = await run({ systemPull: 1.0, orbitalCompactness: 1, linkCohesion: 1, layoutPreset: preset }); + controller.destroy(); URL.revokeObjectURL(url); + return { gravity, presets }; +}, { bundle: source, scene }); +await browser.close(); +console.log("=== GRAVITY sweep (systemPull -> node distance from center) ==="); +for (const [p, d] of Object.entries(out.gravity)) console.log(` systemPull=${p}: mean=${d.mean.toFixed(1)} median=${d.median.toFixed(1)} max=${d.max.toFixed(1)}`); +console.log("=== PRESET sweep (layoutPreset -> distance from center, pull=1.0) ==="); +for (const [k, d] of Object.entries(out.presets)) console.log(` ${k.padEnd(9)}: mean=${d.mean.toFixed(1)} median=${d.median.toFixed(1)} max=${d.max.toFixed(1)}`); diff --git a/_smoke.mjs b/_smoke.mjs new file mode 100644 index 0000000..e69de29 diff --git a/_smoke2.mjs b/_smoke2.mjs new file mode 100644 index 0000000..e69de29 diff --git a/deploy/railway-template.json b/deploy/railway-template.json new file mode 100644 index 0000000..1c5ccc3 --- /dev/null +++ b/deploy/railway-template.json @@ -0,0 +1,61 @@ +{ + "format": "engraphis-railway-template-composer-source/v1", + "name": "Engraphis Pro or Team", + "description": "Customer-mode Engraphis dashboard with persistent memory, secure hosted trial activation, and scoped Team sync tokens.", + "source": { + "repository": "https://github.com/Coding-Dev-Tools/engraphis", + "branch": "main", + "dockerfile": "Dockerfile" + }, + "service": { + "healthcheck": "/api/ready", + "public_domain": true, + "volume": { + "mount_path": "/data" + } + }, + "variables": { + "ENGRAPHIS_SERVICE_MODE": { + "value": "customer", + "required": true + }, + "ENGRAPHIS_DB_PATH": { + "value": "/data/engraphis.db", + "required": true + }, + "ENGRAPHIS_STATE_DIR": { + "value": "/data/.engraphis", + "required": true + }, + "ENGRAPHIS_TEAM_MODE": { + "value": "1", + "required": true + }, + "ENGRAPHIS_JSON_LOGS": { + "value": "1", + "required": true + }, + "ENGRAPHIS_FORWARDED_ALLOW_IPS": { + "value": "*", + "required": true + }, + "ENGRAPHIS_CLOUD_URL": { + "value": "https://license.engraphis.com", + "required": true + }, + "ENGRAPHIS_DASHBOARD_URL": { + "value": "https://${{RAILWAY_PUBLIC_DOMAIN}}", + "required": true + }, + "ENGRAPHIS_RELAY_URL": { + "value": "https://${{RAILWAY_PUBLIC_DOMAIN}}", + "required": true + }, + "ENGRAPHIS_DEPLOYMENT_TOKEN": { + "value": "${{ secret(48) }}", + "secret": true, + "prompt": "Railway generates this unique ownership secret. Copy it into the hosted onboarding wizard, then seal it after the first admin is created.", + "required": true + } + } +} diff --git a/docs/AGENT_CONNECT.md b/docs/AGENT_CONNECT.md index 30944e0..311aa48 100644 --- a/docs/AGENT_CONNECT.md +++ b/docs/AGENT_CONNECT.md @@ -11,8 +11,8 @@ of each member running a local MCP server + syncing, one admin hosts a single in > **Pro solo?** Agent-connect (direct writes to a cloud instance via `/api/remember` or > `/mcp`) requires a **Team** license. If you're a Pro member hosting on Railway, your -> agents run locally and sync to your Railway instance via cloud sync — activate the same -> Pro key on each local instance, then set `ENGRAPHIS_RELAY_URL` to your Railway deployment URL. See +> agents run locally and sync through your Railway instance with a scoped user token; +> never distribute the account license key. Set `ENGRAPHIS_RELAY_URL` to your deployment URL. See > [HOSTING_RAILWAY.md](./HOSTING_RAILWAY.md) for the Pro solo path. ## How it works @@ -20,10 +20,11 @@ of each member running a local MCP server + syncing, one admin hosts a single in 1. **Admin** deploys one instance and activates Team before first-admin setup: load a purchased key with `ENGRAPHIS_LICENSE_KEY`, or start a Team trial and open its emailed confirmation link. -2. **Admin** creates the first account, then invites members (email + password + role). - Each active user consumes a seat. -3. **Member** signs in at the dashboard URL (e.g. `https://memory.example.com/`) with - email + password — no key and no local install. If the license later lapses, the +2. **Admin** creates the first account, then sends a pending invitation with email and + role. It reserves a seat for 72 hours; the admin never chooses the recipient's password. +3. **Member** opens the scanner-safe invitation and sets their own password. Acceptance + atomically creates the user. Expiry or revocation releases the reserved seat. The + member then signs in at the dashboard URL — no key and no local install. If the license later lapses, the authentication wall remains in place and existing users can still sign in. 4. **Member** opens **Settings → Connect your agent → Create token** and copies the one-time bearer token. @@ -33,7 +34,10 @@ of each member running a local MCP server + syncing, one admin hosts a single in Agents authenticate with a **per-user bearer token** (`Authorization: Bearer `), minted from the dashboard. The token is bound to the member: their role, their personal -folders, and their seat. Disabling the member instantly invalidates their token. +folders, and their seat. New tokens expire after 90 days; the hosted server stores only +their hashes, and each token carries explicit scopes. A client that saves a token for +recurring sync must retain the raw bearer locally as an owner-only credential. Disabling +or deleting the member instantly invalidates every session and token. Viewer tokens can call read routes, but write/governance routes return `403`; `/api/remember` and `/mcp` require the `member` or `admin` role. @@ -42,7 +46,7 @@ Token management (requires a browser session): | Method | Path | Purpose | |---|---|---| -| `POST` | `/api/auth/token` `{label}` | Mint a token (raw token returned **once**) | +| `POST` | `/api/auth/token` `{label, scopes?}` | Mint a 90-day token (raw token returned **once**) | | `GET` | `/api/auth/tokens` | List your tokens (never includes the raw token) | | `DELETE` | `/api/auth/token/{id}` | Revoke one of your tokens | | `GET` | `/api/auth/connect-info` | Verify a token + discover the API base / snippet | diff --git a/docs/COMMERCIAL_OPERATIONS.md b/docs/COMMERCIAL_OPERATIONS.md new file mode 100644 index 0000000..19b00a3 --- /dev/null +++ b/docs/COMMERCIAL_OPERATIONS.md @@ -0,0 +1,248 @@ +# Commercial operations runbook (v1.0) + +## Production topology + +`team.engraphis.com` runs `ENGRAPHIS_SERVICE_MODE=customer`: dashboard, memory API, +authentication, invitations, and customer sync only. `license.engraphis.com` runs +`ENGRAPHIS_SERVICE_MODE=vendor`: issuance, leases, deployment trials, Polar webhooks, +transactional email, and authenticated operations checks. New signed keys use +`https://license.engraphis.com`; the old `team.engraphis.com/license/v1/*` proxy is retained +for the 90-day compatibility window, with `Deprecation`, `Sunset`, and successor `Link` +headers. It is removed in v1.1; the customer proxy strips cookies, forwarding headers, and +all vendor secrets. + +Never place the Ed25519 signing seed, Polar webhook secret, vendor admin token, or +Engraphis Resend key on a customer service. + +## Release readiness + +`GET /api/ready` is the public serving gate used by the orchestrator. On the customer +service it checks the database/embedder path. On the vendor service it checks service mode, +the approved signer, writable registry, exact Polar webhook/organization/products and +idempotency store, mail configuration and worker liveness, and disk capacity. It deliberately +does not include backup age, admin-monitoring configuration, delivery backlogs, or externally +triggerable alert counters: those conditions require operator action, and restarting or +draining an otherwise healthy first deployment cannot repair them. + +The full operational gates remain authenticated. `GET /api/ops/ready` on the customer +service requires an admin or operations bearer and returns boolean-only service-mode, +database-volume capacity, and backup-freshness checks. `GET /ops/ready` on the control plane +requires the vendor admin bearer and adds admin-token configuration, backup freshness, +delivery-webhook configuration, webhook/email backlog health, and the Polar processing-lease +check. A content-free rejected-lease count is reported for alerting but cannot make readiness +fail, because invalid public traffic must not let an attacker drain the service. None of these +endpoints returns secrets, customer addresses, license keys, storage paths, or provider +payloads. + +Generate the vendor administrator credential independently of every customer/API token: + +```bash +python -c "import secrets; print(secrets.token_urlsafe(48))" +``` + +`ENGRAPHIS_VENDOR_ADMIN_TOKEN` fails closed unless it contains 32-4096 non-whitespace +characters without control ASCII. Use the provider-generated Polar and Resend webhook +secrets; each verifier/readiness gate requires at least 16 bytes of raw key material (or +16 decoded bytes for an encoded `whsec_` value). Short placeholders are intentionally +rejected. The Ed25519 seed is exactly 32 bytes represented by 64 hex characters. When +`ENGRAPHIS_VENDOR_SIGNING_KEY` names a file, the resolved target must be a non-empty regular +file no larger than 1 KiB and, on POSIX, owner-only (`chmod 600`). Prefer +`scripts.license_admin keygen` over creating this file by hand. + +The signer release flag deliberately remains false until the issued-key inventory and +offline rotation ceremony are complete. If no keys exist, rotate cleanly. If keys exist, +ship key IDs plus the dual verifier, reissue, retain the old verifier for 30 days, then +revoke and remove it. Keep the private seed only in the production secret store and an +encrypted recovery backup. + +Run the PII-free inventory against the production registry before the ceremony: + +```bash +python -m scripts.license_admin inventory --db-path +``` + +`rotation_requires_migration: true` means the reviewed release must pin the old verifier +alongside the new signing-key ID, reissue active keys, and keep the old verifier for 30 days. +New manual keys default to the signed `https://license.engraphis.com` control-plane URL and +are recorded in the registry as part of issuance. Inventory groups persisted +`signing_key_id` values; pre-migration rows are reported as `unknown` rather than assigned an +unverified signer. Cached leases accept only explicitly pinned current/previous keys and fail +closed as soon as their signer is removed from that set. + +Do not generate or install the new signer until the inventory, an online SQLite backup of +the registry, and an encrypted recovery backup of the old seed are recorded. Generate into +a new offline path; do not overwrite the old seed: + +```bash +python -m scripts.license_admin keygen \ + --key-file /vendor_signing-YYYY-MM-DD.key +``` + +For a non-empty registry, deploy verifier compatibility before reissuing: pin the new public +key as `_VENDOR_PUBKEY_HEX`, retain the old public key in +`_PREVIOUS_VENDOR_PUBKEY_HEXES`, and leave `VENDOR_SIGNER_RELEASE_READY = False`. Create a +private source file containing one existing license key per line. The registry intentionally +does not retain raw keys; recover them only from protected fulfillment/customer records. The +command refuses a missing or extra fingerprint relative to inventory, preserves every signed +claim except `signing_key_id`, prints no customer address or license key, and leaves old keys +active: + +```bash +# Preflight only: no registry or output mutation. +python -m scripts.license_admin rotation-reissue \ + --db-path \ + --source-file \ + --new-key-file /vendor_signing-YYYY-MM-DD.key \ + --output-file + +# After operator/reviewer approval, write the 0600 delivery manifest and registry audit. +python -m scripts.license_admin rotation-reissue \ + --db-path \ + --source-file \ + --new-key-file /vendor_signing-YYYY-MM-DD.key \ + --output-file \ + --apply +``` + +The replacement manifest contains customer addresses and live license keys. Never commit, +email as an attachment, or log it; use it only through the approved delivery channel. If the +process stops after writing the manifest but before committing the registry, rerun the same +command with `--apply --resume`. + +After every replacement is delivered and activated, preflight retirement. The apply command +is hard-gated by the registry audit's 30-day age and refuses to revoke a source without an +active replacement: + +```bash +python -m scripts.license_admin rotation-retire \ + --db-path \ + --manifest-file + +python -m scripts.license_admin rotation-retire \ + --db-path \ + --manifest-file \ + --confirm-activated --apply +``` + +Only then remove the old public key from `_PREVIOUS_VENDOR_PUBKEY_HEXES`, deploy, and rerun +readiness plus production synthetics. The retirement command never edits the verifier pin or +destroys either seed. + +## Billing + +Production requires `POLAR_ORGANIZATION_ID`, `POLAR_WEBHOOK_SECRET`, and all four exact +product IDs from `engraphis/commercial_manifest.json`. Unknown products, wrong-organization +events, and malformed events are rejected. Durable event/order idempotency covers duplicate +delivery. Polar provides paid monthly/annual checkout only; the application owns the +three-day no-card trial. + +Exercise every product and lifecycle in Polar test mode. A real Pro monthly purchase and a +real Team monthly purchase/refund require designated inboxes and execution-time approval. + +## Transactional email + +Trial, purchase, invitation, reset, and key-reissue messages enter the durable outbox. +`GET /ops/email` returns redacted state and provider-message fingerprints; failed operations +remain recoverable, and `POST /ops/email/retry` retries due work. Verified Resend webhook +events update delivered, bounced, and complained states. Readiness fails on terminal delivery +failures, an old backlog, or a statistically meaningful bounce rate above the configured +threshold. A provider-only outage leaves the key solely in that durable outbox; it does not +create a redundant plaintext fallback. `undelivered_license_keys.tsv` is created only when +durable enqueue itself fails. While that fallback exists, authenticated vendor operations +readiness reports `manual_fulfillment=false` until an operator completes the documented +reconcile/deliver/remove-or-encrypted-archive procedure. + +Customer deployments with no local provider relay password-reset requests server-to-server +to `POST /license/v1/password-reset` using their active Pro/Team key. The control plane checks +revocation, binds the reset link to the deployment-claim origin (trial) or the key's pinned +origin (paid), applies per-key/per-recipient limits, and idempotently queues the message. It +never returns or logs the reset token. Provider downtime leaves the outbox item pending for +bounded retry; the public forgot-password response remains the same for known and unknown +addresses. + +Preserve the existing Resend DKIM and `send.engraphis.com` SPF records. Before GA, configure +inbound `keys@`, `billing@`, `support@`, and `dmarc@`; publish DMARC at `p=none`, observe for +seven days, review alignment reports, then move to `p=quarantine`. DNS changes require +execution-time authorization. + +## Backup and restore + +Run `scripts/commercial_backup.py backup` daily against an off-volume mount with a 64-hex +`ENGRAPHIS_BACKUP_KEY`. Retain 30 daily artifacts. The command snapshots memory, user/auth, +relay/control-plane, and durable Polar webhook SQLite databases with the online backup API, +runs integrity checks, encrypts the archive with AES-256-GCM, decrypts and verifies it, then +updates the marker used by readiness. The vendor archive therefore preserves the registry's +issued-license/order state and the Polar store's delivery claims and `subscription_seats` +ordering baseline; both are required to prevent duplicate fulfillment or stale seat changes +after a restore. + +Customer-mode archives also include only this reviewed state allowlist when each file exists: +`license.key`, `machine_id`, `lease.sig`, `sync.token`, `sync.read_only`, `trial.json`, +`trial_used.json`, and `.clock_anchor`. Each entry is capped at 1 MiB, symlinks are rejected, +and the backup code never walks the state directory. Vendor mode includes none of those +customer files. Its separate, strict allowlist contains only `undelivered_license_keys.tsv` +beside `ENGRAPHIS_WEBHOOK_STATE`, when that manual-fulfillment fallback exists. It can contain +buyer addresses and live license keys; it is capped at 1 MiB and restored with mode `0600`. +No unreviewed file or signing seed can enter either archive. Both encrypted archive types +contain live credentials, so protect each artifact and its backup key accordingly. + +Set `ENGRAPHIS_RELAY_DB` and `ENGRAPHIS_WEBHOOK_STATE` to persistent-volume paths on the +vendor service. Vendor-mode backup fails before creating an artifact or updating readiness if +either database is absent. Do not create an empty substitute during a backup: initialize the +real control-plane stores during staging setup and prove they contain the expected tables. + +Set `ENGRAPHIS_BACKUP_OUTPUT_DIR` to the off-volume mount and +`ENGRAPHIS_BACKUP_STATUS_FILE` to the on-volume marker on both managed services. The daily +`commercial encrypted backups` workflow calls the protected customer and control-plane +backup endpoints; neither response exposes the artifact path or encryption key. +Set a separate strong `ENGRAPHIS_API_TOKEN` on the managed customer service and copy it to +the Actions secret `ENGRAPHIS_CUSTOMER_OPS_TOKEN`. Do not put the deployment ownership +token in that secret: it intentionally does not bypass Team authentication after the first +admin exists. + +The `commercial encrypted restore drill` workflow runs monthly without production secrets. It +creates representative vendor registry, order, delivery-idempotency, and seat-baseline state, +encrypts it with an ephemeral key, and restores it only into a new runner-temporary directory. +The restore command refuses any destination that already exists, so the drill cannot overwrite +staging or production data. + +Restore output is deliberately staged: SQLite databases appear at the restore root and the +reviewed private-state allowlist appears under `/.engraphis/` with mode `0600`. Set the +target deployment's path environment variables before running restore so its owner-only +`RESTORE_PLAN.json` resolves the intended live destinations. Review that +`engraphis-restore-plan/v1` document and validate every staged file before acting. With all +writers stopped, place the databases at the reviewed paths and copy each staged `.engraphis` +file only to the exact destination recorded in the plan. Customer files target +`ENGRAPHIS_STATE_DIR`; the vendor `undelivered_license_keys.tsv` targets the directory beside +`ENGRAPHIS_WEBHOOK_STATE`. Before restarting a restored vendor service, reconcile every fallback +row against the restored registry, Polar order state, and durable email outbox. Deliver each +still-undelivered key exactly once, then remove the live fallback file or move it to an encrypted, +access-restricted retention archive. The plan records +`automatic_overwrite=false`; the restore command never overwrites live data for you. After +restart, create a new encrypted backup and require both serving and authenticated operations +readiness. + +That synthetic drill validates the mechanism, not backup freshness or access to the production +archive store. Run `verify` daily and, before every production release, restore a current +production-like artifact into an empty staging directory, restore the staged customer state +into a disposable `ENGRAPHIS_STATE_DIR`, and run the commercial smoke suite. Losing the backup +key makes encrypted backups unrecoverable. + +## Monitoring and rollback + +The hourly `commercial production synthetics` workflow checks public customer readiness, +authenticated customer storage readiness, control-plane readiness/details, and the +non-mutating trial dependency chain. GitHub Actions failure notifications are the baseline +alert channel. Infrastructure alerts must also cover volume free space and uptime. +Set `ENGRAPHIS_JSON_LOGS=1` on both hosted services. JSON logs contain bounded event text +and exception types; bearer tokens, license keys, secret assignments, and email-address +shapes are redacted before emission. +The control-plane outbox readiness check also alerts on exhausted retries, stale backlog, +and the 24-hour bounce/complaint rate. Tune `ENGRAPHIS_EMAIL_MAX_BACKLOG_AGE_SECONDS`, +`ENGRAPHIS_EMAIL_MAX_BOUNCE_RATE`, and `ENGRAPHIS_EMAIL_BOUNCE_MIN_SAMPLE` only from +observed production delivery volume. + +Roll back immediately on entitlement leakage, unsigned issuance, duplicate trials, +invitation privilege escalation, purchase without delivery, data-loss symptoms, or sustained +readiness failure. Public checkout remains disabled until 24 hours of clean production +synthetics after deployment. diff --git a/docs/HOSTING_RAILWAY.md b/docs/HOSTING_RAILWAY.md index 5bcc7cc..d10e2ef 100644 --- a/docs/HOSTING_RAILWAY.md +++ b/docs/HOSTING_RAILWAY.md @@ -1,223 +1,163 @@ -# Host on Railway (5 minutes) +# Host Engraphis on Railway -Deploy **one** Engraphis instance on Railway and access your memories from any browser, -on any device. Two paths, one repository workflow: +The v1.0 customer deployment is one persistent dashboard for Pro or Team. It does not +issue licenses, process Polar events, or hold the vendor signing key; those responsibilities +belong only to `license.engraphis.com`. -| | **Pro solo** | **Team admin** | -|---|---|---| -| Who | A Pro member (individual) | A Team administrator | -| License | Pro key or Pro trial | Team key or Team trial | -| Users | One admin (you) | Admin + invited members (seats) | -| Agent connect | Local agents sync to your Railway relay | Members connect agents directly to the cloud instance | -| Cost | ~$10–25/mo infra + $10/mo Pro | ~$10–25/mo infra + $20/mo/seat Team | +## Preferred: official template -Both paths use the same Docker image and the same volume + proxy -setup. The only difference is the license key you activate and whether you invite members. +The composer worksheet is +[`deploy/railway-template.json`](../deploy/railway-template.json). It is not an importable +Railway schema. Once Railway assigns the reviewed public template code, use the button in +the README. Until then, deploy this GitHub repository and apply the same settings manually. -For the solo/local-first lane (agent runs on your machine, optional cloud sync) see -[SYNC.md](./SYNC.md); for how agents connect to a hosted Team instance see -[AGENT_CONNECT.md](./AGENT_CONNECT.md). +The template must create: ---- +- one service built from `Dockerfile`; +- one persistent volume mounted at `/data`; +- a generated public HTTPS domain; +- `/api/ready` as the health check; and +- an `ENGRAPHIS_DEPLOYMENT_TOKEN` generated with Railway's `${{ secret(48) }}` template + function. Copy it into onboarding, then seal the variable after first-admin setup. -## Pro solo path (cloud dashboard + sync relay) +## Manual deployment -A Pro member deploys one instance to get a cloud-accessible dashboard (analytics, -automation, export) and a self-hosted sync relay — your local agents sync through your -own Railway instance instead of the vendor relay. One admin, no member seats. +1. Create a Railway project and deploy `Coding-Dev-Tools/engraphis` from GitHub. +2. Add a persistent volume mounted at `/data` before completing onboarding. +3. Generate a Railway public domain for the service. +4. Set these variables, replacing the domain and deployment token: -### What you need -- A Railway account. Roughly **~$10–25/mo** for one small always-on service + a persistent - volume. -- A **Pro license key** (purchase, or start a Pro trial from the dashboard once it's up). +```dotenv +ENGRAPHIS_SERVICE_MODE=customer +ENGRAPHIS_DB_PATH=/data/engraphis.db +ENGRAPHIS_STATE_DIR=/data/.engraphis +ENGRAPHIS_TEAM_MODE=1 +ENGRAPHIS_JSON_LOGS=1 +ENGRAPHIS_FORWARDED_ALLOW_IPS=* +ENGRAPHIS_CLOUD_URL=https://license.engraphis.com +ENGRAPHIS_DASHBOARD_URL=https://YOUR-DOMAIN.up.railway.app +ENGRAPHIS_RELAY_URL=https://YOUR-DOMAIN.up.railway.app +ENGRAPHIS_DEPLOYMENT_TOKEN=YOUR-UNIQUE-32+-CHARACTER-SECRET +``` -### 1. Deploy -In Railway choose **New Project → Deploy from GitHub repo**, select your Engraphis fork -or `Coding-Dev-Tools/engraphis`, and deploy it. Railway builds the Dockerfile, which -defaults to the v2 dashboard on port `8700` and runs as a non-root user. +Do not add any of these to a customer service: -### 2. Add a persistent volume (required) -Without this, activated license keys, the one-time trial, and **all memories** are lost -on every redeploy. In Railway: **service → Settings → Volumes → New Volume → mount path -`/data`**. Allocate at least **3 GiB** with the default 2 GiB per-account relay quota; -the database, license registry, and model cache need additional headroom. The Dockerfile -already writes the DB and license state under `/data`. +- `ENGRAPHIS_VENDOR_SIGNING_KEY` +- `ENGRAPHIS_VENDOR_ADMIN_TOKEN` +- `POLAR_WEBHOOK_SECRET` +- `POLAR_ORGANIZATION_ID` +- `POLAR_*_PRODUCT_ID` +- `ENGRAPHIS_RESEND_API_KEY` for Engraphis-operated transactional mail -### 3. Trust Railway's proxy and set the public URL -Railway fronts the container with a TLS proxy. In Railway: **service → Variables → add**: +Those are control-plane secrets. Customer deployments use the signed license service URL +and can optionally configure their own mail provider. Without one, invitations and password +resets are relayed server-to-server through the control plane using the active Pro/Team key; +reset tokens never appear in browser API responses or deployment logs. -``` -ENGRAPHIS_FORWARDED_ALLOW_IPS=* -ENGRAPHIS_DASHBOARD_URL=https:// -ENGRAPHIS_RELAY_URL=https:// -ENGRAPHIS_CLOUD_URL=https://team.engraphis.com -ENGRAPHIS_API_TOKEN= -``` +Operators of a separate vendor control plane must follow +[`COMMERCIAL_OPERATIONS.md`](COMMERCIAL_OPERATIONS.md): the vendor admin token is an +independent 32+-character secret, Polar/Resend webhook signing material provides at least +16 raw/decoded bytes, and a file-backed Ed25519 seed is a regular file no larger than 1 KiB +with owner-only (`0600`) permissions on POSIX. These requirements do not change the +prohibition above for customer services. + +## Hosted onboarding + +Open the generated HTTPS domain while signed out. The wizard performs this sequence: + +1. enter the deployment token; +2. choose a Pro or Team three-day trial; +3. enter and confirm an email address; +4. return to the deployment after confirmation; and +5. create the first admin with a chosen password. + +The confirmation link uses scanner-safe GET/POST semantics. The browser never receives the +signed key: the customer server claims it server-to-server, stores it under +`/data/.engraphis`, and activates it without a Railway redeploy. A pending claim identifier +is safe to keep in browser storage, so closing and reopening the page recovers activation. + +For an existing paid license, set `ENGRAPHIS_LICENSE_KEY` as a Railway secret. Paid Pro and +Team licenses renew online leases at `license.engraphis.com`; Free usage remains local and +does not call the license service. -Use the generated Railway domain first, or a custom domain you own. This is the canonical -URL **for your deployment**; `team.engraphis.com` is Engraphis's managed service, not a -customer-owned hostname. `ENGRAPHIS_DASHBOARD_URL` drives reset links, -redirects, and hosted MCP Host/Origin checks. `ENGRAPHIS_RELAY_URL` makes the cloud -dashboard exchange data through the relay mounted on this same instance. -`ENGRAPHIS_CLOUD_URL` keeps license leases, hosted trials, and fallback invite delivery on -Engraphis's managed issuer instead of sending those requests to the customer sync relay. -`ENGRAPHIS_API_TOKEN` proves deployment ownership when you start a hosted trial or create -the first admin; enter it in the corresponding hosted setup field. You may remove it after -setup unless service automation uses it. - -### 4. Activate Pro -Add `ENGRAPHIS_LICENSE_KEY=` in Railway Variables and redeploy. The key is -server-validated. Alternatively, open the dashboard, choose **Start Pro trial**, and open -the confirmation link sent to your email. For a hosted first boot, copy the confirmed key -into the private `ENGRAPHIS_LICENSE_KEY` Railway variable and redeploy; browser activation -remains admin-only so an arbitrary key holder cannot claim a fresh public instance. - -### 5. Create your admin account -Once `/api/license` reports `plan: "pro"`, the dashboard presents **Create admin -account**. Enter the `ENGRAPHIS_API_TOKEN` from Railway along with the account fields. -This is a single-admin instance — you can't invite members (that requires -Team). The admin account gives you a browser session (session cookie) and the ability to -mint a per-user bearer token for API access. - -### 6. Use it -- **Browser dashboard:** sign in at your Railway deployment URL with email + password. - All Pro features are unlocked: analytics, export, automation, cloud sync. -- **Sync relay:** enable auto-sync in the cloud dashboard (or use **Sync now**). Activate - the same Pro key on each local instance, then set - `ENGRAPHIS_RELAY_URL=https://`. Your local agents write locally; - each sync pass exchanges changes through the hosted relay, and the cloud dashboard - shows them after its next sync pass. -- **API access:** sign in, open **Settings → Connect your agent → Create token**, and use - the bearer token for HTTP API access (`GET /api/recall` is read-enabled; write - endpoints like `POST /api/remember` require a Team license — Pro solo uses cloud sync - for writes, not direct agent-connect). - -### 7. Configure the canonical domain -Follow the Team custom-domain steps below. Set both -`ENGRAPHIS_DASHBOARD_URL` and `ENGRAPHIS_RELAY_URL` to that domain, then update local instances to use -that relay URL so links, password resets, and sync all use the canonical domain. - ---- - -## Team admin path (multi-user, members join with credentials) - -Deploy one instance for your team; members sign in at your URL and connect their agents -over HTTP/MCP — **no local install for members**. The admin does a one-time deploy; -everyone else just logs in. A Team license (the instance's) is required for agent-connect; -members never need a key to log in. - -### What you need -- A Railway account (the **admin's** — Railway hosting is billed to the admin's account, - not yours). Roughly **~$10–25/mo** for one small always-on service + a persistent volume. -- A **Team license key** (purchase, or start a Team trial from the dashboard once it's up). - -### 1. Deploy -In Railway choose **New Project → Deploy from GitHub repo → select your Engraphis fork -or `Coding-Dev-Tools/engraphis`**. Railway builds from the -Dockerfile, which defaults to the v2 **team** dashboard on port `8700` and runs as a -non-root user. (`railway.json` tells Railway the healthcheck at `/api/health`.) - -The canonical public URL is the generated Railway domain or a custom domain **you own**. -Set `ENGRAPHIS_DASHBOARD_URL` to that URL so invites, password resets, redirects, and -hosted MCP checks agree. Set `ENGRAPHIS_RELAY_URL` to the same URL when local Pro clients -will sync through this deployment. `team.engraphis.com` remains the managed vendor -license/relay service; do not point customer DNS at it. - -### 2. Add a persistent volume (required) -Without this, activated license keys, the one-time trial, and **all memories** are lost -on every redeploy. In Railway: **service → Settings → Volumes → New Volume → mount path -`/data`**. Allocate at least **3 GiB** with the default 2 GiB per-account relay quota; -the DB, registry, and model cache need headroom. The Dockerfile writes the DB and license -state under `/data`. - -### 3. Trust Railway's forwarded headers -Railway fronts the container with a TLS proxy that isn't at `127.0.0.1`. Trust that proxy -so the application interprets the external scheme and client address correctly. In Railway: -**service → Variables → add**: +## Team invitations and sync +Admins invite an email and role; they do not choose a temporary password. The invitation +reserves a seat for 72 hours. Resend invalidates the old link, revoke or expiry releases the +seat, and the user is created only when the recipient chooses a password. + +Each user creates their own 90-day bearer token. The server stores only a hash and the +credential is revocable: + +- viewers: agent read plus `sync:read`; +- members/admins: agent read/write plus `sync:read` and `sync:write`. + +Paste a scoped token into Settings → Cloud sync on each local device. The account-wide Team +license key is never included in member invitations or used as the normal per-user sync +credential. The purchaser receives it only for initial deployment activation and recovery. +Each sync device keeps the raw bearer it must send in an owner-only +`$ENGRAPHIS_STATE_DIR/sync.token` file; protect that file like any other API credential. + +## Persistence and recovery + +The `/data` volume contains memories, auth state, machine identity, activated license, and +customer relay data. A redeploy without this volume is data loss. + +Volume snapshots alone are not enough. Schedule a daily encrypted off-volume backup: + +```bash +export ENGRAPHIS_BACKUP_KEY=<64-hex-secret-from-the-production-secret-store> +python -m scripts.commercial_backup backup \ + --output-dir /mounted-off-volume/engraphis \ + --marker /data/.engraphis/backup-status.json \ + --retention-days 30 ``` -ENGRAPHIS_FORWARDED_ALLOW_IPS=* -ENGRAPHIS_CLOUD_URL=https://team.engraphis.com -ENGRAPHIS_API_TOKEN= + +Set `ENGRAPHIS_BACKUP_STATUS_FILE=/data/.engraphis/backup-status.json`. Verify the newest +artifact daily and run a monthly restore drill into an empty directory: + +```bash +python -m scripts.commercial_backup verify /mounted-off-volume/engraphis/NEWEST.egbak +python -m scripts.commercial_backup restore /mounted-off-volume/engraphis/NEWEST.egbak \ + --output-dir /tmp/engraphis-restore-drill ``` -(You can scope this to Railway's egress range instead of `*` if you prefer.) -Keep `ENGRAPHIS_CLOUD_URL` on the managed issuer if this deployment also sets -`ENGRAPHIS_RELAY_URL` to itself for customer-operated sync. -The API token is required only as proof that you control the deployment while creating -the first admin. Keep it if service automation needs a shared credential; otherwise -remove it from Railway after setup. - -> **Port:** Railway auto-detects `8700` from the Dockerfile's `EXPOSE`. If the deploy -> shows a port mismatch / 502, set the service's **Port** to `8700`. - -### 4. Configure an optional custom domain -For a domain you control, such as `https://memory.example.com`: -1. **Railway → service → Settings → Networking → Custom Domain →** add - `memory.example.com`; Railway shows a CNAME target. -2. In your DNS, add `memory.example.com CNAME → `. Railway auto-issues the - TLS certificate. -3. **Variables → update:** `ENGRAPHIS_DASHBOARD_URL=https://memory.example.com` - (with `https://`, no trailing slash). - -### 5. Activate Team, then bootstrap the admin -`POST /api/auth/setup` deliberately refuses to create the first admin until a paid -entitlement is active. Bootstrap it one of two ways: - -- **Purchased key:** add `ENGRAPHIS_LICENSE_KEY=` in Railway Variables and - redeploy. The key is server-validated and sets the seat cap. -- **Trial:** open the dashboard, enter the deployment's `ENGRAPHIS_API_TOKEN`, choose - **Start Team trial**, and open the confirmation link sent to your email. Copy the - displayed key into Railway's private `ENGRAPHIS_LICENSE_KEY` variable and redeploy. - The trial route works before login but requires the deployment token; activation - remains admin-only. - -Once `/api/license` reports `plan: "team"`, the dashboard presents **Create admin -account**. Enter the `ENGRAPHIS_API_TOKEN` from Railway in the hosted setup form, create -the admin, then use **Settings → License** for later key replacement. -`/api/license/activate` stays admin-only; a purchased key cannot be pasted through that -route before the first admin exists. - -### 6. Invite members (seats) -**Team → Add member** (email + initial password + role: viewer/member/admin). Each member -is a seat; you can't add more active members than your Team license's seats. Members get an -invite email pointing at your dashboard URL; they sign in with email + password — **no -key, no local install**. If the license later lapses, the authentication wall stays active -and existing users can still sign in, while Team-gated operations return `402`. - -### 7. Members connect their agents -Each member signs in, opens **Settings → Connect your agent → Create token**, and pastes -the one-time bearer token into their agent config. Two transports (see -[AGENT_CONNECT.md](./AGENT_CONNECT.md) for full details): - -- **HTTP** (always available): `POST https:///api/remember` and - `GET https:///api/recall` with `Authorization: Bearer `. -- **MCP-over-HTTP:** point an MCP client at `https:///mcp` with the - bearer header. - -Writes land in the same v2 store the dashboard reads; the instance's Team license is what -unlocks the write endpoints (`402` without it). - ---- - -## Cost & limits -- **Infra:** one flat instance per team or solo user on the admin's/member's Railway - account (~$10–25/mo), amortized across seats for Team — *not* per user. Team seats are - $20/mo each; Pro is $10/mo flat ($100/yr). -- **Embedder:** CPU inference of `all-MiniLM-L6-v2` on every write/recall is the main cost - driver. For write-heavy deployments, set `ENGRAPHIS_EMBED_MODEL` to an external embedding - API (the config supports an API embedder) to cut Railway CPU and improve latency. -- **Scale:** the dashboard uses a single SQLite (WAL) store — fine for ~tens of concurrent - agents, not hundreds. Cap seat sales accordingly until a Postgres backend exists. -- **Backups:** Railway volumes are not auto-backed-up. Enable Railway volume backups, or - run `GET /api/export?workspace=…` on a cron, so you're not on the hook for data loss. - -## Security notes -- Expose the instance over **HTTPS only** (Railway does this). Bearer tokens and session - cookies must not transit cleartext. -- Per-user tokens are SHA-256 hashed at rest; the raw token is shown once. Disabling a - member instantly invalidates their tokens. -- `/api/remember` and `/mcp` require an active Team license (`402` otherwise). - A lapse keeps the authentication wall in place; existing users can still log in. -- The auth wall activates on any paid license (Pro or Team), closing the pre-bootstrap - exposure window on Railway. A free/unlicensed instance stays open for local-only use. +The backup command refuses a destination on the live data device unless explicitly put in +drill mode. It uses SQLite online backups, checks integrity, encrypts with AES-256-GCM, and +writes the freshness marker only after decrypting the artifact and rechecking every +database checksum and SQLite integrity result. The separate monthly drill exercises the +copy into a new restore directory. Customer backups also preserve a strict allowlist of +machine/license/lease/trial state and the saved sync credential/policy from +`ENGRAPHIS_STATE_DIR`; each existing file is bounded, symlinks are rejected, and no directory +is walked. These credentials remain protected by the encrypted archive. + +A restore is staged and never overwrites live data. Set the replacement deployment's target +path environment variables before restoring. The command writes databases at the restore root, +the allowed state files under `/.engraphis/`, and an owner-only +`RESTORE_PLAN.json` that maps every staged file to its resolved live destination. Review the +plan, stop all writers, then put the databases at those paths and copy the staged `.engraphis` +content into `ENGRAPHIS_STATE_DIR`. After starting the replacement, create a new encrypted +backup and require both `/api/ready` and authenticated `/api/ops/ready` to pass. + +For an Engraphis-operated managed deployment, configure a separate strong +`ENGRAPHIS_API_TOKEN` and store the same value in GitHub as +`ENGRAPHIS_CUSTOMER_OPS_TOKEN`. Scheduled backup and authenticated readiness workflows use +this revocable operations credential. They must not use `ENGRAPHIS_DEPLOYMENT_TOKEN`, which +is an ownership/onboarding secret rather than a permanent service-account credential. + +## Acceptance checklist + +From a logged-out Railway account, prove all of the following before the template is public: + +- `/api/ready` returns 200 after a clean deploy; +- Pro and Team confirmation activate automatically without showing a key; +- first-admin setup rejects an incorrect deployment token; +- invitation accept, resend, revoke, expiry, and seat limits work; +- viewer sync is read-only and member/admin sync is read/write; +- a redeploy preserves users, licenses, tokens, and memories; +- a verified backup restores databases and the allowlisted `.engraphis` state into a + disposable staging deployment; and +- the browser console has no CSP, accessibility, or network errors. + +Publishing the template, changing DNS, and enabling public checkout remain release-operator +actions and occur only after the production acceptance gates pass. diff --git a/docs/RAILWAY_TEMPLATE.md b/docs/RAILWAY_TEMPLATE.md index 1099071..feca5af 100644 --- a/docs/RAILWAY_TEMPLATE.md +++ b/docs/RAILWAY_TEMPLATE.md @@ -1,81 +1,35 @@ -# Publishing the Engraphis Railway template - -This is the spec for a real, publishable Railway template. It exists because the -"Deploy on Railway" button that used to be in the README **did not work**, and the -replacement has to be created through Railway's UI — it cannot be committed as a file. - -## Why the old button failed - -``` -https://railway.app/new?template=https://raw.githubusercontent.com/.../railway.json -``` - -Railway ignores `?template=`. Verified 2026-07-18 by loading that exact URL: it -renders the generic **New Project** chooser, and the `/new/template?template=` -variant renders the public template marketplace. Neither one references Engraphis. - -The confusion is understandable — both files are called "template-ish" — but they are -different objects: - -| | `railway.json` (in this repo) | A Railway *template* | -|---|---|---| -| What it is | Per-service build + deploy config | A publishable project blueprint | -| Where it lives | The repo | Railway's servers, with a template code | -| Can declare env vars | No | Yes, with descriptions + defaults | -| Can declare a volume | No | Yes | -| Referenced by | Railway, after a service exists | `railway.com/deploy/` | - -So `railway.json` is correct and should stay; it just cannot pre-configure anything a -new user needs. Everything the operator must supply had to be done by hand — which is -exactly the friction a one-click button is supposed to remove. - -## What the template must declare - -Create it at **Railway → project → Settings → Create Template**, from this repo, then -add the following. The Dockerfile already bakes `ENGRAPHIS_PORT`, `ENGRAPHIS_DB_PATH`, -`HF_HOME`, and `ENGRAPHIS_STATE_DIR`; `ENGRAPHIS_HOST` is derived at runtime by -`docker-entrypoint.sh`. None of those belong in the template. - -### Volume (required) - -| Mount path | Why | -|---|---| -| `/data` | Holds `engraphis.db`, `.engraphis/` (activated key, machine id, trial state, **revocation registry**), and the cached embedding model. Without it every redeploy loses the license and re-downloads the model into the healthcheck race. | - -Offer at least 3 GiB by default: the relay permits 2 GiB per account, and the database, -registry, and model cache require headroom on the same volume. - -### Variables - -| Variable | Required | Default to offer | Description to show the user | -|---|---|---|---| -| `ENGRAPHIS_FORWARDED_ALLOW_IPS` | yes | `*` | Trust Railway's proxy for client scheme/IP. Without it, session cookies don't get the `Secure` flag and per-IP limits misread the caller. | -| `ENGRAPHIS_DASHBOARD_URL` | yes | `https://${{RAILWAY_PUBLIC_DOMAIN}}` | Public URL of this instance. Used for invite and password-reset links and for the MCP allowed-hosts list. | -| `ENGRAPHIS_RELAY_URL` | yes | `https://${{RAILWAY_PUBLIC_DOMAIN}}` | This deployment's relay URL for local Pro clients; do not substitute the vendor hostname for a customer deployment. | -| `ENGRAPHIS_CLOUD_URL` | yes | `https://team.engraphis.com` | Managed issuer used for revocable leases, hosted trials, and fallback invite delivery while sync uses this customer deployment. | -| `ENGRAPHIS_LICENSE_KEY` | no | *(empty)* | Your Pro/Team key. A confirmed hosted trial key must be copied here and redeployed before first-admin setup. | -| `ENGRAPHIS_API_TOKEN` | yes | *(user-generated secret)* | Proof of deployment ownership during hosted trial and remote first-admin setup, and an optional service credential afterward. The setup fields ask for it; it may be removed after the admin exists if no service automation uses it. | - -### Vendor-only — do NOT put these in the public template - -`ENGRAPHIS_VENDOR_SIGNING_KEY`, `ENGRAPHIS_VENDOR_ADMIN_TOKEN`, -`ENGRAPHIS_RELAY_PUBLIC_URL`, `POLAR_WEBHOOK_SECRET`, `POLAR_ORGANIZATION_ID`, -`ENGRAPHIS_RESEND_API_KEY`, `ENGRAPHIS_RELAY_DB`, `ENGRAPHIS_LEASE_TTL_HOURS`. - -These belong only to the machine that *sells* Engraphis (team.engraphis.com). A -customer's self-hosted instance is a client of that relay, never a second issuer — it -does not hold the vendor private key and could not sign a valid lease anyway. - -## After publishing - -Railway issues a template code. Put the button back in `README.md`, replacing the -"Deploy on Railway (5-minute guide)" link: - -```markdown -[![Deploy on Railway](https://railway.com/button.svg)](https://railway.com/deploy/) -``` - -Then verify it the way the broken one should have been verified: open the URL in a -logged-out browser and confirm the page names Engraphis and lists the `/data` volume and -the two required variables. A template that renders the generic project chooser is not -deployed correctly, regardless of what the URL looks like. +# Railway template publication runbook + +[`deploy/railway-template.json`](../deploy/railway-template.json) is a reviewable composer +worksheet and the source of truth for the public Railway template. It is not a +Railway-importable JSON schema: enter or verify these settings in Railway's template +composer (or generate a template from the matching staging project). `railway.json` +configures a service after creation; it is not itself a marketplace template. + +## Required template shape + +- Source: `Coding-Dev-Tools/engraphis`, branch `main`, `Dockerfile` build. +- Service mode: `customer`. +- Persistent volume: `/data`. +- Health check: `/api/ready`. +- Public domain references: + `ENGRAPHIS_DASHBOARD_URL=https://${{RAILWAY_PUBLIC_DOMAIN}}` and the same value for + `ENGRAPHIS_RELAY_URL`. +- Managed license service: `ENGRAPHIS_CLOUD_URL=https://license.engraphis.com`. +- Generated ownership secret: `ENGRAPHIS_DEPLOYMENT_TOKEN=${{ secret(48) }}`. Copy it + into the hosted onboarding wizard, then seal it after the first admin is created. + +Vendor signer, Polar, vendor-admin, and Engraphis-operated email secrets must not appear in +the template. + +## Publish gate + +1. Run `python scripts/check_commercial_manifest.py` and the complete repository CI gate. +2. Create the Railway template from a staging project matching the descriptor exactly. +3. Deploy it from a logged-out Railway account and complete the acceptance checklist in + [`HOSTING_RAILWAY.md`](HOSTING_RAILWAY.md). +4. Record the assigned `https://railway.com/deploy/` URL. +5. Add that exact URL to the README only after the logged-out test succeeds. + +Do not substitute `railway.app/new?template=`; Railway ignores that shape +and opens a generic project chooser. diff --git a/docs/SYNC.md b/docs/SYNC.md index 30462df..1d705a8 100644 --- a/docs/SYNC.md +++ b/docs/SYNC.md @@ -65,18 +65,17 @@ your memory store (SQLite) another device's store Syncthing share, a mounted drive, even a git repo. Each device writes one full-state bundle (`bundle-.json`) and overwrites it each sync, so the folder stays small. - **`engraphis/backends/sync_relay.py` — `RelayTransport`.** The managed transport: the - same three `SyncTransport` calls over HTTPS against the vendor relay - (`engraphis/inspector/sync_relay.py`), carrying the device's license key as a bearer - token. The relay verifies that key **server-side** and (on Team) holds each device to a - seat, so patching the local feature check can't unlock sync. Bundles are namespaced by an - account id derived from the license, so customers never see each other's data. + same three `SyncTransport` calls over HTTPS against the customer service + (`engraphis/inspector/sync_relay.py`), carrying an expiring, revocable, per-user token. + The server verifies owner, scope, role, and the account's active entitlement on every + request. Bundles are namespaced by the signed account entitlement, so customers never + see each other's data. - **`get_transport(kind, **kw)`** (in `sync_folder.py`) selects between them by name — `"folder"` (needs `root=`) or `"relay"` (needs `base_url=` + `workspace_id=`) — the same factory pattern as `get_embedder`/`get_vector_index`; `relay` is imported lazily so a folder-only install stays dependency-light. -- **`scripts/sync.py` — the CLI** (`--remote ` or `--relay []`) and the place - the **Pro gate** lives (`require_feature("sync")`), exactly like `scripts/consolidate.py` - gates `--report`. +- **`scripts/sync.py` — the CLI** (`--remote ` or `--relay []`). Folder sync + retains the local Pro gate; relay sync is authorized server-side by the scoped token. ### How the merge converges @@ -123,21 +122,34 @@ python -m scripts.sync --db engraphis.db --workspace acme --remote ~/Dropbox/eng python -m scripts.sync --db engraphis.db --workspace acme --remote ~/Dropbox/engraphis --repo frontend ``` -Or use the **managed relay** instead of a shared folder — same command, `--relay` in place -of `--remote`. The relay authenticates with your license key server-side, so no folder to -set up and no way to bypass the gate by patching the client: +Or use the **managed relay** instead of a shared folder. Create your own device token in +the Team dashboard, then save it in the local dashboard or provide it through +`ENGRAPHIS_SYNC_TOKEN` / `--relay-token`. The browser and email never receive the account +license key: ```bash # Point at a relay host explicitly … -python -m scripts.sync --db engraphis.db --workspace acme --relay https://team.engraphis.com +python -m scripts.sync --db engraphis.db --workspace acme \ + --relay https://team.engraphis.com --relay-token # … or set ENGRAPHIS_RELAY_URL once and pass a bare --relay python -m scripts.sync --db engraphis.db --workspace acme --relay + +# Viewer tokens have sync:read only and must never upload a local bundle +python -m scripts.sync --db engraphis.db --workspace acme --relay --read-only ``` The relay is namespaced by workspace **name**, so every device on the account that syncs -workspace `acme` shares one bucket; the license key isolates your account from every other -customer's. `--relay-key` overrides the device's configured license key if needed. +workspace `acme` shares one bucket. The account entitlement isolates tenants; the user +token determines who may access it. Viewers have `sync:read`; members and admins also have +`sync:write`. New tokens default to 90 days. Revoking a token or disabling/deleting its +owner takes effect immediately. `--relay-key` exists only as a hidden v1.0 migration alias. + +The hosted server stores only the token hash. A device configured through the dashboard +must retain the raw bearer for later sync rounds, so it writes the value atomically to +`$ENGRAPHIS_STATE_DIR/sync.token` (default `~/.engraphis/sync.token`) with owner-only +permissions where the platform supports them. Treat that local file and +`ENGRAPHIS_SYNC_TOKEN` as secrets; revocation makes either copy unusable. Schedule it like any other local job: @@ -166,7 +178,7 @@ Team memory should stay converged without anyone remembering to click — but on relay traffic (and, on a metered host, cost) to a known ceiling regardless of how much the team edits. It is **opt-in** (off by default) and Pro/Team-gated (`engraphis.autosync.run_once` re-checks the license + key each tick and no-ops if the - plan lapsed). + plan lapsed). It uses the saved scoped token and honors its read-only setting. - **CLI + cron/Task Scheduler** (headless, dashboard not running): the `python -m scripts.sync … --relay` line above, on whatever cadence you like. @@ -186,10 +198,11 @@ those plus team management and the auto-sync switches. A member's writes still t the next scheduled sync when an admin has enabled it; writes never bypass the configured cadence based on the writer's role. -The relay is namespaced by an account id derived from the license **email**, so every -device that syncs with the *same* Team key lands in one shared bucket — that is what makes -"team memory" a single converged store. The relay holds each Team device to a live seat -(idle seats are reclaimed), so auto-sync never lets a Team key exceed its seat count. +The relay namespace comes from the signed account entitlement, while authorization comes +from each person's scoped token. Team members therefore share one converged store without +sharing the Team license key. Viewer tokens can list/download only; member/admin tokens +may upload/delete. The legacy Team-key route is disabled by default in customer mode and +can be enabled only for a documented migration window. --- diff --git a/engraphis/__init__.py b/engraphis/__init__.py index 5f90932..93841fd 100644 --- a/engraphis/__init__.py +++ b/engraphis/__init__.py @@ -7,4 +7,4 @@ except PackageNotFoundError: # source tree without an installed distribution # Keep in step with [project] version in pyproject.toml — tests/test_packaging.py # pins the two together so a release cannot ship them out of sync. - __version__ = "0.9.9" + __version__ = "1.0.0" diff --git a/engraphis/app.py b/engraphis/app.py index 8fbd653..850c9a4 100644 --- a/engraphis/app.py +++ b/engraphis/app.py @@ -16,7 +16,6 @@ from fastapi.staticfiles import StaticFiles from engraphis import __version__ -from engraphis.billing import router as billing_router from engraphis.inspector.auth import bearer_ok from engraphis.inspector.cloud_mount import CLOUD_PREFIXES, mount_cloud_endpoints from engraphis.config import settings @@ -43,8 +42,10 @@ def _embedder_ready() -> bool: from engraphis.backends.embedder_st import get_embedder emb = get_embedder(settings.embed_model or None, settings.embed_dim or 384) _embedder_ok = emb is not None and int(emb.dim) > 0 - except Exception as e: # pragma: no cover - defensive; get_embedder falls back itself - logger.warning("Readiness: embedder init failed: %s", e) + except Exception as exc: # pragma: no cover - defensive; get_embedder falls back itself + # Provider/backend exceptions can contain credentialed URLs or local paths. + # Readiness logs need the failure class, not the exception payload. + logger.warning("Readiness: embedder init failed (%s)", type(exc).__name__) _embedder_ok = False return _embedder_ok @@ -77,6 +78,22 @@ async def _lifespan(app: FastAPI): def create_app() -> FastAPI: """Build and configure the FastAPI application.""" configure_logging() + # Hosted JSON logging is credential-redacting. Keep this after the legacy logging + # setup so it replaces that formatter, and pair it with the launcher's log_config=None + # so Uvicorn cannot replace it again after app construction. + from engraphis.observability import configure_structured_logging + configure_structured_logging() + + # The legacy reference entrypoint is still packaged and callable, so it must honor + # the same commercial role boundary as the primary dashboard entrypoint. Without + # this dispatch a customer-mode process mounted the Polar webhook, signer-backed + # issuance routes, and vendor-admin revocation surface merely because an operator + # launched ``engraphis-server`` instead of ``engraphis-dashboard``. + from engraphis.commercial import service_mode + mode = service_mode() + if mode == "vendor": + from engraphis.vendor_app import create_app as create_vendor_app + return create_vendor_app() app = FastAPI( title="Engraphis", @@ -182,11 +199,19 @@ async def _request_log(request: Request, call_next): app.include_router(vault_router) # Purchase fulfillment (Polar order.paid → signed key → email). Shared with the # Inspector so it works regardless of which entrypoint is deployed. - app.include_router(billing_router) + if settings.vendor_service: + from engraphis.billing import router as billing_router + app.include_router(billing_router) # Cloud license (register/verify/REVOKE) + gated Pro sync relay. Previously # mounted only on the retired Inspector, which made revocation inoperable in # production; now served by every shipped entrypoint. See inspector.cloud_mount. - mount_cloud_endpoints(app) + mount_cloud_endpoints( + app, include_license=settings.vendor_service, + include_sync=settings.customer_service) + # Customer mode preserves the pre-split URL only as a bounded compatibility proxy; + # it never mounts the local signer/control-plane implementation. + from engraphis.inspector.license_compat_proxy import mount_license_compat_proxy + mount_license_compat_proxy(app) # ── probes (unauthenticated; see _PUBLIC_PREFIXES) ────────────────────────── @app.get("/api/health") @@ -202,8 +227,8 @@ async def api_ready(): try: get_conn().execute("SELECT 1").fetchone() checks["db"] = True - except Exception as e: - logger.warning("Readiness: db check failed: %s", e) + except Exception as exc: + logger.warning("Readiness: db check failed (%s)", type(exc).__name__) checks["embedder"] = _embedder_ready() ready = all(checks.values()) return JSONResponse({"ready": ready, "checks": checks, "version": __version__}, @@ -238,11 +263,15 @@ async def _consciousness_loop() -> None: persist=True, ) if result.get("persisted"): - logger.info("Thought synthesized: %s", result.get("thought")) + # A synthesized thought is memory content. Never copy it into logs. + logger.info( + "Thought synthesized and persisted (sources=%d)", + int(result.get("source_count") or 0), + ) except asyncio.CancelledError: raise - except Exception as e: - logger.error("Consciousness loop error: %s", e) + except Exception as exc: + logger.error("Consciousness loop error (%s)", type(exc).__name__) app = create_app() diff --git a/engraphis/autosync.py b/engraphis/autosync.py index 06b5971..86a0d44 100644 --- a/engraphis/autosync.py +++ b/engraphis/autosync.py @@ -167,9 +167,11 @@ def run_once(service: Any = None, *, now: Optional[float] = None, when the plan/key isn't ready (so the loop no-ops cheaply instead of hammering the relay).""" from engraphis import licensing - if not licensing.has_feature("sync"): + from engraphis.backends.sync_relay import has_sync_token + has_token = has_sync_token() + if not has_token and not licensing.has_feature("sync"): return {"skipped": "unlicensed"} - if not licensing._read_key_material(): + if not has_token and not licensing._read_key_material(): return {"skipped": "no-key"} from engraphis.routes import v2_api svc = service if service is not None else v2_api.service() diff --git a/engraphis/backends/embedder_api.py b/engraphis/backends/embedder_api.py index 8158268..d7c10f4 100644 --- a/engraphis/backends/embedder_api.py +++ b/engraphis/backends/embedder_api.py @@ -53,9 +53,12 @@ def __init__( self._api_key = api_key or os.environ.get(_DEFAULT_API_KEY_ENV, "") self._dim = dim self._embeddings_url = f"{self._base_url}/v1/embeddings" + # A custom endpoint can contain embedded credentials or signed query + # parameters, while provider-controlled model identifiers are also untrusted + # log input. Do not copy either into logs. logger.info( - "ApiEmbedder(model=%s, base_url=%s, dim=%s)", - self.model, self._base_url, self._dim or "auto", + "ApiEmbedder(custom_endpoint=%s, dim=%s)", + bool(base_url), self._dim or "auto", ) @property @@ -131,10 +134,9 @@ def embed( for item in items: emb = item.get("embedding") if emb is None: - logger.warning( - "Item index %s missing 'embedding' key, using zero vector", - item.get("index", "?"), - ) + # Never log the provider-controlled ``index`` value; a malformed + # response can otherwise inject PII, credentials, or new log lines. + logger.warning("Embedding item missing vector; using zero vector") emb = [0.0] * (self._dim or 384) vecs.append(emb) diff --git a/engraphis/backends/extractor.py b/engraphis/backends/extractor.py index 3902807..773670f 100644 --- a/engraphis/backends/extractor.py +++ b/engraphis/backends/extractor.py @@ -107,6 +107,18 @@ def _defang(value: str, limit: int) -> str: ) +def _llm_activity_metadata(llm: Any, mode: str) -> dict[str, str]: + """Describe a successful extraction without storing prompts or provider responses.""" + activity = {"mode": mode} + provider = _defang(str(getattr(llm, "provider", "") or ""), 128) + model = _defang(str(getattr(llm, "model", "") or ""), 256) + if provider: + activity["provider"] = provider + if model: + activity["model"] = model + return activity + + class PassthroughExtractor: """The offline default: one fact, the text as given.""" @@ -168,7 +180,9 @@ def _parse(self, raw: str) -> list[ExtractedFact]: out.append(ExtractedFact(content=content, title=_defang(str(item.get("title") or ""), 1_000), mtype=mtype, importance=importance, - keywords=[k for k in keywords if k])) + keywords=[k for k in keywords if k], + metadata={"llm_extraction": + _llm_activity_metadata(self.llm, "llm")})) return out @@ -289,7 +303,9 @@ def _parse_and_validate(self, raw: Any) -> list[ExtractedFact]: extra = {k: v for k, v in fact.items() if k not in { "content", "title", "mtype", "importance", "keywords", }} - metadata: dict[str, Any] = {} + metadata: dict[str, Any] = { + "llm_extraction": _llm_activity_metadata(self.llm, "llm_structured") + } if extra: metadata["structured_extraction"] = extra if entities: diff --git a/engraphis/backends/graph_extractor.py b/engraphis/backends/graph_extractor.py index 167dd99..5d528f3 100644 --- a/engraphis/backends/graph_extractor.py +++ b/engraphis/backends/graph_extractor.py @@ -49,13 +49,26 @@ "Monday", "Tuesday", "Wednesday", "Thursday", "Friday", "Saturday", "Sunday", "January", "February", "March", "April", "May", "June", "July", "August", "September", "October", "November", "December", + # Common capitalized sentence/workflow fragments. They are not stable identities + # and otherwise become high-degree co-occurrence hubs in technical memories. + "Active", "Action", "Actions", "Add", "Added", "All", "Also", "Artifact", + "Artifacts", "Author", "Because", "Check", "Checked", "Comment", "Comments", + "Commit", "Connection", "Connections", "Detail", "Details", "False", "Input", + "Key", "Keys", "Local", "Manifest", "Merge", "Merged", "Missing", "Only", + "Outcome", "Output", "Per", "Possible", "Reason", "Request", "Response", "Result", + "Results", "Run", "Running", "Scan", "Scanned", "Status", "Test", "Tests", + "Title", "True", "Verdict", "Approval", "Approved", "Categories", "Degraded", + "Error", "Errors", "Failed", "Passed", "Rejected", "Skipped", "Success", "Verify", + "Warning", "Warnings", } +_STOPWORD_KEYS = {value.casefold() for value in _STOPWORDS} # Extracted text is untrusted: strip the same control chars service.py strips from # direct writes so an entity name can't smuggle a hidden-instruction / escape payload # into the graph, and cap length so a pathological match can't bloat a node. _CONTROL_RE = re.compile(r"[\x00-\x08\x0b\x0c\x0e-\x1f\x7f]") _MAX_NAME = 200 +_MAX_ENTITIES = 128 _MAX_RELATIONS = 20 _MAX_COOCCUR_ENTITIES = 8 # cap pairwise co-occurrence per memory (<= 28 edges) _COOCCUR_WEIGHT = 0.5 # weaker than a specific relation so PPR prefers real edges @@ -96,7 +109,9 @@ def _extract_entities(text: str) -> list[tuple[str, str]]: out: list[tuple[str, str]] = [] for m in _ENTITY_RE.finditer(text or ""): raw = (m.group(0) or "").strip() - if not raw or raw in _STOPWORDS or raw.lower() in ("user", "the user"): + if not raw or raw.casefold() in _STOPWORD_KEYS or raw.casefold() in ( + "user", "the user" + ): continue if raw.startswith("#"): ent, etype = raw, "hashtag" @@ -106,12 +121,14 @@ def _extract_entities(text: str) -> list[tuple[str, str]]: ent, etype = raw, "email" else: ent, etype = _canon_concept(raw), "person_or_concept" - if len(ent) < 2 or ent in _STOPWORDS: + if len(ent) < 2 or ent.casefold() in _STOPWORD_KEYS: continue key = ent.lower() if key not in seen: seen.add(key) out.append((ent, etype)) + if len(out) >= _MAX_ENTITIES: + break return out @@ -132,6 +149,8 @@ def _extract_entities_from_doc(title: str, content: str) -> list[tuple[str, str] if key not in seen: seen.add(key) out.append((ent, etype)) + if len(out) >= _MAX_ENTITIES: + return out return out @@ -241,7 +260,8 @@ def get_graph_extractor(kind: str = "none"): def feed(store: Any, content: str, *, workspace_id: str, repo_id: Optional[str] = None, title: str = "", extractor: Any = None, - provenance: Optional[dict] = None) -> dict: + provenance: Optional[dict] = None, commit: bool = True, + extraction: Any = None) -> dict: """Extract entities/relations from free text and write them into the knowledge graph, scoped to ``(workspace_id, repo_id)``. @@ -256,16 +276,17 @@ def feed(store: Any, content: str, *, workspace_id: str, repo_id: Optional[str] Returns ``{"entities": , "relations": }``. """ extractor = extractor or NullGraphExtractor() - result = extractor.extract(content, title=title) + result = extraction if extraction is not None else extractor.extract(content, title=title) name_to_id: dict[str, str] = {} - for name, etype in result.entities: + for name, etype in result.entities[:_MAX_ENTITIES]: clean = _defang(name) if not clean or clean in name_to_id: continue name_to_id[clean] = store.upsert_entity( Node(id="", name=clean, ntype=_defang(etype), - workspace_id=workspace_id, repo_id=repo_id) + workspace_id=workspace_id, repo_id=repo_id), + commit=commit, ) prov = dict(provenance or {}) @@ -291,11 +312,11 @@ def feed(store: Any, content: str, *, workspace_id: str, repo_id: Optional[str] specific_pairs.add(frozenset((sid, did))) existing = edge_by_key.get(key) if existing is not None: - store.add_edge_support(existing.id, prov) + store.add_edge_support(existing.id, prov, commit=commit) continue eid = store.upsert_edge(Edge(id="", src=sid, dst=did, relation=relation, workspace_id=workspace_id, repo_id=repo_id, - provenance=prov)) + provenance=prov), commit=commit) edge_by_key[key] = Edge(id=eid, src=sid, dst=did, relation=relation) written_relations += 1 @@ -315,13 +336,13 @@ def feed(store: Any, content: str, *, workspace_id: str, repo_id: Optional[str] key = (lo, hi, "co_occurs") existing = edge_by_key.get(key) if existing is not None: - store.add_edge_support(existing.id, prov) + store.add_edge_support(existing.id, prov, commit=commit) continue eid = store.upsert_edge(Edge( id="", src=lo, dst=hi, relation="co_occurs", weight=_COOCCUR_WEIGHT, workspace_id=workspace_id, repo_id=repo_id, provenance=prov, - )) + ), commit=commit) edge_by_key[key] = Edge(id=eid, src=lo, dst=hi, relation="co_occurs") written_relations += 1 diff --git a/engraphis/backends/sync_folder.py b/engraphis/backends/sync_folder.py index d683999..7cb3464 100644 --- a/engraphis/backends/sync_folder.py +++ b/engraphis/backends/sync_folder.py @@ -179,8 +179,9 @@ def get_transport(kind: str = "folder", **kw): - ``folder`` (default): shared-directory sync. Requires ``root=``. - ``relay``: the managed Pro relay transport (``RelayTransport``). Requires ``base_url=`` and ``workspace_id=`` (use the workspace - *name*, so every device on the account shares one namespace); ``license_key`` and - ``timeout`` are optional (the key defaults to this device's configured license). + *name*, so every authorized device on the account shares one namespace); + ``license_key`` is a compatibility parameter for a scoped bearer token and + ``timeout`` is optional. The token defaults to the saved per-user sync token. Both implement the ``SyncTransport`` protocol (``core/interfaces.py``) and plug into ``SyncEngine.sync`` unchanged. ``relay`` is imported lazily so a folder-only install diff --git a/engraphis/backends/sync_relay.py b/engraphis/backends/sync_relay.py index 07caad6..fe92f4d 100644 --- a/engraphis/backends/sync_relay.py +++ b/engraphis/backends/sync_relay.py @@ -1,11 +1,12 @@ -"""Relay transport — the managed cloud-sync client (headline Pro upsell). +"""Relay transport — the managed cloud-sync client. Implements the ``SyncTransport`` protocol (``core/interfaces.py``) over HTTPS against the -vendor-hosted relay (``engraphis.inspector.sync_relay``). It carries the device's license -key as a bearer token; the server verifies that key *server-side* before accepting or -returning bundles, so — unlike a purely local feature check — patching the client cannot -unlock sync. It plugs into ``SyncEngine.sync`` exactly like ``FolderTransport``; the sync -engine is unchanged and still treats every pulled bundle as untrusted. +customer-hosted relay (``engraphis.inspector.sync_relay``). It carries an expiring, +revocable, per-user token as a bearer credential; the server verifies its owner, role, +scope, and the account entitlement before accepting or returning bundles. A license-key +fallback exists only for the documented customer migration window. It plugs into +``SyncEngine.sync`` exactly like ``FolderTransport``; the sync engine is unchanged and +still treats every pulled bundle as untrusted. Dependency-light on purpose: stdlib ``urllib`` only, no ``requests``. """ @@ -16,9 +17,12 @@ import ipaddress import json import math +import os import re +import tempfile import urllib.error import urllib.request +from pathlib import Path from typing import Iterable, List, Optional, Tuple from urllib.parse import quote, urlsplit, urlunsplit @@ -67,11 +71,122 @@ def _urlopen_no_redirect(req, *, timeout: float): def _current_key() -> str: - """The license key configured on this device (env or ~/.engraphis/license.key).""" + """A scoped user sync token, falling back to a legacy license during migration.""" + configured = os.environ.get("ENGRAPHIS_SYNC_TOKEN", "").strip() + if configured: + return configured + path = _sync_token_path() + try: + stored = path.read_text(encoding="utf-8").strip() + if stored: + return stored + except OSError: + pass from engraphis import licensing return licensing._read_key_material() +def _sync_token_path() -> Path: + state = os.environ.get("ENGRAPHIS_STATE_DIR", "").strip() + root = Path(state).expanduser() if state else Path.home() / ".engraphis" + return root / "sync.token" + + +def _sync_read_only_path() -> Path: + return _sync_token_path().with_name("sync.read_only") + + +def _atomic_private_text(path: Path, value: str) -> None: + """Atomically write one owner-only state value next to the sync credential.""" + path.parent.mkdir(parents=True, exist_ok=True) + fd, temp_name = tempfile.mkstemp(prefix=".%s." % path.name, dir=str(path.parent)) + temp_path = Path(temp_name) + try: + try: + os.chmod(temp_path, 0o600) + except OSError: + pass + with os.fdopen(fd, "w", encoding="utf-8") as handle: + fd = -1 + handle.write(value + "\n") + handle.flush() + os.fsync(handle.fileno()) + os.replace(str(temp_path), str(path)) + try: + os.chmod(path, 0o600) + except OSError: + pass + except BaseException: + if fd >= 0: + os.close(fd) + try: + temp_path.unlink() + except OSError: + pass + raise + + +def save_sync_token(token: str) -> None: + """Atomically persist a per-user bearer token with owner-only permissions.""" + value = str(token or "").strip() + if (len(value) < 24 or len(value) > 8192 + or any(ord(char) < 32 or ord(char) == 127 for char in value)): + raise ValueError("sync token must be a bounded single-line bearer token") + _atomic_private_text(_sync_token_path(), value) + + +def save_sync_read_only(enabled: bool) -> None: + """Persist the no-upload policy beside the token, independent of project ``.env``. + + This state is deliberately separate from the token so it can be inspected without + touching credential material. The API writes the restrictive value before replacing + a token and the permissive value afterwards, so a partial update fails read-only. + """ + _atomic_private_text(_sync_read_only_path(), "1" if enabled else "0") + + +def sync_read_only() -> bool: + """Return the durable upload policy; malformed/unreadable saved state fails closed.""" + configured = os.environ.get("ENGRAPHIS_SYNC_READ_ONLY") + if configured is not None and configured.strip(): + raw = configured.strip().lower() + if raw in ("1", "true", "yes", "on"): + return True + if raw in ("0", "false", "no", "off"): + return False + return True + path = _sync_read_only_path() + try: + raw = path.read_text(encoding="utf-8").strip().lower() + except FileNotFoundError: + raw = "" + except OSError: + return True + if raw in ("1", "true", "yes", "on"): + return True + if raw in ("0", "false", "no", "off", ""): + return False + # Corrupt policy files must never turn uploads back on. + return True + + +def clear_sync_token() -> None: + for path in (_sync_token_path(), _sync_read_only_path()): + try: + path.unlink() + except FileNotFoundError: + pass + + +def has_sync_token() -> bool: + if os.environ.get("ENGRAPHIS_SYNC_TOKEN", "").strip(): + return True + try: + return bool(_sync_token_path().read_text(encoding="utf-8").strip()) + except OSError: + return False + + def _current_machine_id() -> str: """This device's stable id, best-effort. Sent to the relay so Team seat enforcement can bind the caller to a seat; harmless for Pro (the relay ignores it there).""" @@ -98,8 +213,8 @@ def _validated_base_url(value: str) -> str: raise ValueError("relay URL must be an absolute http(s) URL") try: parts.port - except ValueError as exc: - raise ValueError("relay URL has an invalid port") from exc + except ValueError: + raise ValueError("relay URL has an invalid port") from None if parts.username is not None or parts.password is not None: raise ValueError("relay URL must not contain embedded credentials") if "\\" in parts.netloc or any(char.isspace() for char in parts.netloc): @@ -123,11 +238,12 @@ def _safe_bundle_name(name: object) -> str: class RelayTransport: - """A ``SyncTransport`` backed by the vendor relay. + """A ``SyncTransport`` backed by the customer sync relay. ``base_url`` is the relay root (e.g. ``https://team.engraphis.com``). ``workspace_id`` - scopes bundles to one workspace. ``license_key`` defaults to this device's configured - key. All three protocol calls send ``Authorization: Bearer ``. + scopes bundles to one workspace. The compatibility parameter ``license_key`` accepts + the scoped token and defaults to ``ENGRAPHIS_SYNC_TOKEN`` or the locally saved token. + All protocol calls send ``Authorization: Bearer ``. """ def __init__(self, base_url: str, workspace_id: str, *, @@ -150,7 +266,7 @@ def __init__(self, base_url: str, workspace_id: str, *, len(key) > 8192 or any(ord(char) < 32 or ord(char) == 127 for char in key) ): - raise ValueError("relay license key must be a bounded single-line value") + raise ValueError("relay bearer token must be a bounded single-line value") self.key = key machine_id = str(_current_machine_id() or "").strip() self.machine_id = machine_id if ( @@ -159,8 +275,8 @@ def __init__(self, base_url: str, workspace_id: str, *, ) else "" try: timeout_value = float(timeout) - except (TypeError, ValueError) as exc: - raise ValueError("relay timeout must be a number") from exc + except (TypeError, ValueError): + raise ValueError("relay timeout must be a number") from None if not math.isfinite(timeout_value) or timeout_value <= 0: raise ValueError("relay timeout must be a positive finite number") self.timeout = min(timeout_value, 300.0) @@ -185,22 +301,17 @@ def _request(self, url: str, *, method: str, data: Optional[bytes] = None, raise RelayError("relay response exceeded the client safety limit") return body except urllib.error.HTTPError as exc: - body = b"" - try: - body = exc.read(MAX_RELAY_NAMES_BYTES + 1) - if len(body) > MAX_RELAY_NAMES_BYTES: - body = body[:MAX_RELAY_NAMES_BYTES] - except Exception: - pass - msg = body.decode("utf-8", "replace") if body else str(exc) + # Never propagate an untrusted relay response body or the HTTPError's + # request URL. Either can contain PII, signed query data, or reflected + # credentials and these errors are surfaced by sync APIs and CLIs. if exc.code == 402: - raise RelayError("relay rejected the license (upgrade/renew required): %s" - % msg, status=402) from exc - raise RelayError("relay request failed (%s): %s" % (exc.code, msg), - status=exc.code) from exc - except urllib.error.URLError as exc: - raise RelayUnreachable( - "could not reach the relay at %s: %s" % (self.base, exc.reason)) + raise RelayError( + "relay rejected the license (upgrade/renew required)", status=402 + ) from None + raise RelayError("relay request failed (HTTP %s)" % exc.code, + status=exc.code) from None + except urllib.error.URLError: + raise RelayUnreachable("could not reach the relay") from None # ── SyncTransport protocol ─────────────────────────────────────────────────────── def push(self, name: str, data: bytes) -> None: diff --git a/engraphis/billing.py b/engraphis/billing.py index 416f3ae..38aa975 100644 --- a/engraphis/billing.py +++ b/engraphis/billing.py @@ -20,6 +20,7 @@ import hmac import json import logging +import math import os import sqlite3 import threading @@ -45,14 +46,18 @@ _MAX_BODY_BYTES = 65_536 +def _log_ref(value: object) -> str: + """Return a non-reversible correlation id for untrusted/provider identifiers.""" + return hashlib.sha256(str(value or "").encode("utf-8", "replace")).hexdigest()[:12] + + def _decode_webhook_secret(secret: str) -> bytes: """Decode a Polar / Standard-Webhooks secret into raw HMAC key bytes. - Polar issues secrets in the form ``whsec_``. The ``whsec_`` prefix is - a label, NOT part of the key material, so it must be stripped before decoding - — decoding the whole string yields the wrong key and every real delivery then - fails the signature check. The base64 body may be unpadded, so pad it back. - A bare base64 secret (no prefix) is accepted too, for tests and manual setups. + This is the legacy Standard-Webhooks compatibility decoder. ``whsec_`` is a + label, not key material, so it is stripped before decoding. Current Polar raw + and ``polar_whs_`` secrets are handled by :func:`_webhook_secret_candidates`. + The base64 body may be unpadded, so pad it back. """ secret = (secret or "").strip() if secret.startswith("whsec_"): @@ -61,6 +66,46 @@ def _decode_webhook_secret(secret: str) -> bytes: return base64.b64decode(secret + pad, validate=True) +def _webhook_secret_candidates(secret: str) -> tuple[bytes, ...]: + """Return HMAC key candidates for Polar's current and legacy secret formats. + + Polar accepts operator-chosen raw secrets and its API may return values beginning + ``polar_whs_``; its SDK base64-encodes that raw text before handing it to a Standard + Webhooks verifier. Older Engraphis deployments documented a Standard Webhooks + ``whsec_`` value (and some used bare base64), so retain those formats too. + The signed request still has to match exactly one candidate; accepting both encodings + is compatibility, not a signature bypass. + """ + clean = (secret or "").strip() + if not clean: + return () + if clean.startswith("whsec_"): + try: + return (_decode_webhook_secret(clean),) + except (binascii.Error, ValueError): + return () + + candidates = [clean.encode("utf-8")] + # Bare base64 was the pre-v1.0 documented compatibility form. Prefer Polar's raw + # interpretation, but also verify the historical decoded form when it is valid. + try: + decoded = _decode_webhook_secret(clean) + except (binascii.Error, ValueError): + decoded = b"" + if decoded and decoded not in candidates: + candidates.append(decoded) + return tuple(candidates) + + +def webhook_secret_ready() -> bool: + """Return whether Polar signing material meets the minimum security boundary.""" + return any( + len(candidate) >= 16 + for candidate in _webhook_secret_candidates( + os.environ.get("POLAR_WEBHOOK_SECRET", "")) + ) + + # ── idempotency ─────────────────────────────────────────────────────────────── # Polar re-delivers an event until it receives a 2xx. Without dedup, a retry (or a # crash between minting a key and answering 2xx) would mint a *second* valid key @@ -68,9 +113,10 @@ def _decode_webhook_secret(secret: str) -> bytes: # We claim each Standard-Webhooks ``webhook-id`` (stable across retries of one # event) with an ATOMIC ``INSERT`` into a small SQLite table BEFORE fulfilling, so # the reservation is durable across workers/replicas and process restarts. On a -# fulfillment failure we release the claim so Polar's retry can try again. If no -# durable path is available we fall back to an in-process set (still correct for a -# single worker). A dedup-store error must never block a real purchase. +# fulfillment failure we release the claim so Polar's retry can try again. Combined +# development mode retains the historical in-process fallback, but the isolated vendor +# service always resolves a deterministic SQLite path and fails closed if it cannot use +# it. A control plane must never acknowledge a purchase without durable delivery state. _mem_lock = threading.Lock() _mem_seen: "set[str]" = set() _RESERVATION_TTL_SECONDS = 300 @@ -81,8 +127,12 @@ class WebhookStateError(RuntimeError): def _dedup_path() -> Optional[str]: + from engraphis.commercial import service_mode + vendor_mode = service_mode() == "vendor" override = os.environ.get("ENGRAPHIS_WEBHOOK_STATE", "").strip() if override: + if vendor_mode and override == ":memory:": + raise WebhookStateError("vendor webhook state cannot use an in-memory store") return override db = os.environ.get("ENGRAPHIS_DB_PATH", "").strip() if db and db != ":memory:": @@ -90,6 +140,24 @@ def _dedup_path() -> Optional[str]: return str(Path(db).expanduser().resolve().parent / ".engraphis_webhooks.db") except (OSError, RuntimeError) as exc: raise WebhookStateError("could not resolve durable webhook state path") from exc + # The vendor service may not run the customer memory database, so ENGRAPHIS_DB_PATH + # is commonly absent there. Keep its Polar ledger beside the durable license registry + # (or in ENGRAPHIS_STATE_DIR) instead of silently dropping to process memory. + if vendor_mode: + relay_db = os.environ.get("ENGRAPHIS_RELAY_DB", "").strip() + if relay_db == ":memory:": + raise WebhookStateError("vendor webhook state cannot use an in-memory store") + state_dir = os.environ.get("ENGRAPHIS_STATE_DIR", "").strip() + try: + if relay_db: + root = Path(relay_db).expanduser().resolve().parent + elif state_dir: + root = Path(state_dir).expanduser().resolve() + else: + root = (Path.home() / ".engraphis").resolve() + return str(root / "polar-webhooks.db") + except (OSError, RuntimeError) as exc: + raise WebhookStateError("could not resolve vendor webhook state path") from exc return None @@ -99,7 +167,16 @@ def _dedup_conn() -> Optional[sqlite3.Connection]: return None conn = None try: - Path(path).parent.mkdir(parents=True, exist_ok=True) + if path != ":memory:": + database = Path(path).expanduser() + database.parent.mkdir(parents=True, exist_ok=True, mode=0o700) + descriptor = os.open(str(database), os.O_RDWR | os.O_CREAT, 0o600) + os.close(descriptor) + try: + os.chmod(database, 0o600) + except OSError: + pass + path = str(database) conn = sqlite3.connect(path, timeout=30) conn.execute("PRAGMA journal_mode=WAL") conn.execute( @@ -126,10 +203,6 @@ def _dedup_conn() -> Optional[sqlite3.Connection]: conn.execute( "ALTER TABLE subscription_seats ADD COLUMN event_ts REAL") conn.commit() - try: - os.chmod(path, 0o600) - except OSError: - pass return conn except (OSError, sqlite3.Error) as exc: if conn is not None: @@ -137,6 +210,50 @@ def _dedup_conn() -> Optional[sqlite3.Connection]: raise WebhookStateError("durable webhook state store unavailable") from exc +def webhook_state_ready(*, require_durable: bool = False) -> bool: + """Return whether the Polar ledger can acquire a durable write transaction. + + Vendor readiness calls this with ``require_durable=True``. Merely resolving a path is + insufficient: a missing/unmounted/read-only volume must hold readiness closed before + checkout traffic is enabled. + """ + conn = None + try: + path = _dedup_path() + if not path: + return not require_durable + conn = _dedup_conn() + if conn is None: + return not require_durable + conn.execute("BEGIN IMMEDIATE") + conn.execute("SELECT 1 FROM processed LIMIT 1").fetchone() + conn.execute("ROLLBACK") + return True + except (OSError, sqlite3.Error, WebhookStateError): + return False + finally: + if conn is not None: + conn.close() + + +def webhook_backlog_healthy() -> bool: + """Return false when a Polar delivery is stuck beyond its processing lease.""" + conn = None + try: + conn = _dedup_conn() + if conn is None: + return False + stale = int(conn.execute( + "SELECT COUNT(*) FROM processed WHERE state='processing' AND ts<=?", + (time.time() - _RESERVATION_TTL_SECONDS,)).fetchone()[0]) + return stale == 0 + except (OSError, sqlite3.Error, WebhookStateError): + return False + finally: + if conn is not None: + conn.close() + + def claim_webhook(webhook_id: str) -> str: """Atomically determine this delivery's state and claim it if free. @@ -292,7 +409,8 @@ def record_known_seats(subscription_id: str, seats: int, conn.close() def _finalize_webhook(delivery_id: str, fulfillment_id: str, - seat_baseline: Optional[tuple] = None) -> None: + seat_baseline: Optional[tuple] = None, + transient_claim: str = "") -> None: """Persist the seat baseline and complete both claims in one transaction. ``seat_baseline`` is ``(subscription_id, seats)`` or ``(subscription_id, seats, @@ -320,6 +438,13 @@ def _finalize_webhook(delivery_id: str, fulfillment_id: str, (now, claim_id)) if cur.rowcount != 1: raise WebhookStateError("webhook claim was not pending") + if transient_claim: + cur = conn.execute( + "DELETE FROM processed WHERE webhook_id=? AND state='processing'", + (transient_claim,), + ) + if cur.rowcount != 1: + raise WebhookStateError("transient webhook lock was not pending") except sqlite3.Error as exc: raise WebhookStateError("could not atomically finalize webhook") from exc finally: @@ -328,10 +453,14 @@ def _finalize_webhook(delivery_id: str, fulfillment_id: str, def _release_claims(*claim_ids: str) -> None: """Best-effort rollback used only while returning a retryable failure.""" for claim_id in claim_ids: + if not claim_id: + continue try: release_webhook(claim_id) - except WebhookStateError: - logger.exception("polar webhook: could not release claim %s", claim_id) + except WebhookStateError as exc: + logger.error( + "polar webhook: could not release claim ref=%s (%s)", + _log_ref(claim_id), type(exc).__name__) def _subscription_id(data: dict) -> str: @@ -366,7 +495,11 @@ def _event_organization_id(event: dict, data: dict) -> str: """Best-effort extraction of the Polar organization id from an event, checked in the locations Polar populates across order/subscription payloads.""" product = data.get("product") or {} + if not isinstance(product, dict): + product = {} subscription = data.get("subscription") or {} + if not isinstance(subscription, dict): + subscription = {} for candidate in ( data.get("organization_id"), event.get("organization_id"), @@ -378,19 +511,23 @@ def _event_organization_id(event: dict, data: dict) -> str: return "" -def _organization_mismatch(event: dict, data: dict) -> bool: - """True only when ``POLAR_ORGANIZATION_ID`` is configured AND the event carries a - DIFFERENT organization id. The check is documented (webhooks.py env docs) but was - never enforced. Fails closed on a concrete mismatch; when the env var is unset, or - the payload carries no org id to compare, it does not block fulfillment (so a - payload-shape assumption can never strand a real purchase).""" +def _organization_mismatch(event: dict, data: dict, *, require_present: bool = False) -> bool: + """Compare the signed event organization with ``POLAR_ORGANIZATION_ID``. + + Combined development mode preserves compatibility when a fixture omits the + organization. The vendor service passes ``require_present=True`` and therefore + accepts only an exact, present organization id. + """ expected = os.environ.get("POLAR_ORGANIZATION_ID", "").strip() if not expected: return False found = _event_organization_id(event, data) if not found: + if require_present: + logger.warning("polar webhook: event carries no organization id") + return True logger.warning("polar webhook: POLAR_ORGANIZATION_ID set but event carries no " - "organization id to verify — proceeding") + "organization id to verify — combined-mode compatibility") return False return not hmac.compare_digest(found, expected) @@ -417,7 +554,7 @@ async def polar_webhook(request: Request): it to the buyer. Signature is verified against ``POLAR_WEBHOOK_SECRET``. 202 on success (and on ignored/duplicate events), 400 on unparsable input, - 403 on bad signature/timestamp, 500 on misconfiguration or fulfillment error. + 403 on bad signature/timestamp, and 5xx on configuration or fulfillment errors. """ secret = os.environ.get("POLAR_WEBHOOK_SECRET", "").strip() if not secret: @@ -427,7 +564,9 @@ async def polar_webhook(request: Request): try: content_length = int(request.headers.get("content-length") or 0) except ValueError: - content_length = 0 + return JSONResponse({"error": "invalid content length"}, status_code=400) + if content_length < 0: + return JSONResponse({"error": "invalid content length"}, status_code=400) if content_length > _MAX_BODY_BYTES: return JSONResponse({"error": "payload too large"}, status_code=413) @@ -437,7 +576,6 @@ async def polar_webhook(request: Request): return JSONResponse({"error": "payload too large"}, status_code=413) body.extend(chunk) raw_body = bytes(body) - body_str = raw_body.decode("utf-8", errors="replace") webhook_id = request.headers.get("webhook-id", "") timestamp = request.headers.get("webhook-timestamp", "") @@ -445,58 +583,112 @@ async def polar_webhook(request: Request): if not webhook_id or not timestamp or not signature_header: return JSONResponse({"error": "missing webhook headers"}, status_code=400) + if len(webhook_id) > 255 or len(timestamp) > 32 or len(signature_header) > 4096: + return JSONResponse({"error": "webhook headers too large"}, status_code=400) try: ts = float(timestamp) except ValueError: return JSONResponse({"error": "invalid webhook timestamp"}, status_code=400) + if not math.isfinite(ts): + return JSONResponse({"error": "invalid webhook timestamp"}, status_code=400) if abs(time.time() - ts) > _TIMESTAMP_TOLERANCE: logger.warning("polar webhook: timestamp outside %ds tolerance", _TIMESTAMP_TOLERANCE) return JSONResponse({"error": "webhook timestamp outside tolerance"}, status_code=403) - try: - secret_bytes = _decode_webhook_secret(secret) - except (binascii.Error, ValueError): + secret_candidates = tuple( + candidate for candidate in _webhook_secret_candidates(secret) + if len(candidate) >= 16) + if not secret_candidates: return JSONResponse( - {"error": "POLAR_WEBHOOK_SECRET is not valid base64"}, status_code=500) + {"error": "POLAR_WEBHOOK_SECRET is invalid"}, status_code=500) signed_content = f"{webhook_id}.{timestamp}.".encode("utf-8") + raw_body - expected_digest = hmac.new(secret_bytes, signed_content, hashlib.sha256).digest() - expected_b64 = base64.b64encode(expected_digest).decode("ascii") + expected_values = { + base64.b64encode( + hmac.new(candidate, signed_content, hashlib.sha256).digest()).decode("ascii") + for candidate in secret_candidates + } # webhook-signature is space-separated "v1," pairs (key rotation). Accept # a match against ANY listed signature. presented = [] for token in signature_header.split(): parts = token.split(",", 1) - presented.append(parts[1].strip() if len(parts) == 2 else parts[0].strip()) + if len(parts) == 2 and parts[0].strip() == "v1" and parts[1].strip(): + presented.append(parts[1].strip()) if not presented: return JSONResponse({"error": "invalid signature format"}, status_code=403) - if not any(hmac.compare_digest(expected_b64, p) for p in presented): + if not any( + hmac.compare_digest(expected, supplied) + for expected in expected_values for supplied in presented): logger.warning("polar webhook: invalid signature") return JSONResponse({"error": "invalid signature"}, status_code=403) try: - event = json.loads(body_str) + event = json.loads(raw_body) except (json.JSONDecodeError, UnicodeDecodeError, RecursionError): return JSONResponse({"error": "invalid JSON"}, status_code=400) + if not isinstance(event, dict): + return JSONResponse({"error": "webhook event must be an object"}, status_code=400) + if not isinstance(event.get("type"), str): + return JSONResponse({"error": "webhook event type must be a string"}, status_code=400) + data = event.get("data") + if not isinstance(data, dict): + return JSONResponse({"error": "webhook event data must be an object"}, + status_code=400) + event_type = (event.get("type") or "").strip() - data = event.get("data") or {} + + # The isolated production control plane accepts only the configured organization and + # exact product ids. ``combined`` remains a developer compatibility mode for fixtures + # and local testing, where the historical product-name mapping is still available. + from engraphis.commercial import extract_product_id, product_for_id, service_mode + vendor_mode = service_mode() == "vendor" + if vendor_mode and not os.environ.get("POLAR_ORGANIZATION_ID", "").strip(): + return JSONResponse({"error": "Polar organization is not configured"}, + status_code=503) # Reject a signed event from a DIFFERENT Polar organization when POLAR_ORGANIZATION_ID # is configured (the documented-but-previously-unenforced control). No-op when unset. - if _organization_mismatch(event, data): + if _organization_mismatch(event, data, require_present=vendor_mode): logger.warning("polar webhook: organization id mismatch — rejecting") return JSONResponse({"error": "organization mismatch"}, status_code=403) + event_status = str(data.get("status", "")).strip().lower() + positive_event = event_type == "order.paid" or ( + event_type == "subscription.updated" and event_status == "active") + if vendor_mode and positive_event: + product_id = extract_product_id(data) + if not product_id or product_for_id(product_id) is None: + logger.error("polar webhook: unrecognized product id") + return JSONResponse({"error": "unrecognized product"}, status_code=403) + if event_type == "order.paid" and not (_order_id(data) or _subscription_id(data)): + # Do not mint a paid key that no later refund/revocation event can address. + # Real Polar Order payloads carry an id; this is a malformed signed event and + # must remain visible for operator/manual fulfillment rather than silently + # creating an ungovernable entitlement keyed only by a delivery id. + logger.error("polar webhook: paid order carries no fulfillment identity") + return JSONResponse( + {"error": "paid order carries no order or subscription id"}, + status_code=400) + + # Trials are application-issued and card-free in GA. Polar trial subscriptions are + # deliberately not a second entitlement path on the production control plane. + if vendor_mode and event_type == "subscription.created" \ + and str(data.get("status", "")).strip().lower() == "trialing": + return JSONResponse({"status": "ignored", "reason": "application trial only", + "type": event_type}, status_code=202) + # Negative lifecycle. Refunds revoke immediately; ordinary cancel-at-period-end # intentionally does NOT revoke because the customer keeps the paid period. if event_type in ("subscription.canceled", "subscription.cancelled"): return JSONResponse({"status": "ignored", "reason": "paid period honored", "type": event_type}, status_code=202) - if event_type in _REVOKING_EVENTS: + if event_type in _REVOKING_EVENTS or ( + event_type == "subscription.updated" and event_status == "revoked"): sub_id = _subscription_id(data) if not sub_id and event_type.startswith("subscription."): sub_id = str(data.get("id") or "").strip()[:128] @@ -522,10 +714,14 @@ async def polar_webhook(request: Request): unmappable_claim = "unmappable:" + webhook_id unmappable_state = claim_webhook(unmappable_claim) if unmappable_state == "claimed": + # Persist that this exact signed delivery already received its one + # retryable response. Otherwise a retry after the processing TTL would + # look first-seen forever and never converge. + complete_webhook(unmappable_claim) return JSONResponse( {"error": "missing revoke target", "type": event_type}, status_code=503) - if unmappable_state != "fulfilled": + if unmappable_state == "in_flight": # Latch the claim so any FURTHER redelivery short-circuits straight to # "fulfilled" above. Guarded because complete_webhook only accepts a claim # that is still pending — calling it on an already-fulfilled one raises @@ -542,12 +738,15 @@ async def polar_webhook(request: Request): else: revoked = await asyncio.to_thread(_reg.revoke_by_order, order_id) target = {"order_id": order_id} - except Exception: # noqa: BLE001 — retryable: let Polar redeliver the revoke - logger.exception("polar webhook: revocation failed for %s", sub_id or order_id) + except Exception as exc: # noqa: BLE001 — retryable: let Polar redeliver the revoke + logger.error( + "polar webhook: revocation failed target_ref=%s (%s)", + _log_ref(sub_id or order_id), type(exc).__name__) return JSONResponse({"error": "revocation failed"}, status_code=503) reason = "refund" if event_type == "order.refunded" else "subscription_revoked" - logger.info("polar webhook: %s revoked %d key(s) for %s", - event_type, revoked, target) + logger.info( + "polar webhook: %s revoked %d key(s) target_ref=%s", + event_type, revoked, _log_ref(sub_id or order_id)) return JSONResponse({"status": "revoked", "reason": reason, "revoked": revoked, "keys_revoked": revoked, **target}, status_code=202) @@ -566,6 +765,7 @@ async def polar_webhook(request: Request): # A non-trial subscription.created is a no-op: its paid key comes from order.paid, so # a canceled trial can never keep Pro — the short trial key just expires. pending_seat_baseline = None # (sub_id, seats, event_ts); persisted after key issuance + seat_lock_claim = "" if event_type == "order.paid": from engraphis.inspector.webhooks import ( _extract_seats, handle_order_paid as _fulfill) @@ -586,37 +786,51 @@ async def polar_webhook(request: Request): fulfillment_key = "trial:" + sub_id pending_seat_baseline = (sub_id, _extract_seats(data), _event_modified_at(data)) elif event_type == "subscription.updated": - status = str(data.get("status", "")).strip().lower() + status = event_status sub_id = str(data.get("id") or "").strip()[:128] - if status == "revoked": - try: - from engraphis.inspector import license_registry as _reg - revoked = await asyncio.to_thread(_reg.revoke_by_subscription, sub_id) - except Exception: # noqa: BLE001 — retryable: let Polar redeliver the revoke - logger.exception("polar webhook: revocation failed for %s", sub_id) - return JSONResponse({"error": "revocation failed"}, status_code=503) - return JSONResponse({"status": "revoked", "reason": "subscription_revoked", - "revoked": revoked, "subscription_id": sub_id}, - status_code=202) if status != "active" or not sub_id: return JSONResponse({"status": "ignored", "reason": "not an active " "subscription", "type": event_type}, status_code=202) + # Different subscription.updated deliveries have different idempotency keys, so + # delivery-level claims do not serialize them. Hold one durable per-subscription + # mutex from baseline read through issuance/finalization: otherwise an older and + # newer seat update can both mint, and whichever finishes last revokes the correct + # replacement. A concurrent caller gets a retryable response and re-evaluates the + # now-current baseline on redelivery. + seat_lock_claim = "seatlock:" + sub_id + try: + seat_lock_state = claim_webhook(seat_lock_claim) + except WebhookStateError as exc: + logger.error( + "polar webhook: could not reserve subscription seat lock (%s)", + type(exc).__name__) + return JSONResponse({"error": "webhook state unavailable"}, status_code=503) + if seat_lock_state != "claimed": + return JSONResponse({"status": "processing", "key_issued": False}, + status_code=503) from engraphis.inspector.webhooks import _extract_seats new_seats = _extract_seats(data) event_ts = _event_modified_at(data) try: prior = get_seat_baseline(sub_id) - except WebhookStateError: - logger.exception("polar webhook: could not read seat baseline") + except WebhookStateError as exc: + _release_claims(seat_lock_claim) + logger.error( + "polar webhook: could not read seat baseline (%s)", + type(exc).__name__) return JSONResponse({"error": "webhook state unavailable"}, status_code=503) if prior is None: # First sighting seeds the baseline; the initial paid key came from order.paid. try: persisted = record_known_seats(sub_id, new_seats, event_ts) - except WebhookStateError: - logger.exception("polar webhook: could not seed seat baseline") + except WebhookStateError as exc: + _release_claims(seat_lock_claim) + logger.error( + "polar webhook: could not seed seat baseline (%s)", + type(exc).__name__) return JSONResponse({"error": "webhook state unavailable"}, status_code=503) reason = "baseline recorded" if persisted else "durable baseline unavailable" + _release_claims(seat_lock_claim) return JSONResponse({"status": "ignored", "reason": reason, "type": event_type}, status_code=202) prior_seats, prior_ts = prior @@ -625,6 +839,7 @@ async def polar_webhook(request: Request): # not regress a newer one (and revoke the correct key). Only applies when both # timestamps are known; without them we fall back to seat-count comparison. if event_ts is not None and prior_ts is not None and event_ts <= prior_ts: + _release_claims(seat_lock_claim) return JSONResponse({"status": "ignored", "reason": "out-of-order update", "type": event_type}, status_code=202) if prior_seats == new_seats: @@ -633,10 +848,14 @@ async def polar_webhook(request: Request): if event_ts is not None and (prior_ts is None or event_ts > prior_ts): try: record_known_seats(sub_id, new_seats, event_ts) - except WebhookStateError: - logger.exception("polar webhook: could not advance seat anchor") + except WebhookStateError as exc: + _release_claims(seat_lock_claim) + logger.error( + "polar webhook: could not advance seat anchor (%s)", + type(exc).__name__) return JSONResponse({"error": "webhook state unavailable"}, status_code=503) + _release_claims(seat_lock_claim) return JSONResponse({"status": "ignored", "reason": "no seat-count change", "type": event_type}, status_code=202) pending_seat_baseline = (sub_id, new_seats, event_ts) @@ -660,40 +879,53 @@ async def polar_webhook(request: Request): try: delivery_state = claim_webhook(delivery_claim) if delivery_state == "fulfilled": - logger.info("polar webhook: duplicate delivery %s ignored", webhook_id) + _release_claims(seat_lock_claim) + logger.info( + "polar webhook: duplicate delivery ref=%s ignored", _log_ref(webhook_id)) return JSONResponse( {"status": "duplicate", "key_issued": False}, status_code=202) if delivery_state == "in_flight": - logger.info("polar webhook: delivery %s already in flight — retry later", - webhook_id) + _release_claims(seat_lock_claim) + logger.info( + "polar webhook: delivery ref=%s already in flight — retry later", + _log_ref(webhook_id)) return JSONResponse({"status": "processing", "key_issued": False}, status_code=503) delivery_reserved = True fulfillment_state = claim_webhook(fulfillment_claim) if fulfillment_state == "fulfilled": complete_webhook(delivery_claim) - logger.info("polar webhook: %s already fulfilled — no second key", fulfillment_key) + _release_claims(seat_lock_claim) + logger.info( + "polar webhook: fulfillment ref=%s already fulfilled — no second key", + _log_ref(fulfillment_key)) return JSONResponse({"status": "already_fulfilled", "key_issued": False}, status_code=202) if fulfillment_state == "in_flight": # A concurrent delivery for the same order/trial/version is minting the key. # Release our delivery claim and have Polar retry — by then it's fulfilled. - _release_claims(delivery_claim) - logger.info("polar webhook: %s fulfillment in flight — retry later", - fulfillment_key) + _release_claims(delivery_claim, seat_lock_claim) + logger.info( + "polar webhook: fulfillment ref=%s in flight — retry later", + _log_ref(fulfillment_key)) return JSONResponse({"status": "processing", "key_issued": False}, status_code=503) - except WebhookStateError: + except WebhookStateError as exc: if delivery_reserved: _release_claims(delivery_claim) - logger.exception("polar webhook: durable reservation failed") + _release_claims(seat_lock_claim) + logger.error("polar webhook: durable reservation failed (%s)", type(exc).__name__) return JSONResponse({"error": "webhook state unavailable"}, status_code=503) try: # Blocking work (Ed25519 sign + email) runs off the event loop. - key = await asyncio.to_thread(_fulfill, data) + fulfillment_data = dict(data) + # The fulfillment handler uses this server-derived, signature-covered identity + # only for durable email/key idempotency. Overwrite any caller-supplied field. + fulfillment_data["_engraphis_fulfillment_id"] = fulfillment_key + key = await asyncio.to_thread(_fulfill, fulfillment_data) except Exception as exc: # noqa: BLE001 - external-provider boundary - _release_claims(delivery_claim, fulfillment_claim) + _release_claims(delivery_claim, fulfillment_claim, seat_lock_claim) logger.error("polar webhook: fulfillment failed (%s)", type(exc).__name__) return JSONResponse({"error": "license fulfillment failed; retry delivery"}, status_code=500) @@ -701,15 +933,18 @@ async def polar_webhook(request: Request): if not key: # Nothing issued (missing email) — release the claims so a corrected delivery # isn't permanently suppressed. - _release_claims(delivery_claim, fulfillment_claim) - return JSONResponse({"status": "fulfilled", "key_issued": False}, status_code=202) + _release_claims(delivery_claim, fulfillment_claim, seat_lock_claim) + return JSONResponse( + {"error": "license fulfillment incomplete; retry delivery"}, + status_code=503) try: # Baseline advancement and both durable completion markers share one commit. - _finalize_webhook(delivery_claim, fulfillment_claim, pending_seat_baseline) - except WebhookStateError: - _release_claims(delivery_claim, fulfillment_claim) - logger.exception("polar webhook: durable finalization failed") + _finalize_webhook( + delivery_claim, fulfillment_claim, pending_seat_baseline, seat_lock_claim) + except WebhookStateError as exc: + _release_claims(delivery_claim, fulfillment_claim, seat_lock_claim) + logger.error("polar webhook: durable finalization failed (%s)", type(exc).__name__) return JSONResponse({"error": "webhook state unavailable"}, status_code=503) - logger.info("polar webhook: issued key for %s", fulfillment_key) + logger.info("polar webhook: issued key fulfillment_ref=%s", _log_ref(fulfillment_key)) return JSONResponse({"status": "fulfilled", "key_issued": True}, status_code=202) diff --git a/engraphis/cloud_license.py b/engraphis/cloud_license.py index 396cec4..a85fadd 100644 --- a/engraphis/cloud_license.py +++ b/engraphis/cloud_license.py @@ -19,7 +19,7 @@ Online-only enforcement (product policy): paid features ALWAYS require a live lease. The client resolves a server URL from ``ENGRAPHIS_CLOUD_URL`` -> the key's signed ``cloud_url`` --> the built-in vendor relay (``settings.relay_url``), and fails closed if none resolves. +-> ``https://license.engraphis.com``, and fails closed if none resolves. There is deliberately no offline unlock path for Pro/Team. """ from __future__ import annotations @@ -89,8 +89,8 @@ def validate_cloud_base_url(value: str) -> str: raise ValueError("license server URL must be an absolute http(s) URL") try: parts.port - except ValueError as exc: - raise ValueError("license server URL has an invalid port") from exc + except ValueError: + raise ValueError("license server URL has an invalid port") from None if parts.username is not None or parts.password is not None: raise ValueError("license server URL must not contain embedded credentials") if "\\" in parts.netloc or any(char.isspace() for char in parts.netloc): @@ -176,11 +176,10 @@ def machine_id() -> str: pass except OSError as exc: logger.warning( - "machine_id: could not persist device id to %s (%s); using an in-process " - "id for this run. Trial and cloud-lease binding need a writable home " - "directory — mount a persistent volume for %s (or set HOME to a writable " - "path).", - _MACHINE_ID_FILE, exc, _MACHINE_ID_FILE.parent) + "machine_id: could not persist device id (%s); using an in-process id " + "for this run. Trial and cloud-lease binding need a writable persistent " + "state directory.", + type(exc).__name__) _machine_id_cache[key] = mid # stable for the rest of the process regardless return mid @@ -188,20 +187,38 @@ def machine_id() -> str: # ── lease token (same ENGR1-style envelope, distinct prefix) ───────────────────────── def compose_lease(payload: dict, secret: bytes) -> str: - """Vendor-side: sign a lease payload. (Server imports this.)""" - from engraphis.licensing import _b64u_encode, ed25519_sign - body = json.dumps(payload, separators=(",", ":"), sort_keys=True).encode("utf-8") + """Vendor-side: sign a lease payload and bind it to its signing-key id. + + The payload copy keeps the caller's object immutable. A supplied, mismatched key id + is rejected rather than silently corrected so a bad rotation configuration cannot + mint a lease whose metadata disagrees with its signature. + """ + from engraphis.licensing import _b64u_encode, ed25519_public_key, ed25519_sign + if not isinstance(payload, dict): + # Preserve the old ability to compose malformed signed values for verifier tests. + body = json.dumps(payload, separators=(",", ":"), sort_keys=True).encode("utf-8") + sig = ed25519_sign(secret, body) + return "%s.%s.%s" % (_LEASE_PREFIX, _b64u_encode(body), _b64u_encode(sig)) + signed_payload = dict(payload) + signing_key_id = ed25519_public_key(secret).hex()[:16] + supplied = str(signed_payload.get("signing_key_id", "") or "").strip().lower() + if supplied and supplied != signing_key_id: + raise ValueError("lease signing-key id does not match the signing secret") + signed_payload["signing_key_id"] = signing_key_id + body = json.dumps( + signed_payload, separators=(",", ":"), sort_keys=True).encode("utf-8") sig = ed25519_sign(secret, body) return "%s.%s.%s" % (_LEASE_PREFIX, _b64u_encode(body), _b64u_encode(sig)) def verify_lease(token: str, *, now: Optional[float] = None) -> dict: - """Verify a lease against the pinned vendor public key. Raise LicenseError if bad. + """Verify a lease against the pinned current/rotation keys. Raise LicenseError if bad. Checks the signature and that the lease has not expired (using the monotonic clock, so a rolled-back system clock cannot resurrect an expired lease).""" from engraphis.licensing import ( - LicenseError, _b64u_decode, ed25519_verify, vendor_public_key, _monotonic_now, + LicenseError, _b64u_decode, _monotonic_now, ed25519_verify, + vendor_public_keys, ) token = (token or "").strip() parts = token.split(".") @@ -212,7 +229,9 @@ def verify_lease(token: str, *, now: Optional[float] = None) -> dict: sig = _b64u_decode(parts[2]) except Exception: raise LicenseError("lease is not valid base64url") - if not ed25519_verify(vendor_public_key(), body, sig): + verified_with = next( + (pub for pub in vendor_public_keys() if ed25519_verify(pub, body, sig)), None) + if verified_with is None: raise LicenseError("lease signature is invalid (tampered or wrong vendor key)") try: payload = json.loads(body.decode("utf-8")) @@ -225,6 +244,14 @@ def verify_lease(token: str, *, now: Optional[float] = None) -> dict: # not — mirror parse_key's isinstance guard rather than rely on that. if not isinstance(payload, dict): raise LicenseError("lease payload is not a JSON object") + verified_key_id = verified_with.hex()[:16] + signing_key_id = str(payload.get("signing_key_id", "") or "").strip().lower() + if signing_key_id and signing_key_id != verified_key_id: + raise LicenseError("lease signing-key id does not match its signature") + # Legacy cached leases did not carry a key id. They remain verifiable only while + # their signer is present in the explicitly pinned dual-key set; surface the derived + # id to callers so all accepted leases have one consistent representation. + payload["signing_key_id"] = verified_key_id exp = payload.get("expires") now = _monotonic_now() if now is None else now if exp is None: @@ -324,12 +351,12 @@ def register(base_url: str, key: str, mid: str, *, timeout: float = _REGISTER_TI def send_team_invite(base_url: str, key: str, to: str, name: str, role: str, - invited_by: str, *, dashboard_url: str = "", + invited_by: str, *, dashboard_url: str = "", invite_url: str = "", timeout: float = _INVITE_TIMEOUT) -> Tuple[bool, str]: - """POST a team-invite request to the vendor relay's ``/license/v1/team-invite``. + """POST a team-invite request to the vendor control plane's ``/license/v1/team-invite``. Used by a self-hosted Team dashboard (``routes.v2_team.add_user``) that has no - email delivery of its own configured — the vendor relay sends the notification + email delivery of its own configured — the vendor control plane sends the notification through ITS configured mail provider instead, gated server-side by *key* actually carrying the ``team`` feature (see ``inspector.license_cloud.team_invite``). Returns ``(sent, reason)``: on any @@ -344,7 +371,8 @@ def send_team_invite(base_url: str, key: str, to: str, name: str, role: str, return False, "relay URL must use HTTPS (except loopback) and contain no credentials" url = base + "/license/v1/team-invite" data = json.dumps({"key": key, "to": to, "name": name, "role": role, - "invited_by": invited_by, "dashboard_url": dashboard_url} + "invited_by": invited_by, "dashboard_url": dashboard_url, + "invite_url": invite_url} ).encode("utf-8") req = urllib.request.Request( url, data=data, method="POST", headers=_JSON_HEADERS) @@ -365,12 +393,49 @@ def send_team_invite(base_url: str, key: str, to: str, name: str, role: str, return False, "daily relayed-invite limit reached; retry tomorrow" return False, "relay rejected invite (HTTP %d)" % exc.code except (urllib.error.URLError, ValueError, TimeoutError, OSError): - return False, "relay is unreachable; check ENGRAPHIS_RELAY_URL and the network" + return False, "license service is unreachable; check ENGRAPHIS_CLOUD_URL and the network" + + +def send_password_reset(base_url: str, key: str, to: str, name: str, reset_url: str, + *, timeout: float = _INVITE_TIMEOUT) -> Tuple[bool, str]: + """Ask the control plane to durably queue a deployment-bound reset email. + + The reset token is sent only in the server-to-server request. It is never returned + by the control plane or included in this function's failure reason. + """ + try: + base = validate_cloud_base_url(base_url) + except ValueError: + return False, "license server URL is invalid" + data = json.dumps({ + "key": key, + "to": to, + "name": name, + "reset_url": reset_url, + }).encode("utf-8") + req = urllib.request.Request( + base + "/license/v1/password-reset", data=data, method="POST", + headers=_JSON_HEADERS, + ) + try: + with urllib.request.urlopen(req, timeout=timeout) as resp: # nosec B310 + body = json.loads(resp.read().decode("utf-8")) + return bool(body.get("queued")), "" + except urllib.error.HTTPError as exc: + if exc.code == 402: + return False, "an active Pro or Team license is required" + if exc.code == 409: + return False, "dashboard origin does not match this license" + if exc.code == 429: + return False, "password-reset email rate limit reached" + return False, "license service rejected password-reset delivery" + except (urllib.error.URLError, ValueError, TimeoutError, OSError): + return False, "license service is unreachable" def request_trial_key(base_url: str, mid: str, plan: str = "team", email: str = "", *, timeout: float = _INVITE_TIMEOUT) -> Tuple[Optional[str], str, bool]: - """POST to the vendor relay's self-serve ``/license/v1/start-trial`` and return + """POST to the vendor control plane's deprecated ``/license/v1/start-trial`` and return ``(key, reason, pending)``. Since 2026-07-14 the relay no longer issues a key synchronously from this call — @@ -408,9 +473,9 @@ def request_trial_key(base_url: str, mid: str, plan: str = "team", email: str = return None, "the free trial has already been used on this device", False if exc.code == 429: return None, "too many trial requests; try again later", False - return None, "trial relay rejected the request (HTTP %d)" % exc.code, False + return None, "trial control plane rejected the request (HTTP %d)" % exc.code, False except (urllib.error.URLError, ValueError, TimeoutError, OSError): - return None, ("trial relay is unreachable; check ENGRAPHIS_RELAY_URL and the " + return None, ("trial control plane is unreachable; check ENGRAPHIS_CLOUD_URL and the " "network"), False @@ -421,6 +486,66 @@ def request_team_trial_key(base_url: str, mid: str, email: str = "", *, return request_trial_key(base_url, mid, plan="team", email=email, timeout=timeout) +def create_trial_claim(base_url: str, deployment_token: str, mid: str, + email: str, plan: str, *, dashboard_url: str = "", + timeout: float = _INVITE_TIMEOUT) -> dict: + """Start an idempotent deployment-bound trial claim on the control plane.""" + base = validate_cloud_base_url(base_url) + data = json.dumps({ + "deployment_token": deployment_token, + "machine_id": mid, + "email": email, + "plan": plan, + "dashboard_url": dashboard_url, + }).encode("utf-8") + req = urllib.request.Request( + base + "/license/v1/trial-claims", data=data, method="POST", + headers=_JSON_HEADERS) + try: + with urllib.request.urlopen(req, timeout=timeout) as resp: # nosec B310 + try: + return json.loads(resp.read().decode("utf-8")) + except (ValueError, UnicodeDecodeError): + raise RuntimeError( + "trial control plane returned an invalid response" + ) from None + except urllib.error.HTTPError as exc: + # The control plane body is untrusted and may reflect email addresses, + # deployment tokens, or other request material. Expose only the status. + raise RuntimeError( + "trial control plane rejected the request (HTTP %d)" % exc.code + ) from None + except (urllib.error.URLError, ValueError, TimeoutError, OSError): + raise RuntimeError("trial control plane is unreachable") from None + + +def claim_trial(base_url: str, claim_id: str, deployment_token: str, mid: str, *, + timeout: float = _INVITE_TIMEOUT) -> dict: + """Retrieve a confirmed key server-to-server for its bound deployment.""" + base = validate_cloud_base_url(base_url) + data = json.dumps({"deployment_token": deployment_token, + "machine_id": mid}).encode("utf-8") + from urllib.parse import quote + url = base + "/license/v1/trial-claims/%s/claim" % quote(claim_id, safe="") + req = urllib.request.Request(url, data=data, method="POST", headers=_JSON_HEADERS) + try: + with urllib.request.urlopen(req, timeout=timeout) as resp: # nosec B310 + try: + return json.loads(resp.read().decode("utf-8")) + except (ValueError, UnicodeDecodeError): + raise RuntimeError( + "trial control plane returned an invalid response" + ) from None + except urllib.error.HTTPError as exc: + if exc.code == 503: + return {"status": "recovery_pending", "ready": False} + raise RuntimeError( + "trial control plane rejected the claim (HTTP %d)" % exc.code + ) from None + except (urllib.error.URLError, ValueError, TimeoutError, OSError): + raise RuntimeError("trial control plane is unreachable") from None + + def gate(lic, key_material: str, *, base_url: Optional[str] = None) -> Tuple[bool, str]: """Decide whether a validly-signed paid key may unlock features on this device. @@ -428,7 +553,7 @@ def gate(lic, key_material: str, *, base_url: Optional[str] = None) -> Tuple[boo override or the URL signed into the key) takes precedence over ``ENGRAPHIS_CLOUD_URL``. **With no server resolved at all this fails CLOSED** — there is deliberately no offline path to paid features (the caller, ``licensing._cloud_gate``, resolves a - default relay URL and also fails closed, so this only bites a deliberately blanked + default license-service URL and also fails closed, so this only bites a deliberately blanked config — defense in depth). In cloud mode, every cache refresh contacts the server: an authoritative denial fails closed immediately, while a transient network failure may use an existing unexpired lease as offline grace. Without such a lease, failure @@ -436,7 +561,7 @@ def gate(lic, key_material: str, *, base_url: Optional[str] = None) -> Tuple[boo base = (base_url or "").strip().rstrip("/") or cloud_url() if not base: # Online-only: no server to verify against ⇒ no offline path to paid features. - # (The caller, licensing._cloud_gate, already resolves a default relay URL, so + # (The caller, licensing._cloud_gate, already resolves a default license URL, so # this only bites a deliberately blanked config — defense in depth. Fail closed.) return False, ("server-side license verification is required but no license " "server is configured") @@ -465,8 +590,10 @@ def gate(lic, key_material: str, *, base_url: Optional[str] = None) -> Tuple[boo return True, "" if cached is not None: return True, "" - return False, ("cloud license verification failed — could not reach license server at %s " - "(offline or network error)" % base) + # The configured endpoint path can contain customer identifiers even after URL + # validation. Do not reflect it through license errors shown by the dashboard. + return False, ("cloud license verification failed — could not reach the configured " + "license server (offline or network error)") def revalidate(lic, key_material: str, *, base_url: Optional[str] = None) -> str: diff --git a/engraphis/commercial.py b/engraphis/commercial.py new file mode 100644 index 0000000..3fcc88f --- /dev/null +++ b/engraphis/commercial.py @@ -0,0 +1,290 @@ +"""Commercial control-plane configuration and release-readiness checks. + +This module contains no customer data and never returns secret material. It is shared by +the vendor-only ASGI app, billing webhook, release doctor, and tests so production gating +cannot drift between entrypoints. +""" +from __future__ import annotations + +import hmac +import hashlib +import json +import math +import os +import shutil +import time +from pathlib import Path +from typing import Optional + +from engraphis.config import SERVICE_MODES, settings + + +PRODUCT_ENV = { + "POLAR_PRO_MONTHLY_PRODUCT_ID": ("pro", "monthly"), + "POLAR_PRO_ANNUAL_PRODUCT_ID": ("pro", "annual"), + "POLAR_TEAM_MONTHLY_PRODUCT_ID": ("team", "monthly"), + "POLAR_TEAM_ANNUAL_PRODUCT_ID": ("team", "annual"), +} + + +def manifest() -> dict: + path = Path(__file__).with_name("commercial_manifest.json") + return json.loads(path.read_text(encoding="utf-8")) + + +def expected_product_ids() -> dict: + expected = {} + for plan_name in ("pro", "team"): + for interval, product in manifest()["plans"][plan_name]["products"].items(): + expected[product["env"]] = { + "id": product["id"], "plan": plan_name, "interval": interval} + return expected + + +def service_mode() -> str: + mode = (settings.service_mode or "").strip().lower() + if mode not in SERVICE_MODES: + raise RuntimeError( + "ENGRAPHIS_SERVICE_MODE must be one of: %s" % ", ".join(SERVICE_MODES)) + return mode + + +def vendor_admin_token_ready() -> bool: + """Require a bounded, high-entropy control-plane administrator credential.""" + token = os.environ.get("ENGRAPHIS_VENDOR_ADMIN_TOKEN", "").strip() + return 32 <= len(token) <= 4096 and all( + char.isascii() and 33 <= ord(char) < 127 for char in token) + + +def product_catalog() -> dict: + """Return exact configured Polar product ids without exposing any credential.""" + catalog = {} + expected = expected_product_ids() + for env_name, (plan, interval) in PRODUCT_ENV.items(): + product_id = os.environ.get(env_name, "").strip() + if product_id and expected.get(env_name, {}).get("id") == product_id: + catalog[product_id] = {"plan": plan, "interval": interval, + "env": env_name} + return catalog + + +def product_for_id(product_id: str) -> Optional[dict]: + return product_catalog().get(str(product_id or "").strip()) + + +def extract_product_id(data: dict) -> str: + """Handle the Polar order/subscription shapes used by signed webhooks.""" + if not isinstance(data, dict): + return "" + product = data.get("product") or {} + price = data.get("price") or {} + candidates = [ + data.get("product_id"), + product.get("id") if isinstance(product, dict) else product, + price.get("product_id") if isinstance(price, dict) else "", + ] + for candidate in candidates: + value = str(candidate or "").strip() + if value: + return value[:128] + return "" + + +def _signer_matches() -> bool: + try: + from engraphis.inspector.webhooks import _load_signing_secret + from engraphis.licensing import ed25519_public_key, vendor_public_key + actual = ed25519_public_key(_load_signing_secret()) + return hmac.compare_digest(actual, vendor_public_key()) + except Exception: + return False + + +def _registry_writable() -> bool: + try: + from engraphis.inspector import license_registry + conn = license_registry.connect() + try: + conn.execute("BEGIN IMMEDIATE") + conn.execute("SELECT 1").fetchone() + conn.execute("ROLLBACK") + finally: + conn.close() + return True + except Exception: + return False + + +def _disk_ok() -> bool: + try: + from engraphis.inspector import license_registry + db_path = Path(license_registry._db_path()).expanduser().resolve() + root = db_path.parent + root.mkdir(parents=True, exist_ok=True) + minimum = max(1, int(os.environ.get( + "ENGRAPHIS_VENDOR_MIN_FREE_BYTES", str(256 * 1024 * 1024)))) + return shutil.disk_usage(root).free >= minimum + except Exception: + return False + + +def _customer_disk_ok() -> bool: + try: + root = Path(settings.db_path).expanduser().resolve().parent + root.mkdir(parents=True, exist_ok=True) + minimum = max(1, int(os.environ.get( + "ENGRAPHIS_CUSTOMER_MIN_FREE_BYTES", str(256 * 1024 * 1024)))) + return shutil.disk_usage(root).free >= minimum + except Exception: + return False + + +def _backup_fresh() -> bool: + """Validate the marker and encrypted artifact written by the backup job. + + A status file is only an attestation, not proof: recompute the artifact digest and + require the file to live directly in the configured off-volume directory. This + prevents a copied/edited marker, a path traversal, or a same-size damaged archive + from holding production readiness open. + """ + marker = os.environ.get("ENGRAPHIS_BACKUP_STATUS_FILE", "").strip() + output = os.environ.get("ENGRAPHIS_BACKUP_OUTPUT_DIR", "").strip() + if not marker or not output: + return False + try: + maximum = max(60, int(os.environ.get( + "ENGRAPHIS_BACKUP_MAX_AGE_SECONDS", "93600"))) # 26 hours + marker_source = Path(marker).expanduser() + if marker_source.is_symlink(): + return False + marker_path = marker_source.resolve(strict=True) + if not marker_path.is_file(): + return False + output_path = Path(output).expanduser().resolve(strict=True) + if not output_path.is_dir(): + return False + status = json.loads(marker_path.read_text(encoding="utf-8")) + if not isinstance(status, dict) \ + or status.get("schema") != "engraphis-backup-status/v1": + return False + created_at = float(status["created_at"]) + artifact_source = Path(str(status["artifact"])).expanduser() + expected_size = int(status["bytes"]) + checksum = str(status["sha256"]) + if not artifact_source.is_absolute() or artifact_source.is_symlink() \ + or expected_size <= 0 \ + or len(checksum) != 64 or any(c not in "0123456789abcdef" for c in checksum): + return False + now = time.time() + age = now - created_at + if not math.isfinite(created_at) or not -300 <= age <= maximum: + return False + artifact = artifact_source.resolve(strict=True) + if artifact.parent != output_path or not artifact.is_file(): + return False + artifact_stat = artifact.stat() + marker_mtime = marker_path.stat().st_mtime + if abs(marker_mtime - created_at) > 300 \ + or abs(artifact_stat.st_mtime - created_at) > 300 \ + or artifact_stat.st_size != expected_size: + return False + digest = hashlib.sha256() + with artifact.open("rb") as fh: + for chunk in iter(lambda: fh.read(1024 * 1024), b""): + digest.update(chunk) + return hmac.compare_digest(digest.hexdigest(), checksum) + except (KeyError, OSError, TypeError, ValueError, json.JSONDecodeError): + return False + + +def run_configured_backup() -> dict: + """Create and verify one encrypted backup without exposing its path or key.""" + output = os.environ.get("ENGRAPHIS_BACKUP_OUTPUT_DIR", "").strip() + marker = os.environ.get("ENGRAPHIS_BACKUP_STATUS_FILE", "").strip() + if not output or not marker: + raise RuntimeError("off-volume backup storage is not configured") + try: + retention = max(1, int(os.environ.get( + "ENGRAPHIS_BACKUP_RETENTION_DAYS", "30"))) + except ValueError as exc: + raise RuntimeError("backup retention is invalid") from exc + from scripts.commercial_backup import backup + try: + artifact = backup(Path(output), Path(marker), retention, False) + except SystemExit as exc: + raise RuntimeError("backup configuration was rejected") from exc + return { + "ok": bool(artifact.is_file()), + "verified": bool(artifact.is_file() and _backup_fresh()), + } + + +def customer_operations_readiness() -> dict: + """Secret-free managed-customer storage readiness for authenticated monitoring.""" + checks = { + "service_mode": service_mode() == "customer", + "disk": _customer_disk_ok(), + "backup": _backup_fresh(), + } + checks["ready"] = all(checks.values()) + return checks + + +def vendor_serving_readiness() -> dict: + """Dependencies required to safely receive live licensing and billing traffic. + + This intentionally excludes backup freshness and SLO/alert signals. An orchestrator + draining a healthy process cannot repair those conditions and may instead deadlock a + first deployment before the authenticated backup endpoint can run. The full + operational gate remains :func:`vendor_readiness`. + """ + from engraphis.billing import webhook_secret_ready, webhook_state_ready + from engraphis.inspector.webhooks import email_configured + from engraphis.licensing import VENDOR_SIGNER_RELEASE_READY + + products = product_catalog() + checks = { + "service_mode": service_mode() == "vendor", + "signer": _signer_matches(), + "signer_release_ready": bool(VENDOR_SIGNER_RELEASE_READY), + "registry": _registry_writable(), + "polar_webhook": webhook_secret_ready(), + "polar_organization": bool(os.environ.get("POLAR_ORGANIZATION_ID", "").strip()), + "polar_products": len(products) == len(PRODUCT_ENV), + "polar_idempotency": webhook_state_ready(require_durable=True), + "email": bool(email_configured()), + "disk": _disk_ok(), + } + checks["ready"] = all(checks.values()) + return checks + + +def vendor_readiness() -> dict: + """Return the full secret-free operational release gate.""" + from engraphis.billing import webhook_backlog_healthy + from engraphis.email_outbox import health as email_outbox_health + from engraphis.inspector.license_registry import rejected_lease_health + from engraphis.inspector.webhooks import manual_fulfillment_clear + from engraphis.resend_events import webhook_secret_ready + + checks = vendor_serving_readiness() + checks.pop("ready", None) + try: + outbox_healthy = bool(email_outbox_health()["healthy"]) + except Exception: + outbox_healthy = False + checks.update({ + "vendor_admin_token": vendor_admin_token_ready(), + "polar_backlog": webhook_backlog_healthy(), + "rejected_leases": rejected_lease_health(), + "email_webhook": webhook_secret_ready(), + "email_outbox": outbox_healthy, + "manual_fulfillment": manual_fulfillment_clear(), + "backup": _backup_fresh(), + }) + # Invalid public registration attempts are an operator alert, not a dependency. + # Making this attacker-controlled signal a readiness gate lets anyone force the + # orchestrator to drain/restart a healthy licensing service by submitting bad keys. + checks["ready"] = all( + value for name, value in checks.items() if name != "rejected_leases") + return checks diff --git a/engraphis/commercial_manifest.json b/engraphis/commercial_manifest.json new file mode 100644 index 0000000..c6e2453 --- /dev/null +++ b/engraphis/commercial_manifest.json @@ -0,0 +1,63 @@ +{ + "schema": "engraphis-commercial/v1", + "version": "1.0.0", + "license_server": "https://license.engraphis.com", + "managed_dashboard": "https://team.engraphis.com", + "trial": { + "days": 3, + "card_required": false, + "plans": ["pro", "team"] + }, + "plans": { + "free": { + "monthly_usd": 0, + "annual_usd": 0, + "billing_unit": "installation" + }, + "pro": { + "monthly_usd": 10, + "annual_usd": 100, + "billing_unit": "installation", + "products": { + "monthly": { + "id": "6349d29a-b4c0-4b8e-b9e8-071705351c9c", + "env": "POLAR_PRO_MONTHLY_PRODUCT_ID", + "checkout_url": "https://buy.polar.sh/polar_cl_n6CR3ERqOus2VUhRrGrsRUqOB8yjDTeEU7p1r3CRrae?product_id=6349d29a-b4c0-4b8e-b9e8-071705351c9c" + }, + "annual": { + "id": "5e10d5d2-607a-4e9c-ad1c-88cb48c37e11", + "env": "POLAR_PRO_ANNUAL_PRODUCT_ID", + "checkout_url": "https://buy.polar.sh/polar_cl_n6CR3ERqOus2VUhRrGrsRUqOB8yjDTeEU7p1r3CRrae?product_id=5e10d5d2-607a-4e9c-ad1c-88cb48c37e11" + } + } + }, + "team": { + "monthly_usd": 20, + "annual_usd": 200, + "billing_unit": "seat", + "products": { + "monthly": { + "id": "6444a48a-7a61-4c8e-8045-07a930f8f381", + "env": "POLAR_TEAM_MONTHLY_PRODUCT_ID", + "checkout_url": "https://buy.polar.sh/polar_cl_n6CR3ERqOus2VUhRrGrsRUqOB8yjDTeEU7p1r3CRrae?product_id=6444a48a-7a61-4c8e-8045-07a930f8f381" + }, + "annual": { + "id": "929d926a-2981-4ad7-95bd-f52dabf0794e", + "env": "POLAR_TEAM_ANNUAL_PRODUCT_ID", + "checkout_url": "https://buy.polar.sh/polar_cl_n6CR3ERqOus2VUhRrGrsRUqOB8yjDTeEU7p1r3CRrae?product_id=929d926a-2981-4ad7-95bd-f52dabf0794e" + } + } + } + }, + "features": { + "offline_free_core": true, + "online_paid_license_verification": true, + "opt_in_cloud_sync": true, + "multi_user_roles": true, + "team_audit_export": true, + "per_user_agent_tokens": true, + "end_to_end_encrypted_sync": false, + "sso": false, + "contractual_sla": false + } +} diff --git a/engraphis/config.py b/engraphis/config.py index b6883e4..4f06c2f 100644 --- a/engraphis/config.py +++ b/engraphis/config.py @@ -3,6 +3,7 @@ import json import os +import re import sqlite3 import sys from contextlib import contextmanager @@ -221,10 +222,17 @@ def _configured_db_path(root: Path = _PROJECT_ROOT) -> str: return str(target) -#: Vendor-hosted managed service for sync, leases, trials, and invite delivery. This is -#: the default target when ``ENGRAPHIS_RELAY_URL`` is not overridden. +#: Vendor-hosted managed sync service. Customer deployments normally override this with +#: their own dashboard URL; local Pro clients retain the managed default. DEFAULT_RELAY_URL = "https://team.engraphis.com" +#: Isolated commercial control plane for paid-license leases, trials, fulfillment, and +#: transactional mail. Keeping this distinct from the dashboard removes the signing seed +#: and billing webhook secret from the customer-facing memory service. +DEFAULT_LICENSE_SERVER_URL = "https://license.engraphis.com" + +SERVICE_MODES = ("customer", "vendor", "combined") + # Keys issued before the custom domain migration carry this URL inside their signed # payload. Preserve the signature, but route that one retired vendor host to the current # managed service. Arbitrary signed URLs remain authoritative. @@ -232,6 +240,13 @@ def _configured_db_path(root: Path = _PROJECT_ROOT) -> str: "https://engraphis-production.up.railway.app", }) +# Existing signed keys point at the old combined host. License verification may migrate +# that exact vendor URL without altering arbitrary customer-signed endpoints. +RETIRED_LICENSE_SERVER_URLS = frozenset({ + "https://team.engraphis.com", + "https://engraphis-production.up.railway.app", +}) + def _env(key: str, default: str = "") -> str: return os.environ.get(key, default).strip() @@ -251,6 +266,73 @@ def _env_float(key: str, default: float) -> float: return default +def persist_project_env(values: dict[str, str], path: Optional[Path] = None) -> Path: + """Upsert non-secret runtime settings in the project-local ``.env`` atomically. + + Dashboard controls use this for settings that must survive a restart. Explicit + process-environment values still remain authoritative on the next launch because + python-dotenv loads with ``override=False``. + """ + target = Path(path) if path is not None else Path.cwd() / ".env" + clean: dict[str, str] = {} + for key, value in values.items(): + name = str(key or "").strip() + if not re.fullmatch(r"[A-Z][A-Z0-9_]*", name): + raise ValueError("environment setting names must be uppercase identifiers") + text = str(value) + if "\n" in text or "\r" in text: + raise ValueError("environment setting values must be single-line") + clean[name] = text + + existed = target.exists() + existing = target.read_text(encoding="utf-8") if existed else "" + # Replacing an existing .env through a fresh default-mode file can silently widen + # permissions from 0600 to 0644 while the preserved lines still contain API keys. + # Carry the original mode forward; new files start private regardless of umask. + try: + mode = target.stat().st_mode & 0o777 if existed else 0o600 + except OSError: + mode = 0o600 + lines = existing.splitlines() + found: set[str] = set() + rendered: list[str] = [] + for line in lines: + match = re.match(r"^(\s*)(?:export\s+)?([A-Z][A-Z0-9_]*)\s*=", line) + if match and match.group(2) in clean: + key = match.group(2) + if key not in found: + rendered.append(f"{match.group(1)}{key}={clean[key]}") + found.add(key) + continue + rendered.append(line) + if rendered and rendered[-1].strip(): + rendered.append("") + for key, value in clean.items(): + if key not in found: + rendered.append(f"{key}={value}") + + target.parent.mkdir(parents=True, exist_ok=True) + temporary = target.with_name( + f".{target.name}.tmp-{os.getpid()}-{uuid.uuid4().hex}" + ) + try: + with open(temporary, "w", encoding="utf-8", newline="\n") as handle: + handle.write("\n".join(rendered).rstrip() + "\n") + handle.flush() + os.fsync(handle.fileno()) + try: + os.chmod(temporary, mode) + except OSError: + pass + os.replace(temporary, target) + finally: + try: + temporary.unlink() + except FileNotFoundError: + pass + return target + + @dataclass class Settings: host: str = field(default_factory=lambda: _env("ENGRAPHIS_HOST", "127.0.0.1")) @@ -279,6 +361,13 @@ class Settings: not in ("0", "false", "no", "off") ) + # Production roles are isolated. ``combined`` preserves local development and legacy + # self-host behavior; the official Railway template sets ``customer`` and the vendor + # control plane sets ``vendor``. + service_mode: str = field( + default_factory=lambda: _env("ENGRAPHIS_SERVICE_MODE", "combined").lower() + ) + # Managed relay base URL. Client sync uses it when `--relay-url` is omitted, and paid # license flows fall back to it when a signed key or explicit cloud override supplies # no URL. Set an empty ENGRAPHIS_RELAY_URL to require an explicit target. @@ -314,6 +403,13 @@ class Settings: llm_extra_headers: dict = field( default_factory=lambda: _parse_headers(_env("ENGRAPHIS_LLM_EXTRA_HEADERS", "")) ) + # A successful dashboard connection test enables schema-validated extraction unless + # the user explicitly turned it off. The separate flag is what makes an automatic + # default and a durable on/off control compatible. + llm_auto_extract: bool = field( + default_factory=lambda: _env("ENGRAPHIS_LLM_AUTO_EXTRACT", "1").lower() + not in ("0", "false", "no", "off") + ) # Optional cross-encoder reranker model. Empty (default) -> IdentityReranker (offline). rerank_model: str = field(default_factory=lambda: _env("ENGRAPHIS_RERANK_MODEL", "")) @@ -322,6 +418,12 @@ class Settings: # heuristic NER, no API key, populated on every ingest; "none" disables graph # population. Defaults on so the Graph tab works out of the box for every install. graph_extractor: str = field(default_factory=lambda: _env("ENGRAPHIS_GRAPH_EXTRACTOR", "regex").lower()) + # Analytical Galaxy v2 is the validated default; setting the rollout flag to 0 + # restores the legacy ForceGraph surface for one compatibility release. + graph_ui_v2: bool = field( + default_factory=lambda: _env("ENGRAPHIS_GRAPH_UI_V2", "1").lower() + not in ("0", "false", "no", "off") + ) # Optional host-LLM importance/retention classification. "none" keeps the fully # deterministic local write path; "llm" asks the configured provider for a bounded @@ -348,6 +450,14 @@ def base_url(self) -> str: from engraphis.netutil import display_base_url return display_base_url(self.host, self.port) + @property + def customer_service(self) -> bool: + return self.service_mode in ("customer", "combined") + + @property + def vendor_service(self) -> bool: + return self.service_mode in ("vendor", "combined") + def _parse_headers(raw: str) -> dict: if not raw: @@ -382,9 +492,15 @@ def canonicalize_relay_url(url: str) -> str: return DEFAULT_RELAY_URL if normalized in RETIRED_RELAY_URLS else normalized +def canonicalize_license_server_url(url: str) -> str: + """Normalize a license-server URL and migrate the retired combined host.""" + normalized = (url or "").strip().rstrip("/") + return (DEFAULT_LICENSE_SERVER_URL + if normalized in RETIRED_LICENSE_SERVER_URLS else normalized) + + def resolve_license_server_url(signed_url: str = "") -> str: """Resolve the license server, including known vendor-host migrations.""" - override = canonicalize_relay_url(_env("ENGRAPHIS_CLOUD_URL", "")) - signed = canonicalize_relay_url(signed_url) - relay = canonicalize_relay_url(settings.relay_url) - return override or signed or relay + override = canonicalize_license_server_url(_env("ENGRAPHIS_CLOUD_URL", "")) + signed = canonicalize_license_server_url(signed_url) + return override or signed or DEFAULT_LICENSE_SERVER_URL diff --git a/engraphis/core/engine.py b/engraphis/core/engine.py index e4d34ff..290bb81 100644 --- a/engraphis/core/engine.py +++ b/engraphis/core/engine.py @@ -485,8 +485,8 @@ def _resolve_and_store(self, content: str, *, text: str, vec: np.ndarray, self.index.delete([decision.target_id]) except Exception as exc: # noqa: BLE001 — merely stale in the index; recall # re-checks validity on read, so log (don't audit) and continue. - logger.warning("vector-index delete failed for %s: %s", - decision.target_id, exc) + logger.warning("vector-index delete failed for %s (%s)", + decision.target_id, type(exc).__name__) self.store.audit("resolver", "invalidate", decision.target_id, decision.reason) linked = self._evolve(mid, neighbors, exclude={decision.target_id}) out = {"id": mid, "op": "invalidate", "superseded": [decision.target_id], @@ -660,8 +660,19 @@ def ingest(self, text: str, *, workspace_id: str, repo_id: Optional[str] = None, results = [] base_metadata = dict(metadata or {}) - for f in facts: - fact_own = getattr(f, "metadata", {}) or {} + source_sha256 = hashlib.sha256(text.encode("utf-8", "replace")).hexdigest() + for fact_index, f in enumerate(facts, start=1): + fact_own = dict(getattr(f, "metadata", {}) or {}) + if isinstance(fact_own.get("llm_extraction"), dict): + # Group all facts derived from one source without retaining the raw + # source or prompt. The dashboard activity viewer can therefore explain + # one input -> N memories while keeping provider payloads private. + fact_own["llm_extraction"] = { + **fact_own["llm_extraction"], + "source_sha256": source_sha256, + "fact_index": fact_index, + "fact_count": len(facts), + } # This is the one place that can tell the two apart: ``fact_own`` is computed # fresh from the Extractor's real output, while ``base_metadata`` is the # caller's argument. The extractor's keys win the merge, so vouching by name diff --git a/engraphis/core/graph_scene.py b/engraphis/core/graph_scene.py new file mode 100644 index 0000000..e25caf5 --- /dev/null +++ b/engraphis/core/graph_scene.py @@ -0,0 +1,1706 @@ +"""Deterministic evidence-backed graph scene construction. + +This module is deliberately pure: callers provide scoped entity, edge and support +rows, and receive JSON-ready canonical graph scenes. SQLite/FastAPI integration stays +in the service and route layers. +""" +from __future__ import annotations + +import hashlib +import heapq +import json +import math +import re +from bisect import bisect_left, bisect_right +from collections import Counter, defaultdict, deque +from typing import Any, Iterable, Mapping, Optional, Sequence + + +ALGORITHM_VERSION = "galaxy-v2" +PUBLIC_REFERENCE_ID_LIMIT = 200 +PUBLIC_FACET_LIMIT = 100 +GOLDEN_ANGLE = math.pi * (3.0 - math.sqrt(5.0)) +_STOPWORDS = { + "a", "an", "and", "are", "as", "at", "be", "by", "for", "from", "in", + "is", "it", "of", "on", "or", "that", "the", "this", "to", "was", "were", + "with", "unknown", "untitled", "none", "null", + # Capitalized sentence fragments produced by the fully-offline regex extractor are + # not useful entity identities. Keep this deliberately conservative and limited to + # unambiguous function words, booleans, generic workflow verbs, and directions; it is + # only applied to ``person_or_concept`` nodes, never code symbols or typed entities. + "all", "also", "any", "both", "each", "either", "every", "more", "most", + "other", "same", "several", "some", "such", "than", "then", "there", "here", + "too", "very", "yes", "no", "true", "false", "one", "two", "three", + "first", "second", "last", "left", "right", "new", "old", "now", + "can", "cannot", "could", "did", "do", "does", "doing", "done", "had", + "has", "have", "having", "may", "might", "must", "shall", "should", "will", + "would", "run", "running", "fix", "fixed", "create", "created", "review", + "reviewed", "blocked", "refusing", "investigate", "overall", "subject", + "reason", "action", "actions", "outcome", "add", "added", "check", "checked", + "scan", "scanned", "merge", "merged", "comment", "comments", "artifact", + "artifacts", "manifest", "key", "keys", "per", "local", "test", "tests", + "verdict", "connection", "connections", "input", "output", "request", + "response", "result", "results", "status", "detail", "details", + "active", "author", "because", "commit", "missing", "only", "possible", + "title", "available", "existing", "expected", "following", "given", "next", + "previous", "required", "single", "still", "total", "used", "using", "without", + "approval", "approved", "categories", "degraded", "error", "errors", "failed", + "passed", "rejected", "skipped", "success", "verify", "warning", "warnings", + "see", "successful", "prose", "supported", "generated", "matched", + "enumerated", "reached", "posted", "completed", +} +_HARD_BOILERPLATE_PREFIXES = { + "if", "generated", "matched", "enumerated", "reached", "posted", "completed", + "supported", +} +_SEARCH_FRAGMENT_PREFIXES = _HARD_BOILERPLATE_PREFIXES | { + # Sentence-openers observed in legacy/offline extraction output. These are too + # broad to erase from an analytical scene ("Full Stack", for example, can be a + # valid concept), but they should not crowd out a direct identity suggestion. + "no", "add", "added", "full", "three", "orphan", "ignored", "ignores", + "compiled", "codex-descended", +} +_BOILERPLATE_SUFFIXES = ("-based", "-side", "-level", "-version") + + +def _row(row: Mapping[str, Any]) -> dict[str, Any]: + return dict(row) + + +def _loads(raw: Any) -> dict[str, Any]: + if isinstance(raw, dict): + return raw + try: + value = json.loads(raw or "{}") + except (TypeError, ValueError, RecursionError): + return {} + return value if isinstance(value, dict) else {} + + +def _memory_ids(provenance: Any) -> list[str]: + value = _loads(provenance) + candidates: list[Any] = [value.get("memory_id")] + if isinstance(value.get("memory_ids"), list): + candidates.extend(value["memory_ids"]) + result: list[str] = [] + for candidate in candidates: + memory_id = str(candidate or "") + if memory_id and memory_id not in result: + result.append(memory_id) + return result + + +def _clamp(value: float, low: float = 0.0, high: float = 1.0) -> float: + return max(low, min(high, value)) + + +def _quantile(values: Sequence[float], fraction: float) -> float: + if not values: + return 0.0 + ordered = sorted(values) + position = (len(ordered) - 1) * fraction + lower = int(math.floor(position)) + upper = int(math.ceil(position)) + if lower == upper: + return ordered[lower] + weight = position - lower + return ordered[lower] * (1.0 - weight) + ordered[upper] * weight + + +def _percentile(value: float, ordered: Sequence[float]) -> float: + if len(ordered) <= 1: + return 1.0 if ordered else 0.0 + return (bisect_right(ordered, value) - 1) / (len(ordered) - 1) + + +def _mass_percentile(value: float, ordered: Sequence[float]) -> float: + """Tie-aware percentile for non-negative mass inputs. + + Zero means no evidence and must remain zero. Positive ties receive their mid-rank + instead of the top of the tie block; otherwise thousands of isolates all acquire a + near-maximal mass merely because they share the same zero degree/support values. + """ + if value <= 0.0 or not ordered: + return 0.0 + if len(ordered) == 1: + return 1.0 + left = bisect_left(ordered, value) + right = bisect_right(ordered, value) + midpoint = (left + right - 1) / 2.0 + return _clamp(midpoint / (len(ordered) - 1)) + + +def is_obvious_entity_noise(label: str, entity_type: str) -> bool: + """Conservatively flag extractor fragments without deleting graph identity rows.""" + if entity_type not in {"concept", "person_or_concept"}: + return False + normalized = " ".join(label.casefold().split()) + tokens = re.findall(r"[a-z0-9]+", normalized) + if len(normalized) < 2 or not tokens: + return True + if normalized in _STOPWORDS or all(token in _STOPWORDS for token in tokens): + return True + if len(tokens) > 1 and ( + tokens[0] in _HARD_BOILERPLATE_PREFIXES + or any(normalized.startswith(f"{prefix} ") or normalized.startswith(f"{prefix}-") + for prefix in _HARD_BOILERPLATE_PREFIXES if "-" in prefix) + ): + return True + dashed = re.sub(r"\s*[\N{EN DASH}\N{EM DASH}_/]\s*", "-", normalized) + return dashed.endswith(_BOILERPLATE_SUFFIXES) + + +def is_broad_search_fragment(label: str, entity_type: str) -> bool: + """Demote likely sentence fragments without removing them from graph scenes.""" + if is_obvious_entity_noise(label, entity_type): + return True + if entity_type not in {"concept", "person_or_concept"}: + return False + normalized = " ".join(label.casefold().split()) + tokens = re.findall(r"[a-z0-9]+", normalized) + return len(tokens) > 1 and ( + tokens[0] in _SEARCH_FRAGMENT_PREFIXES + or any(normalized.startswith(f"{prefix} ") or normalized.startswith(f"{prefix}-") + for prefix in _SEARCH_FRAGMENT_PREFIXES if "-" in prefix) + ) + + +def _combined_confidence(values: Iterable[float]) -> float: + complement = 1.0 + seen = False + for value in values: + seen = True + complement *= 1.0 - _clamp(float(value), 0.05, 0.99) + return 1.0 - complement if seen else 0.50 + + +def _relation_factor(layer: str, relation: str) -> float: + if relation == "co_occurs": + return 0.25 + if layer in {"entity", "causal"}: + return 1.0 + if layer == "temporal": + return 0.90 + return 0.80 + + +def _source_default(relation: str, provenance: Any) -> tuple[str, float]: + if relation == "co_occurs": + return "co_occurrence", 0.25 + raw = str(_loads(provenance).get("source") or "").casefold() + if "manual" in raw or "schema" in raw: + return "manual", 1.0 + if "structured" in raw: + return "structured", 0.80 + if "regex" in raw or "backfill" in raw: + return "regex_proximity", 0.55 + return "legacy_unknown", 0.50 + + +def _stable_id(prefix: str, *parts: Any) -> str: + payload = "\x1f".join(str(part) for part in parts).encode("utf-8") + return prefix + hashlib.sha256(payload).hexdigest()[:16] + + +def _components(node_ids: Sequence[str], edges: Sequence[dict]) -> dict[str, str]: + adjacent: dict[str, set[str]] = {node_id: set() for node_id in node_ids} + for edge in edges: + adjacent.setdefault(edge["source"], set()).add(edge["target"]) + adjacent.setdefault(edge["target"], set()).add(edge["source"]) + result: dict[str, str] = {} + components: list[list[str]] = [] + for start in sorted(adjacent): + if start in result: + continue + members: list[str] = [] + queue = deque([start]) + result[start] = "" + while queue: + current = queue.popleft() + members.append(current) + for neighbor in sorted(adjacent[current]): + if neighbor not in result: + result[neighbor] = "" + queue.append(neighbor) + components.append(members) + components.sort(key=lambda members: (-len(members), min(members))) + for index, members in enumerate(components): + for member in members: + result[member] = f"component_{index}" + return result + + +def _louvain(node_ids: Sequence[str], edges: Sequence[dict]) -> dict[str, str]: + """Deterministic first-level weighted Louvain local moving. + + Sorted traversal and canonical tie-breaking make identical inputs produce + identical communities without relying on process-randomized hash order. + """ + adjacency: dict[str, dict[str, float]] = {node_id: {} for node_id in node_ids} + for edge in edges: + source, target = edge["source"], edge["target"] + weight = max(float(edge.get("strength") or 0.0), 0.0001) + adjacency[source][target] = adjacency[source].get(target, 0.0) + weight + adjacency[target][source] = adjacency[target].get(source, 0.0) + weight + degree = {node_id: sum(adjacency[node_id].values()) for node_id in node_ids} + total = sum(degree.values()) + community = {node_id: node_id for node_id in node_ids} + totals = dict(degree) + if total <= 0.0: + return {node_id: _stable_id("community_", node_id) for node_id in node_ids} + for _ in range(24): + moved = False + for node_id in sorted(node_ids): + current = community[node_id] + node_degree = degree[node_id] + weights: dict[str, float] = defaultdict(float) + for neighbor, weight in adjacency[node_id].items(): + weights[community[neighbor]] += weight + totals[current] -= node_degree + best = current + best_gain = 0.0 + for candidate in sorted(weights): + gain = weights[candidate] - (totals.get(candidate, 0.0) * node_degree / total) + if gain > best_gain + 1e-12: + best, best_gain = candidate, gain + community[node_id] = best + totals[best] = totals.get(best, 0.0) + node_degree + if best != current: + moved = True + if not moved: + break + grouped: dict[str, list[str]] = defaultdict(list) + for node_id, raw_id in community.items(): + grouped[raw_id].append(node_id) + stable = { + raw_id: _stable_id("community_", *sorted(members)) + for raw_id, members in grouped.items() + } + return {node_id: stable[raw_id] for node_id, raw_id in community.items()} + + +def build_canonical_graph( + entity_rows: Sequence[Mapping[str, Any]], + edge_rows: Sequence[Mapping[str, Any]], + support_rows: Sequence[Mapping[str, Any]], + *, + include_weak_cooccurrence: bool = False, + layers: Optional[set[str]] = None, + relations: Optional[set[str]] = None, + min_support: int = 1, + min_confidence: float = 0.0, +) -> dict[str, Any]: + """Canonicalize and score the complete filtered graph before scene caps.""" + members: dict[str, list[dict]] = defaultdict(list) + member_to_canonical: dict[str, str] = {} + for raw in entity_rows: + entity = _row(raw) + canonical_id = str(entity.get("canonical_id") or entity.get("id") or "") + entity_id = str(entity.get("id") or "") + if not entity_id or not canonical_id: + continue + members[canonical_id].append(entity) + member_to_canonical[entity_id] = canonical_id + + nodes: dict[str, dict] = {} + for canonical_id, group in sorted(members.items()): + labels = Counter(str(item.get("name") or canonical_id) for item in group) + label = sorted(labels, key=lambda item: (-labels[item], item.casefold(), item))[0] + types = Counter(str(item.get("etype") or "person_or_concept") for item in group) + entity_type = sorted(types, key=lambda item: (-types[item], item))[0] + repo_ids = sorted({str(item["repo_id"]) for item in group if item.get("repo_id")}) + nodes[canonical_id] = { + "id": canonical_id, + "canonical_id": canonical_id, + "label": label, + "type": entity_type, + "member_ids": sorted(str(item["id"]) for item in group), + "member_count": len(group), + "repo_ids": repo_ids, + "aliases": sorted(labels, key=lambda item: (item.casefold(), item)), + } + + supports_by_edge: dict[str, list[dict]] = defaultdict(list) + for raw in support_rows: + support = _row(raw) + supports_by_edge[str(support.get("edge_id") or "")].append(support) + + bundled: dict[tuple[str, str, str, str, bool], dict] = {} + for raw in edge_rows: + edge = _row(raw) + source = member_to_canonical.get(str(edge.get("src") or "")) + target = member_to_canonical.get(str(edge.get("dst") or "")) + relation = str(edge.get("relation") or "related") + layer = str(edge.get("layer") or "semantic") + if not source or not target or source == target: + continue + if layers is not None and layer not in layers: + continue + if relations is not None and relation not in relations: + continue + directed = relation not in {"co_occurs", "related", "associated_with"} + if not directed and target < source: + source, target = target, source + edge_id = str(edge.get("id") or _stable_id("edge_", source, target, relation, layer)) + evidence = [dict(item) for item in supports_by_edge.get(edge_id, [])] + if not evidence: + source_kind, default_confidence = _source_default(relation, edge.get("provenance")) + memory_ids = _memory_ids(edge.get("provenance")) + evidence = [{ + "edge_id": edge_id, + "memory_id": memory_id, + "source_kind": source_kind, + "confidence": default_confidence, + "provenance": edge.get("provenance") or "{}", + } for memory_id in memory_ids] + if not evidence: + evidence = [{ + "edge_id": edge_id, + "memory_id": "", + "source_kind": "legacy_unknown", + "confidence": 0.50, + "provenance": edge.get("provenance") or "{}", + }] + memory_ids = {str(item.get("memory_id") or "") for item in evidence} + memory_ids.discard("") + key = (source, target, relation, layer, directed) + item = bundled.get(key) + if item is None: + item = { + "id": edge_id, + "source": source, + "target": target, + "relation": relation, + "layer": layer, + "directed": directed, + "weight": max(0.05, min(4.0, float(edge.get("weight") or 1.0))), + "_confidence_by_support": {}, + "_support_ids": set(), + "_support_rows": [], + "_memory_types": set(), + "_support_times": [], + "underlying_edge_ids": [], + } + bundled[key] = item + item["weight"] = max(item["weight"], float(edge.get("weight") or 1.0)) + for index, row in enumerate(evidence): + memory_id = str(row.get("memory_id") or "") + support_key = memory_id or f"anonymous:{edge_id}:{index}" + support_confidence = float( + row.get("confidence") if row.get("confidence") is not None else 0.50 + ) + item["_confidence_by_support"][support_key] = max( + support_confidence, + item["_confidence_by_support"].get(support_key, 0.0), + ) + item["_support_ids"].update(memory_ids) + item["_support_rows"].extend(evidence) + item["_memory_types"].update( + str(row.get("memory_type") or "") for row in evidence + if row.get("memory_type") + ) + item["_support_times"].extend( + float(row["support_time"]) for row in evidence + if row.get("support_time") is not None + ) + item["underlying_edge_ids"].append(edge_id) + + edges = [] + raw_logs: list[float] = [] + for key in sorted(bundled): + item = bundled[key] + all_underlying_ids = sorted(set(item["underlying_edge_ids"])) + item["_underlying_edge_ids_all"] = set(all_underlying_ids) + item["underlying_edge_ids"] = all_underlying_ids[:PUBLIC_REFERENCE_ID_LIMIT] + item["underlying_edge_ids_truncated"] = ( + len(all_underlying_ids) > PUBLIC_REFERENCE_ID_LIMIT + ) + if len(all_underlying_ids) > 1: + item["id"] = _stable_id("bundle_", *all_underlying_ids) + item["bundled_edge_count"] = len(all_underlying_ids) + # The confidence map is keyed by stable memory id or a per-row anonymous key, + # so it counts identified and legacy anonymous evidence without double-counting + # duplicate rows for the same memory. + item["support_count"] = len(item["_confidence_by_support"]) + all_support_ids = set(item["_support_ids"]) + item["_support_ids_all"] = all_support_ids + item["support_memory_ids"] = sorted(all_support_ids)[:PUBLIC_REFERENCE_ID_LIMIT] + item["support_ids_truncated"] = len(all_support_ids) > PUBLIC_REFERENCE_ID_LIMIT + item["confidence"] = _combined_confidence( + item["_confidence_by_support"].values() + ) + # Filters apply to the canonical display relation after parallel member-level + # rows have been bundled. Applying them above would discard two independent + # one-support alias edges that together form a supported canonical relation. + if (item["support_count"] < max(0, int(min_support)) + or item["confidence"] < min_confidence): + continue + if (item["relation"] == "co_occurs" and item["support_count"] <= 1 + and not include_weak_cooccurrence): + continue + item["memory_types"] = sorted(item["_memory_types"]) + item["support_time_min"] = ( + min(item["_support_times"]) if item["_support_times"] else None + ) + item["support_time_max"] = ( + max(item["_support_times"]) if item["_support_times"] else None + ) + support_boost = 1.0 + min(math.log2(1.0 + item["support_count"]) / 4.0, 0.75) + raw_strength = ( + max(0.05, min(4.0, item["weight"])) + * item["confidence"] + * support_boost + * _relation_factor(item["layer"], item["relation"]) + ) + item["_raw_log"] = math.log1p(raw_strength) + raw_logs.append(item["_raw_log"]) + edges.append(item) + low, high = _quantile(raw_logs, 0.05), _quantile(raw_logs, 0.95) + for edge in edges: + edge["strength"] = ( + 1.0 if high - low <= 1e-12 + else _clamp((edge["_raw_log"] - low) / (high - low)) + ) + + degree = {node_id: 0.0 for node_id in nodes} + node_supports: dict[str, set[str]] = {node_id: set() for node_id in nodes} + adjacency: dict[str, dict[str, float]] = {node_id: {} for node_id in nodes} + for edge in edges: + source, target = edge["source"], edge["target"] + strength = edge["strength"] + degree[source] += strength + degree[target] += strength + node_supports[source].update(edge["_support_ids_all"]) + node_supports[target].update(edge["_support_ids_all"]) + adjacency[source][target] = adjacency[source].get(target, 0.0) + strength + adjacency[target][source] = adjacency[target].get(source, 0.0) + strength + + pagerank = {node_id: 1.0 / max(1, len(nodes)) for node_id in nodes} + damping = 0.85 + for _ in range(32): + base = (1.0 - damping) / max(1, len(nodes)) + updated = {node_id: base for node_id in nodes} + dangling = sum(pagerank[node_id] for node_id in nodes if degree[node_id] <= 0.0) + spread = damping * dangling / max(1, len(nodes)) + for node_id in updated: + updated[node_id] += spread + for source in sorted(nodes): + if degree[source] <= 0.0: + continue + for target, weight in sorted(adjacency[source].items()): + updated[target] += damping * pagerank[source] * weight / degree[source] + pagerank = updated + + degree_values = sorted(degree.values()) + pagerank_values = sorted(pagerank.values()) + support_values = sorted(float(len(value)) for value in node_supports.values()) + repo_values = sorted(float(len(node["repo_ids"])) for node in nodes.values()) + max_pagerank = max(pagerank.values(), default=1.0) or 1.0 + for node_id, node in nodes.items(): + obvious_noise = is_obvious_entity_noise(node["label"], node["type"]) + quality = 0.0 if obvious_noise else 1.0 + support_count = len(node_supports[node_id]) + mass_score = quality * ( + 0.45 * _mass_percentile(degree[node_id], degree_values) + + 0.30 * _mass_percentile(pagerank[node_id], pagerank_values) + + 0.15 * _mass_percentile(float(support_count), support_values) + + 0.10 * _mass_percentile(float(len(node["repo_ids"])), repo_values) + ) + node.update({ + "weighted_degree": round(degree[node_id], 6), + "pagerank": round(pagerank[node_id] / max_pagerank, 6), + "support_count": support_count, + "entity_quality": quality, + "mass_score": round(_clamp(mass_score), 6), + "gravity_mass": round(1.0 + 7.0 * (_clamp(mass_score) ** 1.35), 6), + "visual_radius": round(2.5 + 6.0 * math.sqrt(_clamp(mass_score)), 6), + "anchor_eligible": bool(quality), + }) + + components = _components(sorted(nodes), edges) + communities = _louvain(sorted(nodes), edges) + eligible = [node for node in nodes.values() if node["anchor_eligible"]] or list(nodes.values()) + global_anchor = min( + eligible, + key=lambda node: (-node["mass_score"], -node["weighted_degree"], node["canonical_id"]), + default=None, + ) + global_id = global_anchor["id"] if global_anchor else "" + community_members: dict[str, list[str]] = defaultdict(list) + for node_id in sorted(nodes): + community_members[communities[node_id]].append(node_id) + community_anchors: dict[str, str] = {} + for community_id, ids in community_members.items(): + pool = [nodes[node_id] for node_id in ids if nodes[node_id]["anchor_eligible"]] + pool = pool or [nodes[node_id] for node_id in ids] + community_anchors[community_id] = min( + pool, + key=lambda node: (-node["mass_score"], -node["weighted_degree"], node["id"]), + )["id"] + + direct_core: dict[str, float] = defaultdict(float) + for edge in edges: + if edge["source"] == global_id: + direct_core[edge["target"]] = max(direct_core[edge["target"]], edge["strength"]) + if edge["target"] == global_id: + direct_core[edge["source"]] = max(direct_core[edge["source"]], edge["strength"]) + for node_id, node in nodes.items(): + community_id = communities[node_id] + role = "global" if node_id == global_id else ( + "community" if community_anchors[community_id] == node_id else "none" + ) + affinity = 1.0 if node_id == global_id else _clamp( + 0.65 * node["mass_score"] + 0.35 * direct_core[node_id] + ) + node.update({ + "component_id": components[node_id], + "community_id": community_id, + "anchor_role": role, + "core_affinity": round(affinity, 6), + "scene_rank": round(_clamp(0.75 * node["mass_score"] + 0.25 * affinity), 6), + }) + + for edge in edges: + source_radius = nodes[edge["source"]]["visual_radius"] + target_radius = nodes[edge["target"]]["visual_radius"] + edge["rest_length"] = round(_clamp( + 12.0 + 14.0 * (1.0 - edge["strength"]) + + 0.8 * (source_radius + target_radius), 14.0, 34.0 + ), 6) + edge["spring_strength"] = round(0.035 + 0.17 * edge["strength"], 6) + edge["tier"] = "context" + edge["visible_by_default"] = True + edge.pop("_raw_log", None) + edge.pop("_confidence_by_support", None) + edge.pop("_support_ids", None) + edge.pop("_support_rows", None) + edge.pop("_memory_types", None) + edge.pop("_support_times", None) + + return { + "nodes": nodes, + "edges": sorted(edges, key=lambda edge: ( + -edge["strength"], edge["source"], edge["target"], edge["relation"], edge["id"] + )), + "member_to_canonical": member_to_canonical, + "community_members": dict(community_members), + "community_anchors": community_anchors, + "global_anchor": global_id, + } + + +class _UnionFind: + def __init__(self, values: Iterable[str]) -> None: + self.parent = {value: value for value in values} + + def find(self, value: str) -> str: + while self.parent[value] != value: + self.parent[value] = self.parent[self.parent[value]] + value = self.parent[value] + return value + + def union(self, left: str, right: str) -> bool: + a, b = self.find(left), self.find(right) + if a == b: + return False + if b < a: + a, b = b, a + self.parent[b] = a + return True + + +def _selected_edges(graph: dict, selected: set[str], level: str, cap: int) -> list[dict]: + candidates = [edge for edge in graph["edges"] + if edge["source"] in selected and edge["target"] in selected] + if level == "overview": + candidates = [edge for edge in candidates if + graph["nodes"][edge["source"]]["community_id"] + == graph["nodes"][edge["target"]]["community_id"]] + retained: set[str] = set() + for community_id, member_ids in graph["community_members"].items(): + members = selected.intersection(member_ids) + forest = _UnionFind(members) + internal = [edge for edge in candidates if edge["source"] in members + and edge["target"] in members] + for edge in sorted(internal, key=lambda item: (-item["strength"], item["id"])): + if forest.union(edge["source"], edge["target"]): + retained.add(edge["id"]) + edge["tier"] = "backbone" + per_node = 4 if level in {"neighborhood", "path"} else 2 + incident: dict[str, list[dict]] = defaultdict(list) + for edge in candidates: + incident[edge["source"]].append(edge) + incident[edge["target"]].append(edge) + if edge["layer"] in {"causal", "temporal"}: + retained.add(edge["id"]) + if edge["tier"] != "backbone": + edge["tier"] = "primary" + for node_id in sorted(selected): + ranked = sorted(incident[node_id], key=lambda item: (-item["strength"], item["id"])) + for edge in ranked[:per_node]: + retained.add(edge["id"]) + if edge["tier"] == "context": + edge["tier"] = "primary" + chosen = [ + {key: value for key, value in edge.items() if not key.startswith("_")} + for edge in candidates if edge["id"] in retained + ] + chosen.sort(key=lambda edge: ( + {"backbone": 0, "primary": 1, "context": 2}.get(edge["tier"], 3), + -edge["strength"], edge["id"], + )) + return chosen[:cap] + + +def _community_summaries(graph: dict, community_ids: set[str], + selected: set[str]) -> list[dict]: + edges = graph["edges"] + result = [] + for community_id in community_ids: + member_ids = graph["community_members"][community_id] + anchor_id = graph["community_anchors"][community_id] + internal = [edge for edge in edges if edge["source"] in member_ids + and edge["target"] in member_ids] + external = [edge for edge in edges if + (edge["source"] in member_ids) != (edge["target"] in member_ids)] + mass = _community_mass(graph, member_ids) + representatives = sorted(member_ids, key=lambda node_id: ( + -graph["nodes"][node_id]["scene_rank"], node_id + ))[:8] + result.append({ + "id": community_id, + "label": f"{graph['nodes'][anchor_id]['label']} System", + "anchor_id": anchor_id, + "mass": round(mass, 6), + "radius": round(_clamp(30.0 + 5.0 * math.sqrt(len(member_ids)), 36.0, 110.0), 6), + "member_count": len(member_ids), + "shown_member_count": len(set(member_ids).intersection(selected)), + "internal_strength": round(sum(edge["strength"] for edge in internal), 6), + "external_strength": round(sum(edge["strength"] for edge in external), 6), + "representative_ids": representatives, + }) + return sorted(result, key=lambda item: (-item["mass"], item["id"])) + + +def _community_mass(graph: dict, member_ids: Iterable[str]) -> float: + """Return the same aggregate mass used by the system-layout contract.""" + return sum( + math.sqrt(max(1.0, float(graph["nodes"][node_id]["gravity_mass"]))) + for node_id in member_ids + ) + + +def _bridge_physics_strength(value: float, ordered: Sequence[float]) -> float: + """Robustly normalize aggregate bridge evidence without flattening the tails. + + The p05/p95 component keeps one extreme bridge from compressing the useful range. + A small empirical-percentile component preserves deterministic distinctions among + values outside those robust bounds, where a plain clamp would make them identical. + """ + if not ordered: + return 0.0 + if len(ordered) == 1 or ordered[-1] - ordered[0] <= 1e-12: + return 1.0 + low, high = _quantile(ordered, 0.05), _quantile(ordered, 0.95) + if high - low <= 1e-12: + robust = _percentile(value, ordered) + else: + robust = _clamp((value - low) / (high - low)) + rank = _percentile(value, ordered) + return _clamp(0.90 * robust + 0.10 * rank) + + +def _bridges(graph: dict, community_ids: set[str], cap: int) -> list[dict]: + grouped: dict[tuple[str, str, str], list[dict]] = defaultdict(list) + for edge in graph["edges"]: + source = graph["nodes"][edge["source"]]["community_id"] + target = graph["nodes"][edge["target"]]["community_id"] + if source == target or source not in community_ids or target not in community_ids: + continue + if target < source: + source, target = target, source + grouped[(source, target, edge["layer"])].append(edge) + result = [] + for (source, target, layer), edges in grouped.items(): + all_edge_ids = sorted(edge["id"] for edge in edges) + relations = Counter() + for edge in edges: + relations[edge["relation"]] += max(1, int(edge["bundled_edge_count"])) + support_ids = { + memory_id for edge in edges for memory_id in edge["_support_ids_all"] + } + anonymous_support_count = sum( + max(0, int(edge["support_count"]) - len(edge["_support_ids_all"])) + for edge in edges + ) + support_count = len(support_ids) + anonymous_support_count + edge_count = sum(max(1, int(edge["bundled_edge_count"])) for edge in edges) + aggregate_strength = sum(max(0.0, float(edge["strength"])) for edge in edges) + # Strength carries most of the signal; unique evidence and relation cardinality + # add bounded corroboration without allowing raw counts to dominate the layout. + physics_raw = ( + 0.60 * math.log1p(aggregate_strength) + + 0.25 * math.log1p(support_count) + + 0.15 * math.log1p(edge_count) + ) + result.append({ + "id": _stable_id("bridge_", source, target, layer), + "source_community": source, + "target_community": target, + "layer": layer, + # Keep the original display field compatible for one contract version. + "strength": round(_clamp(aggregate_strength), 6), + "aggregate_strength": round(aggregate_strength, 6), + "support_count": support_count, + "edge_count": edge_count, + "top_relations": sorted(relations, key=lambda relation: ( + -relations[relation], relation + ))[:5], + "edge_ids": all_edge_ids[:PUBLIC_REFERENCE_ID_LIMIT], + "edge_ids_truncated": len(all_edge_ids) > PUBLIC_REFERENCE_ID_LIMIT, + "_physics_raw": physics_raw, + }) + # Rank before the cap with unsaturated aggregate evidence. Otherwise every bridge + # whose summed display strength exceeds one ties and the cap becomes ID-driven. + result.sort(key=lambda bridge: (-bridge["_physics_raw"], bridge["id"])) + retained = result[:max(0, cap)] + ordered = sorted(bridge["_physics_raw"] for bridge in retained) + for bridge in retained: + bridge["physics_strength"] = round( + _bridge_physics_strength(bridge["_physics_raw"], ordered), 6 + ) + bridge.pop("_physics_raw", None) + retained.sort(key=lambda bridge: ( + -bridge["physics_strength"], -bridge["aggregate_strength"], bridge["id"] + )) + return retained + + +def _facets(graph: dict) -> dict[str, list[dict]]: + types = Counter(node["type"] for node in graph["nodes"].values()) + repos = Counter(repo for node in graph["nodes"].values() for repo in node["repo_ids"]) + layers = Counter(edge["layer"] for edge in graph["edges"]) + relations = Counter(edge["relation"] for edge in graph["edges"]) + memory_types = Counter( + memory_type for edge in graph["edges"] + for memory_type in edge.get("memory_types", []) + ) + support = Counter( + "1" if edge["support_count"] <= 1 else + "2-3" if edge["support_count"] <= 3 else + "4-7" if edge["support_count"] <= 7 else "8+" + for edge in graph["edges"] + ) + confidence = Counter( + "0-49%" if edge["confidence"] < 0.5 else + "50-74%" if edge["confidence"] < 0.75 else + "75-89%" if edge["confidence"] < 0.9 else "90-100%" + for edge in graph["edges"] + ) + support_times = [ + float(value) for edge in graph["edges"] + for value in (edge.get("support_time_min"), edge.get("support_time_max")) + if value is not None + ] + + def items(counter: Counter) -> list[dict]: + return [{"value": value, "count": count} for value, count in sorted( + counter.items(), key=lambda item: (-item[1], item[0]) + )[:PUBLIC_FACET_LIMIT]] + + return { + "entity_types": items(types), + "memory_types": items(memory_types), + "layers": items(layers), + "relations": items(relations), + "repos": items(repos), + "support": items(support), + "confidence": items(confidence), + "time": ([{ + "value": "range", + "count": len(support_times), + "from": min(support_times), + "to": max(support_times), + }] if support_times else []), + } + + +def _complete_relations( + graph: dict[str, Any], + edge_rows: Sequence[Mapping[str, Any]], + support_rows: Sequence[Mapping[str, Any]], + *, + memory_ids: set[str], + include_weak_cooccurrence: bool, + layers: Optional[set[str]], + relations: Optional[set[str]], + min_support: int, + min_confidence: float, +) -> tuple[list[dict[str, Any]], list[dict[str, Any]]]: + """Return every filtered physical relation and its explicit evidence links. + + Normal analytical scenes intentionally bundle parallel canonical relations. A + complete scene has the opposite contract: the physical edge id is the public id, + and each supporting memory is connected to both relation endpoints. The latter + makes evidence selectable without replacing or hiding the factual relation. + """ + supports_by_edge: dict[str, list[dict[str, Any]]] = defaultdict(list) + for raw in support_rows: + support = _row(raw) + supports_by_edge[str(support.get("edge_id") or "")].append(support) + + pending: list[dict[str, Any]] = [] + evidence_pending: list[dict[str, Any]] = [] + raw_logs: list[float] = [] + for raw in sorted(edge_rows, key=lambda item: str(item.get("id") or "")): + edge = _row(raw) + source = graph["member_to_canonical"].get(str(edge.get("src") or "")) + target = graph["member_to_canonical"].get(str(edge.get("dst") or "")) + if not source or not target: + continue + relation = str(edge.get("relation") or "related") + layer = str(edge.get("layer") or "semantic") + if layers is not None and layer not in layers: + continue + if relations is not None and relation not in relations: + continue + edge_id = str(edge.get("id") or _stable_id( + "edge_", source, target, relation, layer + )) + evidence = [dict(item) for item in supports_by_edge.get(edge_id, [])] + if not evidence: + source_kind, default_confidence = _source_default( + relation, edge.get("provenance") + ) + evidence = [{ + "edge_id": edge_id, + "memory_id": memory_id, + "source_kind": source_kind, + "confidence": default_confidence, + "provenance": edge.get("provenance") or "{}", + } for memory_id in _memory_ids(edge.get("provenance"))] + if not evidence: + evidence = [{ + "edge_id": edge_id, + "memory_id": "", + "source_kind": "legacy_unknown", + "confidence": 0.50, + "provenance": edge.get("provenance") or "{}", + }] + + confidence_by_support: dict[str, float] = {} + support_memory_ids: set[str] = set() + for index, support in enumerate(evidence): + memory_id = str(support.get("memory_id") or "") + support_key = memory_id or f"anonymous:{edge_id}:{index}" + confidence_by_support[support_key] = max( + float(support.get("confidence") + if support.get("confidence") is not None else 0.50), + confidence_by_support.get(support_key, 0.0), + ) + if memory_id: + support_memory_ids.add(memory_id) + support_count = len(confidence_by_support) + confidence = _combined_confidence(confidence_by_support.values()) + if support_count < max(0, int(min_support)) or confidence < min_confidence: + continue + if (relation == "co_occurs" and support_count <= 1 + and not include_weak_cooccurrence): + continue + + weight = max(0.05, min(4.0, float(edge.get("weight") or 1.0))) + support_boost = 1.0 + min(math.log2(1.0 + support_count) / 4.0, 0.75) + raw_log = math.log1p( + weight * confidence * support_boost * _relation_factor(layer, relation) + ) + raw_logs.append(raw_log) + pending.append({ + "id": edge_id, + "source": source, + "target": target, + "relation": relation, + "layer": layer, + "directed": relation not in {"co_occurs", "related", "associated_with"}, + "weight": weight, + "confidence": round(confidence, 6), + "support_count": support_count, + "support_memory_ids": sorted(support_memory_ids), + "underlying_edge_ids": [edge_id], + "bundled_edge_count": 1, + "tier": "raw", + "visible_by_default": True, + "connector_kind": "entity_relation", + "_raw_log": raw_log, + }) + for support in evidence: + memory_id = str(support.get("memory_id") or "") + if not memory_id or memory_id not in memory_ids: + continue + source_kind = str(support.get("source_kind") or "legacy_unknown") + evidence_confidence = _clamp(float( + support.get("confidence") + if support.get("confidence") is not None else 0.50 + ), 0.05, 0.99) + for endpoint in sorted({source, target}): + evidence_pending.append({ + "id": _stable_id( + "evidence_", edge_id, memory_id, source_kind, endpoint + ), + "source": memory_id, + "target": endpoint, + "relation": "supports", + "layer": "evidence", + "directed": True, + "weight": evidence_confidence, + "confidence": round(evidence_confidence, 6), + "support_count": 1, + "support_memory_ids": [memory_id], + "underlying_edge_ids": [edge_id], + "bundled_edge_count": 1, + "tier": "evidence", + "visible_by_default": True, + "connector_kind": "evidence", + "source_kind": source_kind, + "strength": round(evidence_confidence, 6), + "rest_length": round(12.0 + 10.0 * (1.0 - evidence_confidence), 6), + "spring_strength": round(0.04 + 0.12 * evidence_confidence, 6), + }) + + low, high = _quantile(raw_logs, 0.05), _quantile(raw_logs, 0.95) + relations_out = [] + for edge in pending: + strength = ( + 1.0 if high - low <= 1e-12 + else _clamp((edge["_raw_log"] - low) / (high - low)) + ) + source_radius = graph["nodes"][edge["source"]]["visual_radius"] + target_radius = graph["nodes"][edge["target"]]["visual_radius"] + edge["strength"] = round(strength, 6) + edge["rest_length"] = round(_clamp( + 12.0 + 14.0 * (1.0 - strength) + + 0.8 * (source_radius + target_radius), 14.0, 34.0 + ), 6) + edge["spring_strength"] = round(0.035 + 0.17 * strength, 6) + edge.pop("_raw_log", None) + relations_out.append(edge) + return ( + sorted(relations_out, key=lambda item: ( + -item["strength"], item["source"], item["target"], + item["relation"], item["id"], + )), + sorted(evidence_pending, key=lambda item: item["id"]), + ) + + +def _complete_bridges(nodes: Mapping[str, dict], edges: Sequence[dict]) -> list[dict]: + """Aggregate every cross-system connector for system-level live gravity. + + These quotient-graph bridges are additive physics metadata; the complete scene + still returns every raw connector in ``edges``. + """ + grouped: dict[tuple[str, str, str], list[dict]] = defaultdict(list) + for edge in edges: + source_node = nodes.get(str(edge.get("source") or "")) + target_node = nodes.get(str(edge.get("target") or "")) + if not source_node or not target_node: + continue + source = source_node["community_id"] + target = target_node["community_id"] + if source == target: + continue + if target < source: + source, target = target, source + grouped[(source, target, str(edge.get("layer") or "semantic"))].append(edge) + pending = [] + for (source, target, layer), grouped_edges in sorted(grouped.items()): + strength = sum(max(0.0, float(edge.get("strength") or 0.0)) + for edge in grouped_edges) + support_ids = { + memory_id for edge in grouped_edges + for memory_id in edge.get("support_memory_ids", []) + } + relations = Counter(str(edge.get("relation") or "related") + for edge in grouped_edges) + raw = ( + 0.60 * math.log1p(strength) + + 0.25 * math.log1p(len(support_ids)) + + 0.15 * math.log1p(len(grouped_edges)) + ) + pending.append({ + "id": _stable_id("bridge_", source, target, layer), + "source_community": source, + "target_community": target, + "layer": layer, + "strength": round(_clamp(strength), 6), + "aggregate_strength": round(strength, 6), + "support_count": len(support_ids), + "edge_count": len(grouped_edges), + "top_relations": sorted(relations, key=lambda relation: ( + -relations[relation], relation + ))[:5], + "edge_ids": sorted(str(edge["id"]) for edge in grouped_edges), + "edge_ids_truncated": False, + "_physics_raw": raw, + }) + ordered = sorted(bridge["_physics_raw"] for bridge in pending) + for bridge in pending: + bridge["physics_strength"] = round( + _bridge_physics_strength(bridge["_physics_raw"], ordered), 6 + ) + bridge.pop("_physics_raw", None) + return sorted(pending, key=lambda bridge: ( + -bridge["physics_strength"], -bridge["aggregate_strength"], bridge["id"] + )) + + +def _build_complete_scene( + workspace: str, + graph: dict[str, Any], + edge_rows: Sequence[Mapping[str, Any]], + support_rows: Sequence[Mapping[str, Any]], + memory_rows: Sequence[Mapping[str, Any]], + memory_link_rows: Sequence[Mapping[str, Any]], + code_memory_link_rows: Sequence[Mapping[str, Any]], + *, + include_weak_cooccurrence: bool, + layers: Optional[set[str]], + relations: Optional[set[str]], + min_support: int, + min_confidence: float, + filters: dict[str, Any], + index_generation: int, +) -> dict[str, Any]: + memory_rows_by_id = { + str(row.get("id") or ""): _row(row) for row in memory_rows if row.get("id") + } + memory_ids = set(memory_rows_by_id) + raw_relations, evidence_edges = _complete_relations( + graph, edge_rows, support_rows, memory_ids=memory_ids, + include_weak_cooccurrence=include_weak_cooccurrence, + layers=layers, relations=relations, min_support=min_support, + min_confidence=min_confidence, + ) + + entity_nodes = {node_id: dict(node) for node_id, node in graph["nodes"].items()} + for node in entity_nodes.values(): + node["node_kind"] = "entity" + node.pop("aliases", None) + node.pop("anchor_eligible", None) + + evidence_targets: dict[str, list[tuple[float, str]]] = defaultdict(list) + for edge in evidence_edges: + evidence_targets[edge["source"]].append(( + float(edge["strength"]), edge["target"] + )) + + memory_community: dict[str, str] = {} + for memory_id in sorted(memory_ids): + candidates = evidence_targets.get(memory_id, []) + if candidates: + target = min(candidates, key=lambda item: (-item[0], item[1]))[1] + memory_community[memory_id] = entity_nodes[target]["community_id"] + for memory_id, memory in sorted(memory_rows_by_id.items()): + if memory_id not in memory_community: + memory_community[memory_id] = _stable_id( + "community_memory_", memory.get("repo_id") or "workspace", + memory.get("mtype") or "semantic", + ) + + memory_degree = Counter() + for edge in evidence_edges: + memory_degree[edge["source"]] += 1 + memory_link_edges = [] + for raw in sorted(memory_link_rows, key=lambda item: ( + str(item.get("a") or ""), str(item.get("b") or ""), + str(item.get("relation") or ""), float(item.get("created_at") or 0.0), + )): + row = _row(raw) + source, target = str(row.get("a") or ""), str(row.get("b") or "") + if source not in memory_ids or target not in memory_ids: + continue + relation = str(row.get("relation") or "related") + layer = str(row.get("layer") or "semantic") + if layers is not None and layer not in layers: + continue + if relations is not None and relation not in relations: + continue + memory_degree[source] += 1 + memory_degree[target] += 1 + memory_link_edges.append({ + "id": _stable_id( + "memlink_", source, target, relation, layer, + row.get("reason") or "", row.get("created_at") or 0.0, + ), + "source": source, + "target": target, + "relation": relation, + "layer": layer, + "directed": False, + "weight": 1.0, + "confidence": 1.0, + "support_count": 1, + "support_memory_ids": sorted({source, target}), + "underlying_edge_ids": [], + "bundled_edge_count": 1, + "tier": "raw", + "visible_by_default": True, + "connector_kind": "memory_link", + "reason": str(row.get("reason") or ""), + "strength": 0.72, + "rest_length": 22.0, + "spring_strength": 0.12, + }) + + code_memory_edges = [] + for raw in sorted(code_memory_link_rows, key=lambda item: str(item.get("id") or "")): + row = _row(raw) + memory_id = str(row.get("memory_id") or "") + symbol_id = f"code:{row.get('symbol_id')}" + if memory_id not in memory_ids or symbol_id not in entity_nodes: + continue + relation = str(row.get("relation") or "mentions") + if layers is not None and "entity" not in layers: + continue + if relations is not None and relation not in relations: + continue + confidence = _clamp(float(row.get("confidence") or 1.0), 0.05, 1.0) + memory_degree[memory_id] += 1 + code_memory_edges.append({ + "id": str(row.get("id") or _stable_id( + "code_memory_", memory_id, symbol_id, relation + )), + "source": memory_id, + "target": symbol_id, + "relation": relation, + "layer": "entity", + "directed": True, + "weight": confidence, + "confidence": round(confidence, 6), + "support_count": 1, + "support_memory_ids": [memory_id], + "underlying_edge_ids": [], + "bundled_edge_count": 1, + "tier": "raw", + "visible_by_default": True, + "connector_kind": "code_memory", + "strength": round(confidence, 6), + "rest_length": round(14.0 + 8.0 * (1.0 - confidence), 6), + "spring_strength": round(0.05 + 0.12 * confidence, 6), + }) + + memory_nodes: dict[str, dict[str, Any]] = {} + degree_values = sorted(float(memory_degree[memory_id]) for memory_id in memory_ids) + for memory_id, memory in sorted(memory_rows_by_id.items()): + title = str(memory.get("title") or "").strip() + summary = str(memory.get("summary") or "").strip() + content = str(memory.get("content") or "").strip() + label = title or summary or content or memory_id + label = " ".join(label.split())[:160] + importance = _clamp(float(memory.get("importance") or 0.0)) + degree_percentile = _mass_percentile( + float(memory_degree[memory_id]), degree_values + ) + mass_score = _clamp(0.08 + 0.34 * importance + 0.18 * degree_percentile, 0.08, 0.60) + memory_nodes[memory_id] = { + "id": memory_id, + "canonical_id": memory_id, + "label": label, + "type": str(memory.get("mtype") or "semantic"), + "node_kind": "memory", + "memory_type": str(memory.get("mtype") or "semantic"), + "scope": str(memory.get("scope") or "workspace"), + "member_ids": [memory_id], + "member_count": 1, + "repo_ids": [str(memory["repo_id"])] if memory.get("repo_id") else [], + "weighted_degree": round(float(memory_degree[memory_id]), 6), + "pagerank": 0.0, + "support_count": int(memory_degree[memory_id]), + "entity_quality": 1.0, + "mass_score": round(mass_score, 6), + "gravity_mass": round(0.55 + 2.2 * (mass_score ** 1.35), 6), + "visual_radius": round(2.0 + 3.5 * math.sqrt(mass_score), 6), + "component_id": f"component_memory_{memory_id}", + "community_id": memory_community[memory_id], + "anchor_role": "none", + "core_affinity": 0.0, + "scene_rank": round(_clamp(0.70 * mass_score + 0.30 * degree_percentile), 6), + "importance": round(importance, 6), + "pinned": bool(memory.get("pinned")), + "valid_from": memory.get("valid_from"), + "ingested_at": memory.get("ingested_at"), + } + + all_nodes: dict[str, dict[str, Any]] = {**entity_nodes, **memory_nodes} + community_members: dict[str, list[str]] = defaultdict(list) + for node_id, node in all_nodes.items(): + community_members[node["community_id"]].append(node_id) + community_anchors = dict(graph["community_anchors"]) + for community_id, member_ids in sorted(community_members.items()): + if community_id not in community_anchors: + community_anchors[community_id] = min(member_ids, key=lambda node_id: ( + -all_nodes[node_id]["scene_rank"], node_id + )) + all_nodes[community_anchors[community_id]]["anchor_role"] = "community" + + global_anchor = graph["global_anchor"] + if not global_anchor and all_nodes: + global_anchor = min(all_nodes, key=lambda node_id: ( + -all_nodes[node_id]["scene_rank"], node_id + )) + all_nodes[global_anchor]["anchor_role"] = "global" + community_anchors[all_nodes[global_anchor]["community_id"]] = global_anchor + + complete_edges = sorted( + [*raw_relations, *evidence_edges, *memory_link_edges, *code_memory_edges], + key=lambda edge: ( + edge["connector_kind"], -float(edge["strength"]), edge["id"] + ), + ) + internal_strength: dict[str, float] = defaultdict(float) + external_strength: dict[str, float] = defaultdict(float) + for edge in complete_edges: + source_community = all_nodes[edge["source"]]["community_id"] + target_community = all_nodes[edge["target"]]["community_id"] + strength = float(edge["strength"]) + if source_community == target_community: + internal_strength[source_community] += strength + else: + external_strength[source_community] += strength + external_strength[target_community] += strength + communities = [] + for community_id, member_ids in sorted(community_members.items()): + anchor_id = community_anchors[community_id] + mass = sum(math.sqrt(max(0.1, float(all_nodes[node_id]["gravity_mass"]))) + for node_id in member_ids) + communities.append({ + "id": community_id, + "label": f"{all_nodes[anchor_id]['label']} System", + "anchor_id": anchor_id, + "mass": round(mass, 6), + "radius": round(_clamp( + 30.0 + 5.0 * math.sqrt(len(member_ids)), 36.0, 180.0 + ), 6), + "member_count": len(member_ids), + "shown_member_count": len(member_ids), + "internal_strength": round(internal_strength[community_id], 6), + "external_strength": round(external_strength[community_id], 6), + "representative_ids": sorted(member_ids, key=lambda node_id: ( + -all_nodes[node_id]["scene_rank"], node_id + ))[:8], + }) + communities.sort(key=lambda item: (-item["mass"], item["id"])) + + hash_payload = { + "algorithm": f"{ALGORITHM_VERSION}-complete-1", + "index_generation": index_generation, + "workspace": workspace, + "filters": filters, + "nodes": [( + node_id, all_nodes[node_id]["node_kind"], all_nodes[node_id]["label"], + all_nodes[node_id]["community_id"], all_nodes[node_id]["mass_score"], + ) for node_id in sorted(all_nodes)], + "edges": [( + edge["id"], edge["source"], edge["target"], edge["relation"], + edge["connector_kind"], edge["strength"], + ) for edge in sorted(complete_edges, key=lambda item: item["id"])], + } + scene_hash = hashlib.sha256(json.dumps( + hash_payload, sort_keys=True, separators=(",", ":") + ).encode("utf-8")).hexdigest() + layout_seed = int(scene_hash[:8], 16) + + positions: dict[str, tuple[float, float]] = {} + for index, community in enumerate(communities): + if global_anchor in community_members[community["id"]]: + positions[community["id"]] = (0.0, 0.0) + else: + radius = 82.0 * math.sqrt(index + 1) + angle = GOLDEN_ANGLE * (index + 1) + (layout_seed % 360) * math.pi / 180.0 + positions[community["id"]] = ( + radius * math.cos(angle), radius * math.sin(angle) + ) + ranks: dict[str, int] = defaultdict(int) + scene_nodes = [] + radius_by_community = {item["id"]: item["radius"] for item in communities} + for node_id in sorted(all_nodes, key=lambda value: ( + -all_nodes[value]["scene_rank"], value + )): + node = dict(all_nodes[node_id]) + community_id = node["community_id"] + center_x, center_y = positions[community_id] + rank = ranks[community_id] + ranks[community_id] += 1 + if node_id == community_anchors[community_id]: + x, y = center_x, center_y + else: + system_radius = radius_by_community[community_id] + orbit = _clamp( + 13.0 + 5.5 * math.sqrt(rank + 1) + + (1.0 - node["mass_score"]) * 0.35 * system_radius, + 14.0, system_radius, + ) + angle = GOLDEN_ANGLE * (rank + 1) + (layout_seed % 180) * math.pi / 180.0 + x = center_x + orbit * math.cos(angle) + y = center_y + orbit * math.sin(angle) + node["x"], node["y"] = round(x, 6), round(y, 6) + scene_nodes.append(node) + + facets = _facets(graph) + memory_type_counts = Counter(node["memory_type"] for node in memory_nodes.values()) + facets["memory_types"] = [{"value": value, "count": count} + for value, count in sorted( + memory_type_counts.items(), key=lambda item: (-item[1], item[0]) + )[:PUBLIC_FACET_LIMIT]] + bridges = _complete_bridges(all_nodes, complete_edges) + return { + "meta": { + "workspace": workspace, + "level": "complete", + "complete_scene": True, + "scene_hash": scene_hash, + "index_generation": index_generation, + "total_nodes": len(scene_nodes), + "total_edges": len(complete_edges), + "shown_nodes": len(scene_nodes), + "shown_edges": len(complete_edges), + "entity_nodes": len(entity_nodes), + "memory_nodes": len(memory_nodes), + "raw_relations": len(raw_relations), + "evidence_connectors": len(evidence_edges), + "memory_connectors": len(memory_link_edges), + "code_memory_connectors": len(code_memory_edges), + "truncated": False, + "degraded": False, + "safety_state": "full", + "query_ms": 0.0, + "layout_seed": layout_seed, + "index_state": "ready", + "filters": filters, + "algorithm_version": f"{ALGORITHM_VERSION}-complete-1", + }, + "nodes": scene_nodes, + "edges": complete_edges, + "communities": communities, + "community_bridges": bridges, + "facets": facets, + } + + +def build_graph_scene( + workspace: str, + entity_rows: Sequence[Mapping[str, Any]], + edge_rows: Sequence[Mapping[str, Any]], + support_rows: Sequence[Mapping[str, Any]], + *, + memory_rows: Sequence[Mapping[str, Any]] = (), + memory_link_rows: Sequence[Mapping[str, Any]] = (), + code_memory_link_rows: Sequence[Mapping[str, Any]] = (), + level: str = "overview", + center_id: Optional[str] = None, + system_id: Optional[str] = None, + seeds: Optional[Sequence[str]] = None, + depth: int = 1, + node_limit: Optional[int] = None, + edge_limit: Optional[int] = None, + include_weak_cooccurrence: bool = False, + layers: Optional[set[str]] = None, + relations: Optional[set[str]] = None, + min_support: int = 1, + min_confidence: float = 0.0, + filters: Optional[dict] = None, + index_generation: int = 4, +) -> dict[str, Any]: + level = level if level in { + "overview", "system", "neighborhood", "path", "complete" + } else "overview" + graph = build_canonical_graph( + entity_rows, edge_rows, support_rows, + include_weak_cooccurrence=include_weak_cooccurrence, + layers=layers, relations=relations, + min_support=min_support, min_confidence=min_confidence, + ) + if level == "complete": + return _build_complete_scene( + workspace, graph, edge_rows, support_rows, memory_rows, + memory_link_rows, code_memory_link_rows, + include_weak_cooccurrence=include_weak_cooccurrence, + layers=layers, relations=relations, min_support=min_support, + min_confidence=min_confidence, filters=filters or {}, + index_generation=index_generation, + ) + caps = { + "overview": (80, 80), + "system": (150, 400), + "neighborhood": (100, 250), + "path": (100, 250), + } + default_node_cap, default_edge_cap = caps[level] + node_cap = min(300, max(1, int(node_limit or default_node_cap))) + edge_cap = min(900, max(0, int(edge_limit if edge_limit is not None else default_edge_cap))) + nodes = graph["nodes"] + ranked_nodes = sorted(nodes, key=lambda node_id: (-nodes[node_id]["scene_rank"], node_id)) + ranked_communities = sorted(graph["community_members"], key=lambda community_id: ( + -_community_mass(graph, graph["community_members"][community_id]), community_id + )) + if graph["global_anchor"]: + core_community = nodes[graph["global_anchor"]]["community_id"] + ranked_communities = [core_community] + [community_id for community_id in ranked_communities + if community_id != core_community] + + selected: set[str] = set() + chosen_communities: set[str] = set() + requested_ids = [value for value in [center_id, *(seeds or [])] if value] + canonical_requested = [graph["member_to_canonical"].get(value, value) + for value in requested_ids] + explicit_requested = {node_id for node_id in canonical_requested if node_id in nodes} + + def eligible(node_id: str) -> bool: + return nodes[node_id]["entity_quality"] > 0 or node_id in explicit_requested + + if system_id: + target_system = system_id + if target_system not in graph["community_members"]: + canonical = graph["member_to_canonical"].get(system_id, system_id) + if canonical in nodes: + explicit_requested.add(canonical) + target_system = nodes.get(canonical, {}).get("community_id", "") + if target_system in graph["community_members"]: + chosen_communities.add(target_system) + selected.update( + node_id for node_id in graph["community_members"][target_system] + if eligible(node_id) + ) + elif canonical_requested: + adjacent: dict[str, set[str]] = defaultdict(set) + for edge in graph["edges"]: + adjacent[edge["source"]].add(edge["target"]) + adjacent[edge["target"]].add(edge["source"]) + queue = deque((node_id, 0) for node_id in canonical_requested if node_id in nodes) + visited: set[str] = set() + while queue: + node_id, distance = queue.popleft() + if node_id in visited or distance > max(0, min(2, int(depth))): + continue + visited.add(node_id) + if eligible(node_id): + selected.add(node_id) + chosen_communities.add(nodes[node_id]["community_id"]) + for neighbor in sorted(adjacent[node_id]): + queue.append((neighbor, distance + 1)) + elif level == "overview": + overview_communities = [ + community_id for community_id in ranked_communities + if any(nodes[node_id]["entity_quality"] > 0 + for node_id in graph["community_members"][community_id]) + ][:24] + chosen_communities.update(overview_communities) + anchors = [graph["community_anchors"][community_id] + for community_id in overview_communities + if nodes[graph["community_anchors"][community_id]]["entity_quality"] > 0] + selected.update(anchors[:node_cap]) + for node_id in ranked_nodes: + if len(selected) >= node_cap: + break + if (nodes[node_id]["community_id"] in chosen_communities + and nodes[node_id]["entity_quality"] > 0): + selected.add(node_id) + else: + target = ranked_communities[0] if ranked_communities else "" + if target: + chosen_communities.add(target) + selected.update( + node_id for node_id in graph["community_members"][target] + if eligible(node_id) + ) + + if len(selected) > node_cap: + forced = { + graph["community_anchors"][community_id] for community_id in chosen_communities + } + forced.add(graph["global_anchor"]) + forced.update(explicit_requested) + selected = set(sorted( + (node_id for node_id in forced if node_id in selected and eligible(node_id)), + key=lambda node_id: ( + 0 if node_id in explicit_requested else 1, + 0 if node_id == graph["global_anchor"] else 1, + -nodes[node_id]["scene_rank"], node_id, + ), + )[:node_cap]) + for node_id in ranked_nodes: + if len(selected) >= node_cap: + break + if eligible(node_id) and ( + not chosen_communities or nodes[node_id]["community_id"] in chosen_communities + ): + selected.add(node_id) + chosen_communities = {nodes[node_id]["community_id"] for node_id in selected} + scene_edges = _selected_edges(graph, selected, level, edge_cap) + communities = _community_summaries(graph, chosen_communities, selected) + bridges = _bridges(graph, chosen_communities, 80) + + hash_payload = { + "algorithm": ALGORITHM_VERSION, + "index_generation": index_generation, + "workspace": workspace, + "level": level, + "filters": filters or {}, + "nodes": [ + ( + node_id, nodes[node_id]["label"], nodes[node_id]["type"], + nodes[node_id]["mass_score"], nodes[node_id]["gravity_mass"], + nodes[node_id]["visual_radius"], nodes[node_id]["community_id"], + nodes[node_id]["anchor_role"], nodes[node_id]["scene_rank"], + ) + for node_id in sorted(selected) + ], + "edges": [ + ( + edge["id"], edge["strength"], edge["rest_length"], + edge["spring_strength"], edge["support_count"], edge["confidence"], + edge["tier"], edge["visible_by_default"], + ) + for edge in sorted(scene_edges, key=lambda item: item["id"]) + ], + "communities": [ + ( + community["id"], community["anchor_id"], community["mass"], + community["radius"], community["member_count"], + community["shown_member_count"], + ) + for community in sorted(communities, key=lambda item: item["id"]) + ], + "bridges": [ + ( + bridge["id"], bridge["aggregate_strength"], + bridge["physics_strength"], bridge["support_count"], + bridge["edge_count"], + ) + for bridge in sorted(bridges, key=lambda item: item["id"]) + ], + } + scene_hash = hashlib.sha256(json.dumps( + hash_payload, sort_keys=True, separators=(",", ":") + ).encode("utf-8")).hexdigest() + layout_seed = int(scene_hash[:8], 16) + + community_positions: dict[str, tuple[float, float]] = {} + for index, community in enumerate(communities): + if graph["global_anchor"] in graph["community_members"][community["id"]]: + community_positions[community["id"]] = (0.0, 0.0) + continue + radius = 145.0 * math.sqrt(index + 1) + angle = GOLDEN_ANGLE * (index + 1) + (layout_seed % 360) * math.pi / 180.0 + community_positions[community["id"]] = ( + radius * math.cos(angle), radius * math.sin(angle) + ) + scene_nodes = [] + ranks_by_community: dict[str, int] = defaultdict(int) + for node_id in sorted(selected, key=lambda value: (-nodes[value]["scene_rank"], value)): + node = dict(nodes[node_id]) + community_id = node["community_id"] + center_x, center_y = community_positions.get(community_id, (0.0, 0.0)) + rank = ranks_by_community[community_id] + ranks_by_community[community_id] += 1 + if node["anchor_role"] in {"global", "community"}: + x, y = center_x, center_y + else: + system_radius = next( + item["radius"] for item in communities if item["id"] == community_id + ) + orbit = _clamp( + 14.0 + 9.0 + ((1.0 - node["mass_score"]) ** 1.4) + * (system_radius - 18.0), 14.0, system_radius + ) + angle = GOLDEN_ANGLE * (rank + 1) + (layout_seed % 180) * math.pi / 180.0 + x, y = center_x + orbit * math.cos(angle), center_y + orbit * math.sin(angle) + node["x"], node["y"] = round(x, 6), round(y, 6) + node.pop("aliases", None) + node.pop("anchor_eligible", None) + scene_nodes.append(node) + + return { + "meta": { + "workspace": workspace, + "level": level, + "scene_hash": scene_hash, + "index_generation": index_generation, + "total_nodes": len(nodes), + "total_edges": len(graph["edges"]), + "shown_nodes": len(scene_nodes), + "shown_edges": len(scene_edges), + "truncated": len(scene_nodes) < len(nodes) or len(scene_edges) < len(graph["edges"]), + "query_ms": 0.0, + "layout_seed": layout_seed, + "index_state": "ready", + "filters": filters or {}, + "algorithm_version": ALGORITHM_VERSION, + }, + "nodes": scene_nodes, + "edges": scene_edges, + "communities": communities, + "community_bridges": bridges, + "facets": _facets(graph), + } + + +def strongest_path(graph: dict[str, Any], source: str, target: str, *, + max_hops: int = 8, max_visits: int = 10_000) -> dict[str, Any]: + source_id = graph["member_to_canonical"].get(source, source) + target_id = graph["member_to_canonical"].get(target, target) + if source_id not in graph["nodes"] or target_id not in graph["nodes"]: + return {"found": False, "node_ids": [], "edge_ids": [], "nodes": [], + "edges": [], "cost": None, "hops": 0} + adjacency: dict[str, list[tuple[str, dict, float]]] = defaultdict(list) + penalties = {"entity": 0.0, "causal": 0.0, "temporal": 0.1, "semantic": 0.2} + for edge in graph["edges"]: + cost = -math.log(max(float(edge["strength"]), 0.02)) + cost += 1.0 if edge["relation"] == "co_occurs" else penalties.get(edge["layer"], 0.2) + adjacency[edge["source"]].append((edge["target"], edge, cost)) + adjacency[edge["target"]].append((edge["source"], edge, cost)) + heap: list[tuple[float, int, str, tuple[str, ...], tuple[str, ...]]] = [ + (0.0, 0, source_id, (source_id,), ()) + ] + best: dict[tuple[str, int], float] = {(source_id, 0): 0.0} + visits = 0 + while heap and visits < max(1, max_visits): + cost, hops, node_id, path_nodes, path_edges = heapq.heappop(heap) + visits += 1 + if node_id == target_id: + edge_by_id = {edge["id"]: edge for edge in graph["edges"]} + return { + "found": True, + "node_ids": list(path_nodes), + "edge_ids": list(path_edges), + "nodes": [ + {key: item for key, item in graph["nodes"][value].items() + if not key.startswith("_") and key != "anchor_eligible"} + for value in path_nodes + ], + "edges": [ + {key: item for key, item in edge_by_id[value].items() + if not key.startswith("_")} + for value in path_edges + ], + "cost": round(cost, 6), + "hops": hops, + "visited": visits, + } + if hops >= max(1, min(8, int(max_hops))): + continue + for neighbor, edge, edge_cost in sorted( + adjacency[node_id], key=lambda item: (item[2], item[1]["id"], item[0]) + ): + if neighbor in path_nodes: + continue + next_cost = cost + edge_cost + key = (neighbor, hops + 1) + if next_cost + 1e-12 >= best.get(key, math.inf): + continue + best[key] = next_cost + heapq.heappush(heap, ( + next_cost, hops + 1, neighbor, + (*path_nodes, neighbor), (*path_edges, edge["id"]), + )) + return {"found": False, "node_ids": [], "edge_ids": [], "nodes": [], + "edges": [], "cost": None, "hops": 0, "visited": visits} diff --git a/engraphis/core/schema.py b/engraphis/core/schema.py index 36bd4d0..140cd84 100644 --- a/engraphis/core/schema.py +++ b/engraphis/core/schema.py @@ -8,7 +8,7 @@ """ from __future__ import annotations -SCHEMA_VERSION = 3 +SCHEMA_VERSION = 4 SCHEMA_SQL = """ CREATE TABLE IF NOT EXISTS schema_migrations ( @@ -100,6 +100,9 @@ name TEXT, etype TEXT, canonical_id TEXT, -- cross-repo entity resolution + normalized_name TEXT NOT NULL DEFAULT '', + canonical_method TEXT NOT NULL DEFAULT 'exact', + canonical_confidence REAL NOT NULL DEFAULT 1.0, created_at REAL, UNIQUE(workspace_id, repo_id, name, etype) ); @@ -128,6 +131,131 @@ CREATE INDEX IF NOT EXISTS idx_edge_workspace_repo ON edges(workspace_id, repo_id, valid_to, expired_at); +-- Evidence is normalized into an indexed, bi-temporal table. ``edges.provenance`` +-- remains populated for one compatibility release and for legacy exports. +CREATE TABLE IF NOT EXISTS edge_supports ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + edge_id TEXT NOT NULL, + memory_id TEXT NOT NULL, + source_kind TEXT NOT NULL DEFAULT 'legacy_unknown', + confidence REAL NOT NULL DEFAULT 0.5, + valid_from REAL, + valid_to REAL, + ingested_at REAL, + expired_at REAL, + provenance TEXT DEFAULT '{}', + FOREIGN KEY(edge_id) REFERENCES edges(id) ON DELETE CASCADE +); +CREATE UNIQUE INDEX IF NOT EXISTS idx_edge_support_live_unique + ON edge_supports(edge_id, memory_id, source_kind) + WHERE valid_to IS NULL AND expired_at IS NULL; +CREATE INDEX IF NOT EXISTS idx_edge_support_edge + ON edge_supports(edge_id, valid_to, expired_at); +CREATE INDEX IF NOT EXISTS idx_edge_support_memory + ON edge_supports(memory_id, valid_to, expired_at); + +-- Explicit, persisted derived-index work. Graph reads never backfill implicitly; +-- writers run one of these bounded jobs and readers receive a rebuilding state while +-- a mutating job is active. ``jobs`` is generic enough for later v2 maintenance jobs, +-- while ``graph_index_state`` is the cheap generation/state lookup used by scene caches. +CREATE TABLE IF NOT EXISTS jobs ( + id TEXT PRIMARY KEY, + workspace_id TEXT NOT NULL, + repo_id TEXT, + kind TEXT NOT NULL, + state TEXT NOT NULL DEFAULT 'queued', + dry_run INTEGER NOT NULL DEFAULT 1, + total_items INTEGER NOT NULL DEFAULT 0, + processed_items INTEGER NOT NULL DEFAULT 0, + counts TEXT NOT NULL DEFAULT '{}', + errors TEXT NOT NULL DEFAULT '[]', + request TEXT NOT NULL DEFAULT '{}', + cancel_requested INTEGER NOT NULL DEFAULT 0, + runner_id TEXT, + heartbeat_at REAL, + created_at REAL NOT NULL, + started_at REAL, + finished_at REAL +); +CREATE INDEX IF NOT EXISTS idx_jobs_scope_state + ON jobs(workspace_id, kind, state, created_at); + +CREATE TABLE IF NOT EXISTS graph_index_state ( + workspace_id TEXT PRIMARY KEY, + generation INTEGER NOT NULL DEFAULT 1, + state TEXT NOT NULL DEFAULT 'ready', + active_job_id TEXT, + updated_at REAL NOT NULL, + last_error TEXT NOT NULL DEFAULT '' +); + +CREATE TRIGGER IF NOT EXISTS trg_graph_generation_entity_insert +AFTER INSERT ON entities WHEN NEW.workspace_id IS NOT NULL BEGIN + INSERT INTO graph_index_state(workspace_id, generation, state, updated_at) + VALUES(NEW.workspace_id, 1, 'ready', CAST(strftime('%s','now') AS REAL)) + ON CONFLICT(workspace_id) DO UPDATE SET + generation=graph_index_state.generation+1, updated_at=excluded.updated_at; +END; +CREATE TRIGGER IF NOT EXISTS trg_graph_generation_entity_update +AFTER UPDATE ON entities WHEN NEW.workspace_id IS NOT NULL BEGIN + INSERT INTO graph_index_state(workspace_id, generation, state, updated_at) + VALUES(NEW.workspace_id, 1, 'ready', CAST(strftime('%s','now') AS REAL)) + ON CONFLICT(workspace_id) DO UPDATE SET + generation=graph_index_state.generation+1, updated_at=excluded.updated_at; +END; +CREATE TRIGGER IF NOT EXISTS trg_graph_generation_entity_delete +AFTER DELETE ON entities WHEN OLD.workspace_id IS NOT NULL BEGIN + INSERT INTO graph_index_state(workspace_id, generation, state, updated_at) + VALUES(OLD.workspace_id, 1, 'ready', CAST(strftime('%s','now') AS REAL)) + ON CONFLICT(workspace_id) DO UPDATE SET + generation=graph_index_state.generation+1, updated_at=excluded.updated_at; +END; +CREATE TRIGGER IF NOT EXISTS trg_graph_generation_edge_insert +AFTER INSERT ON edges WHEN NEW.workspace_id IS NOT NULL BEGIN + INSERT INTO graph_index_state(workspace_id, generation, state, updated_at) + VALUES(NEW.workspace_id, 1, 'ready', CAST(strftime('%s','now') AS REAL)) + ON CONFLICT(workspace_id) DO UPDATE SET + generation=graph_index_state.generation+1, updated_at=excluded.updated_at; +END; +CREATE TRIGGER IF NOT EXISTS trg_graph_generation_edge_update +AFTER UPDATE ON edges WHEN NEW.workspace_id IS NOT NULL BEGIN + INSERT INTO graph_index_state(workspace_id, generation, state, updated_at) + VALUES(NEW.workspace_id, 1, 'ready', CAST(strftime('%s','now') AS REAL)) + ON CONFLICT(workspace_id) DO UPDATE SET + generation=graph_index_state.generation+1, updated_at=excluded.updated_at; +END; +CREATE TRIGGER IF NOT EXISTS trg_graph_generation_edge_delete +AFTER DELETE ON edges WHEN OLD.workspace_id IS NOT NULL BEGIN + INSERT INTO graph_index_state(workspace_id, generation, state, updated_at) + VALUES(OLD.workspace_id, 1, 'ready', CAST(strftime('%s','now') AS REAL)) + ON CONFLICT(workspace_id) DO UPDATE SET + generation=graph_index_state.generation+1, updated_at=excluded.updated_at; +END; +CREATE TRIGGER IF NOT EXISTS trg_graph_generation_support_insert +AFTER INSERT ON edge_supports BEGIN + INSERT INTO graph_index_state(workspace_id, generation, state, updated_at) + SELECT workspace_id, 1, 'ready', CAST(strftime('%s','now') AS REAL) + FROM edges WHERE id=NEW.edge_id + ON CONFLICT(workspace_id) DO UPDATE SET + generation=graph_index_state.generation+1, updated_at=excluded.updated_at; +END; +CREATE TRIGGER IF NOT EXISTS trg_graph_generation_support_update +AFTER UPDATE ON edge_supports BEGIN + INSERT INTO graph_index_state(workspace_id, generation, state, updated_at) + SELECT workspace_id, 1, 'ready', CAST(strftime('%s','now') AS REAL) + FROM edges WHERE id=NEW.edge_id + ON CONFLICT(workspace_id) DO UPDATE SET + generation=graph_index_state.generation+1, updated_at=excluded.updated_at; +END; +CREATE TRIGGER IF NOT EXISTS trg_graph_generation_support_delete +AFTER DELETE ON edge_supports BEGIN + INSERT INTO graph_index_state(workspace_id, generation, state, updated_at) + SELECT workspace_id, 1, 'ready', CAST(strftime('%s','now') AS REAL) + FROM edges WHERE id=OLD.edge_id + ON CONFLICT(workspace_id) DO UPDATE SET + generation=graph_index_state.generation+1, updated_at=excluded.updated_at; +END; + CREATE TABLE IF NOT EXISTS mem_links ( a TEXT, b TEXT, diff --git a/engraphis/core/store.py b/engraphis/core/store.py index df51ddb..bc8835c 100644 --- a/engraphis/core/store.py +++ b/engraphis/core/store.py @@ -12,9 +12,11 @@ import hashlib import json +import re import sqlite3 import threading import time +import unicodedata from pathlib import Path from typing import Any, Callable, Iterable, Optional @@ -80,7 +82,11 @@ def _provenance_memory_ids(provenance: Any) -> list[str]: return [] values = [provenance.get("memory_id")] many = provenance.get("memory_ids") - if isinstance(many, (list, tuple, set)): + if isinstance(many, set): + # Sets are tolerated for compatibility but have no declared order. Sort them + # so they cannot make persisted provenance vary across interpreter processes. + values.extend(sorted(many, key=lambda value: str(value))) + elif isinstance(many, (list, tuple)): values.extend(many) out: list[str] = [] for value in values: @@ -90,13 +96,105 @@ def _provenance_memory_ids(provenance: Any) -> list[str]: return out +def _merge_edge_provenance(values: Iterable[Any], *, merged_ids: Iterable[str] = ()) -> dict: + """Merge compatibility provenance while normalized supports remain authoritative.""" + documents = [value for value in values if isinstance(value, dict)] + merged = dict(documents[0]) if documents else {} + memory_ids: list[str] = [] + sources: set[str] = set() + confidences: list[float] = [] + for document in documents: + for key, value in document.items(): + merged.setdefault(key, value) + for memory_id in _provenance_memory_ids(document): + if memory_id not in memory_ids: + memory_ids.append(memory_id) + source = str(document.get("source") or "") + if source: + sources.add(source) + try: + if document.get("confidence") is not None: + confidences.append(float(document["confidence"])) + except (TypeError, ValueError): + pass + if memory_ids: + # ``memory_id`` is the declared primary source, not the lexicographically + # smallest ULID. ULIDs created in one millisecond do not have a meaningful + # random-suffix order, so sorting here could silently change provenance. + merged["memory_id"] = memory_ids[0] + merged["memory_ids"] = memory_ids + if sources: + merged.setdefault("source", sorted(sources)[0]) + if len(sources) > 1: + merged["sources"] = sorted(sources) + if confidences: + merged["confidence"] = max(confidences) + merged_from = sorted({str(value) for value in merged_ids if value}) + if merged_from: + merged["canonical_deduplicated_from"] = merged_from + return merged + + +def normalize_entity_name(value: str) -> str: + """Conservative canonicalization key used by schema v4. + + It deliberately performs no fuzzy or semantic matching: exact Unicode NFKC, + case-folded, whitespace-normalized variants may share a canonical entity, while + punctuation, type, and workspace remain hard boundaries. Preserving punctuation is + important for names such as ``C++``/``C#`` and ``AT&T``/``ATT``; deleting it would + silently conflate distinct entities. + """ + text = unicodedata.normalize("NFKC", str(value or "")).casefold() + return re.sub(r"\s+", " ", text).strip() + + +_SUPPORT_CONFIDENCE = { + "manual": 1.0, + "schema": 1.0, + "structured": 0.80, + "regex_proximity": 0.55, + "legacy_unknown": 0.50, + "co_occurrence": 0.25, +} + + +def _edge_source_kind(provenance: Any, relation: str = "") -> str: + if relation == "co_occurs": + return "co_occurrence" + if not isinstance(provenance, dict): + return "legacy_unknown" + raw = str( + provenance.get("source_kind") or provenance.get("source") or "" + ).casefold() + if "manual" in raw: + return "manual" + if "schema" in raw: + return "schema" + if "structured" in raw: + return "structured" + if "regex" in raw or "proximity" in raw or "backfill" in raw: + return "regex_proximity" + return "legacy_unknown" + + +def _edge_support_confidence(provenance: Any, source_kind: str) -> float: + raw = provenance.get("confidence") if isinstance(provenance, dict) else None + try: + if raw is not None: + return max(0.0, min(1.0, float(raw))) + except (TypeError, ValueError): + pass + return _SUPPORT_CONFIDENCE.get(source_kind, 0.50) + + def _receipt_metadata(metadata: dict) -> dict: """Keep receipt metadata useful but content-free and bounded.""" allowed = { "mtype", "scope", "resolution", "retention", "extracted", "intent", "k", "result_count", "grounded", "citations", "relation", "layer", "graph_layers", "files_scanned", "files_indexed", "files_removed", "symbols", "edges", - "entities", "relations", "tables", + "entities", "relations", "tables", "dry_run", "error_count", + "entities_added", "relations_added", } out: dict[str, Any] = {} for key in sorted(metadata, key=lambda item: str(item))[:24]: @@ -371,11 +469,16 @@ def init_schema(self) -> None: for stmt in ( "ALTER TABLE memories ADD COLUMN sort_order REAL", "ALTER TABLE edges ADD COLUMN layer TEXT DEFAULT 'semantic'", + "ALTER TABLE entities ADD COLUMN normalized_name TEXT NOT NULL DEFAULT ''", + "ALTER TABLE entities ADD COLUMN canonical_method TEXT NOT NULL DEFAULT 'exact'", + "ALTER TABLE entities ADD COLUMN canonical_confidence REAL NOT NULL DEFAULT 1.0", "ALTER TABLE mem_links ADD COLUMN layer TEXT DEFAULT 'semantic'", "ALTER TABLE mem_links ADD COLUMN reason TEXT DEFAULT ''", "ALTER TABLE code_edges ADD COLUMN layer TEXT DEFAULT 'entity'", "ALTER TABLE symbols ADD COLUMN docstring TEXT DEFAULT ''", "ALTER TABLE receipt_chain_heads ADD COLUMN integrity_error TEXT DEFAULT ''", + "ALTER TABLE jobs ADD COLUMN runner_id TEXT", + "ALTER TABLE jobs ADD COLUMN heartbeat_at REAL", ): try: self.conn.execute(stmt) @@ -397,6 +500,43 @@ def init_schema(self) -> None: f"UPDATE {table} SET layer=? WHERE rowid=?", (inferred, row["rowid"]), ) + # v4 makes canonical identity and edge evidence explicit and indexed. Run the + # backfills before creating representative-only uniqueness indexes so exact + # normalized aliases can safely converge onto one deterministic canonical id. + self._backfill_entity_canonicalization() + self.conn.executescript( + "CREATE UNIQUE INDEX IF NOT EXISTS idx_entity_workspace_canonical " + "ON entities(workspace_id, normalized_name, etype) " + "WHERE repo_id IS NULL AND canonical_id=id AND normalized_name<>'';" + "CREATE UNIQUE INDEX IF NOT EXISTS idx_entity_repo_canonical " + "ON entities(workspace_id, repo_id, normalized_name, etype) " + "WHERE repo_id IS NOT NULL AND canonical_id=id AND normalized_name<>'';" + "CREATE INDEX IF NOT EXISTS idx_entity_canonical " + "ON entities(workspace_id, canonical_id);" + "CREATE INDEX IF NOT EXISTS idx_entity_normalized " + "ON entities(workspace_id, normalized_name, etype);" + ) + self._backfill_edge_supports() + self._deduplicate_live_edges() + self.conn.executescript( + "CREATE UNIQUE INDEX IF NOT EXISTS idx_edge_workspace_live_unique " + "ON edges(workspace_id, src, dst, relation, layer) " + "WHERE workspace_id IS NOT NULL AND repo_id IS NULL " + "AND valid_to IS NULL AND expired_at IS NULL;" + "CREATE UNIQUE INDEX IF NOT EXISTS idx_edge_repo_live_unique " + "ON edges(workspace_id, repo_id, src, dst, relation, layer) " + "WHERE workspace_id IS NOT NULL AND repo_id IS NOT NULL " + "AND valid_to IS NULL AND expired_at IS NULL;" + ) + # Every workspace has a cheap graph generation/state row, including databases + # that already contained graph data before the v4 explorer tables were added. + # Triggers in SCHEMA_SQL advance the generation on subsequent graph mutations. + self.conn.execute( + "INSERT OR IGNORE INTO graph_index_state " + "(workspace_id, generation, state, active_job_id, updated_at, last_error) " + "SELECT id, 1, 'ready', NULL, ?, '' FROM workspaces", + (now_ts(),), + ) # Backfill the independent receipt anchor for databases created before the # anchor table existed. From this point onward every append updates it atomically, # allowing verification to detect deletion of the newest receipt as well as an @@ -424,6 +564,230 @@ def init_schema(self) -> None: ) self.conn.commit() + def _backfill_entity_canonicalization(self) -> None: + rows = [dict(row) for row in self.conn.execute( + "SELECT id, workspace_id, name, etype, canonical_id, normalized_name, " + "canonical_method, canonical_confidence FROM entities " + "ORDER BY workspace_id, etype, id" + ).fetchall()] + groups: dict[tuple[str, str, str], list[dict]] = {} + for row in rows: + normalized = normalize_entity_name(row.get("name") or "") + row["_normalized"] = normalized + key = (str(row.get("workspace_id") or ""), str(row.get("etype") or ""), normalized) + groups.setdefault(key, []).append(row) + for members in groups.values(): + # Existing canonical ids win when present; otherwise the oldest typed id + # is the deterministic representative. Exact variants never cross a + # workspace or entity-type boundary. + existing = sorted({str(row.get("canonical_id") or "") for row in members + if row.get("canonical_id")}) + canonical_id = existing[0] if existing else min(row["id"] for row in members) + merged = len(members) > 1 + for row in members: + method = row.get("canonical_method") or ( + "exact_normalized" if merged else "identity" + ) + if not row.get("canonical_id"): + method = "exact_normalized" if merged else "identity" + # A pre-release v4 build briefly stripped all punctuation. Reopening + # such a database with the conservative normalizer can split a false + # merge (for example C++ vs C#). A singleton that was joined only by + # that automatic method must become its own representative again; + # caller-provided canonical ids remain authoritative. + if not merged and method == "exact_normalized" \ + and row.get("canonical_id") != row["id"]: + canonical_id = row["id"] + method = "identity" + confidence = float(row.get("canonical_confidence") or 1.0) + if ( + row.get("normalized_name") == row["_normalized"] + and row.get("canonical_id") == canonical_id + and row.get("canonical_method") == method + and float(row.get("canonical_confidence") or 0.0) == confidence + ): + continue + self.conn.execute( + "UPDATE entities SET normalized_name=?, canonical_id=?, " + "canonical_method=?, canonical_confidence=? WHERE id=?", + (row["_normalized"], canonical_id, method, confidence, row["id"]), + ) + + def _backfill_edge_supports(self) -> None: + rows = self.conn.execute( + "SELECT id, relation, valid_from, valid_to, ingested_at, expired_at, provenance " + "FROM edges" + ).fetchall() + for row in rows: + provenance = _loads(row["provenance"], {}) + source_kind = _edge_source_kind(provenance, row["relation"] or "") + confidence = _edge_support_confidence(provenance, source_kind) + for memory_id in _provenance_memory_ids(provenance): + # This migration backfill is intentionally append-once. The live-row + # uniqueness index cannot make an ``INSERT OR IGNORE`` idempotent for + # historical supports because partial indexes exclude closed rows. In + # addition to inflating the graph generation on every process start, + # blindly inserting here would resurrect evidence that was explicitly + # invalidated. Any row for this legacy edge/memory/source triple proves + # that its provenance has already been normalized; later lifecycle + # changes remain authoritative. + existing = self.conn.execute( + "SELECT 1 FROM edge_supports WHERE edge_id=? AND memory_id=? " + "AND source_kind=? LIMIT 1", + (row["id"], memory_id, source_kind), + ).fetchone() + if existing is not None: + continue + self.conn.execute( + "INSERT INTO edge_supports " + "(edge_id, memory_id, source_kind, confidence, valid_from, valid_to, " + "ingested_at, expired_at, provenance) VALUES (?,?,?,?,?,?,?,?,?)", + (row["id"], memory_id, source_kind, confidence, + row["valid_from"], row["valid_to"], row["ingested_at"], + row["expired_at"], _dumps(provenance)), + ) + + def _deduplicate_live_edges(self) -> None: + """Converge equivalent live relations without discarding temporal history.""" + rows = [dict(row) for row in self.conn.execute( + "SELECT id, workspace_id, repo_id, src, dst, relation, layer, weight, " + "valid_from, ingested_at, provenance FROM edges " + "WHERE workspace_id IS NOT NULL AND valid_to IS NULL AND expired_at IS NULL " + "ORDER BY workspace_id, repo_id, src, dst, relation, layer, " + "COALESCE(valid_from, ingested_at), id" + ).fetchall()] + groups: dict[tuple, list[dict]] = {} + for row in rows: + source, target = row["src"], row["dst"] + if row["relation"] in {"co_occurs", "related", "associated_with"} \ + and target < source: + source, target = target, source + row["_normalized_src"] = source + row["_normalized_dst"] = target + key = ( + row["workspace_id"], row["repo_id"], source, target, + row["relation"], row["layer"], + ) + groups.setdefault(key, []).append(row) + closed_at = now_ts() + workspace_counts: dict[str, int] = {} + for duplicates in groups.values(): + if len(duplicates) < 2: + row = duplicates[0] + if (row["src"], row["dst"]) != ( + row["_normalized_src"], row["_normalized_dst"]): + self.conn.execute( + "UPDATE edges SET src=?, dst=? WHERE id=?", + (row["_normalized_src"], row["_normalized_dst"], row["id"]), + ) + continue + duplicates.sort(key=lambda row: ( + row["valid_from"] if row["valid_from"] is not None + else row["ingested_at"] if row["ingested_at"] is not None + else float("inf"), + row["id"], + )) + survivor, retired = duplicates[0], duplicates[1:] + retired_ids = [row["id"] for row in retired] + all_ids = [survivor["id"], *retired_ids] + marks = ",".join("?" for _ in all_ids) + support_rows = self.conn.execute( + "SELECT memory_id, source_kind, confidence, valid_from, ingested_at, " + "provenance FROM edge_supports WHERE edge_id IN (" + marks + ") " + "AND valid_to IS NULL AND expired_at IS NULL ORDER BY id", + all_ids, + ).fetchall() + for support in support_rows: + current = self.conn.execute( + "SELECT id, confidence, valid_from, ingested_at, provenance " + "FROM edge_supports WHERE edge_id=? " + "AND memory_id=? AND source_kind=? AND valid_to IS NULL " + "AND expired_at IS NULL", + (survivor["id"], support["memory_id"], support["source_kind"]), + ).fetchone() + if current is None: + self.conn.execute( + "INSERT INTO edge_supports " + "(edge_id, memory_id, source_kind, confidence, valid_from, " + "ingested_at, provenance) VALUES (?,?,?,?,?,?,?)", + ( + survivor["id"], support["memory_id"], + support["source_kind"], support["confidence"], + support["valid_from"], support["ingested_at"], + support["provenance"], + ), + ) + else: + confidence = max( + float(support["confidence"] or 0.0), + float(current["confidence"] or 0.0), + ) + provenance = _merge_edge_provenance([ + _loads(current["provenance"], {}), + _loads(support["provenance"], {}), + ]) + provenance["confidence"] = confidence + support_valid = [value for value in ( + current["valid_from"], support["valid_from"] + ) if value is not None] + support_ingested = [value for value in ( + current["ingested_at"], support["ingested_at"] + ) if value is not None] + self.conn.execute( + "UPDATE edge_supports SET confidence=?, valid_from=?, " + "ingested_at=?, provenance=? WHERE id=?", + ( + confidence, min(support_valid) if support_valid else None, + min(support_ingested) if support_ingested else None, + _dumps(provenance), current["id"], + ), + ) + provenances = [_loads(row["provenance"], {}) for row in duplicates] + merged_provenance = _merge_edge_provenance( + provenances, merged_ids=retired_ids + ) + valid_values = [float(row["valid_from"]) for row in duplicates + if row["valid_from"] is not None] + ingested_values = [float(row["ingested_at"]) for row in duplicates + if row["ingested_at"] is not None] + for row in retired: + provenance = _loads(row["provenance"], {}) + if not isinstance(provenance, dict): + provenance = {} + provenance["canonical_deduplicated_into"] = survivor["id"] + self.conn.execute( + "UPDATE edges SET valid_to=?, provenance=? WHERE id=?", + (closed_at, _dumps(provenance), row["id"]), + ) + retired_marks = ",".join("?" for _ in retired_ids) + self.conn.execute( + "UPDATE edge_supports SET valid_to=? WHERE edge_id IN (" + + retired_marks + ") AND valid_to IS NULL AND expired_at IS NULL", + (closed_at, *retired_ids), + ) + # Retire duplicates before normalizing the survivor endpoints. A pre-release + # v4 database may already have the partial unique index; reversing the + # survivor first would temporarily collide with its still-live twin. + self.conn.execute( + "UPDATE edges SET src=?, dst=?, weight=?, valid_from=?, ingested_at=?, " + "provenance=? WHERE id=?", + ( + survivor["_normalized_src"], survivor["_normalized_dst"], + max(float(row["weight"] or 0.0) for row in duplicates), + min(valid_values) if valid_values else None, + min(ingested_values) if ingested_values else None, + _dumps(merged_provenance), survivor["id"], + ), + ) + workspace_counts[survivor["workspace_id"]] = ( + workspace_counts.get(survivor["workspace_id"], 0) + len(retired) + ) + for workspace_id, count in workspace_counts.items(): + self.audit( + "system", "graph_relation_deduplicate", workspace_id, + f"closed {count} duplicate live relations", commit=False, + ) + @property def schema_version(self) -> int: row = self.conn.execute("SELECT MAX(version) AS v FROM schema_migrations").fetchone() @@ -778,21 +1142,38 @@ def fts_search(self, query: str, k: int = 20, return [(r["id"], 0.5) for r in rows] # ── graph ───────────────────────────────────────────────────────────────── - def upsert_entity(self, node: Node) -> str: + def upsert_entity(self, node: Node, *, commit: bool = True) -> str: + normalized = normalize_entity_name(node.name) existing = self.conn.execute( - "SELECT id FROM entities WHERE workspace_id=? AND repo_id IS ? AND name=? AND etype IS ?", - (node.workspace_id, node.repo_id, node.name, node.ntype), + "SELECT id FROM entities WHERE workspace_id=? AND repo_id IS ? " + "AND normalized_name=? AND etype IS ? ORDER BY id LIMIT 1", + (node.workspace_id, node.repo_id, normalized, node.ntype), ).fetchone() if existing: return existing["id"] nid = node.id or ids.new_id("entity") + canonical_id = node.canonical_id + method = "provided" if canonical_id else "identity" + if not canonical_id: + canonical = self.conn.execute( + "SELECT COALESCE(canonical_id, id) AS canonical_id FROM entities " + "WHERE workspace_id=? AND normalized_name=? AND etype IS ? " + "ORDER BY id LIMIT 1", + (node.workspace_id, normalized, node.ntype), + ).fetchone() + if canonical: + canonical_id = canonical["canonical_id"] + method = "exact_normalized" + canonical_id = canonical_id or nid self.conn.execute( - "INSERT INTO entities(id, workspace_id, repo_id, name, etype, canonical_id, created_at) " - "VALUES (?,?,?,?,?,?,?)", + "INSERT INTO entities(id, workspace_id, repo_id, name, etype, canonical_id, " + "normalized_name, canonical_method, canonical_confidence, created_at) " + "VALUES (?,?,?,?,?,?,?,?,?,?)", (nid, node.workspace_id, node.repo_id, node.name, node.ntype, - node.canonical_id, now_ts()), + canonical_id, normalized, method, 1.0, now_ts()), ) - self.conn.commit() + if commit: + self.conn.commit() return nid def list_entities(self, flt: Optional[SearchFilter] = None, @@ -819,27 +1200,216 @@ def list_entities(self, flt: Optional[SearchFilter] = None, workspace_id=r["workspace_id"], repo_id=r["repo_id"], canonical_id=r["canonical_id"]) for r in rows] - def upsert_edge(self, edge: Edge) -> str: + def upsert_edge(self, edge: Edge, *, commit: bool = True) -> str: eid = edge.id or ids.new_id("edge") layer = normalize_graph_layer(edge.layer, edge.relation).value + source, target = edge.src, edge.dst + if edge.relation in {"co_occurs", "related", "associated_with"} and target < source: + source, target = target, source + incoming_provenance = _merge_edge_provenance([edge.provenance]) + existing = self.conn.execute( + "SELECT id, workspace_id, repo_id, src, dst, relation, layer, weight, " + "valid_from, valid_to, ingested_at, expired_at, provenance " + "FROM edges WHERE id=?", (eid,) + ).fetchone() + replacing = existing is not None + stored_provenance = _loads(existing["provenance"], {}) if existing else {} + incoming_supports = { + (memory_id, _edge_source_kind(incoming_provenance, edge.relation)) + for memory_id in _provenance_memory_ids(incoming_provenance) + } + stored_supports = { + (memory_id, _edge_source_kind(stored_provenance, edge.relation)) + for memory_id in _provenance_memory_ids(stored_provenance) + } + if existing is not None and edge.valid_to is None and edge.expired_at is None \ + and existing["valid_to"] is None and existing["expired_at"] is None \ + and incoming_supports == stored_supports \ + and ( + existing["workspace_id"], existing["repo_id"], + existing["src"], existing["dst"], existing["relation"], existing["layer"], + ) == ( + edge.workspace_id, edge.repo_id, source, target, edge.relation, layer, + ): + merged_provenance = _merge_edge_provenance( + [stored_provenance, incoming_provenance] + ) + desired_weight = max( + float(existing["weight"] or 0.0), float(edge.weight or 0.0) + ) + desired_valid_from = existing["valid_from"] + if edge.valid_from is not None: + desired_valid_from = min( + value for value in (existing["valid_from"], edge.valid_from) + if value is not None + ) + serialized_provenance = _dumps(merged_provenance) + if desired_weight != float(existing["weight"] or 0.0) \ + or desired_valid_from != existing["valid_from"] \ + or serialized_provenance != (existing["provenance"] or "{}"): + self.conn.execute( + "UPDATE edges SET weight=?, valid_from=?, provenance=? WHERE id=?", + (desired_weight, desired_valid_from, serialized_provenance, eid), + ) + self._write_edge_supports( + eid, edge.relation, incoming_provenance, + valid_from=edge.valid_from, valid_to=edge.valid_to, + ingested_at=edge.ingested_at, expired_at=edge.expired_at, + ) + if commit: + self.conn.commit() + return eid + equivalent = None + if edge.valid_to is None and edge.expired_at is None: + equivalent = self.conn.execute( + "SELECT id, weight, valid_from, provenance FROM edges " + "WHERE workspace_id IS ? AND repo_id IS ? AND src=? AND dst=? " + "AND relation=? AND layer=? AND valid_to IS NULL AND expired_at IS NULL " + "AND id<>? ORDER BY id LIMIT 1", + ( + edge.workspace_id, edge.repo_id, source, target, + edge.relation, layer, eid, + ), + ).fetchone() + if equivalent is not None: + if replacing: + closed_at = now_ts() + self.conn.execute( + "UPDATE edges SET valid_to=? WHERE id=? AND valid_to IS NULL", + (closed_at, eid), + ) + self.conn.execute( + "UPDATE edge_supports SET valid_to=? WHERE edge_id=? " + "AND valid_to IS NULL AND expired_at IS NULL", + (closed_at, eid), + ) + existing_provenance = _loads(equivalent["provenance"], {}) + merged_provenance = _merge_edge_provenance( + [existing_provenance, incoming_provenance], + merged_ids=[eid] if replacing else [], + ) + valid_values = [value for value in ( + equivalent["valid_from"], edge.valid_from + ) if value is not None] + self.conn.execute( + "UPDATE edges SET weight=?, valid_from=?, provenance=? WHERE id=?", + ( + max(float(equivalent["weight"] or 0.0), float(edge.weight or 0.0)), + min(valid_values) if valid_values else now_ts(), + _dumps(merged_provenance), equivalent["id"], + ), + ) + self._write_edge_supports( + equivalent["id"], edge.relation, incoming_provenance, + valid_from=edge.valid_from, valid_to=edge.valid_to, + ingested_at=edge.ingested_at, expired_at=edge.expired_at, + ) + if commit: + self.conn.commit() + return str(equivalent["id"]) + if replacing: + # ``upsert_edge`` replaces the supplied edge record. Close its previous + # normalized evidence before writing the replacement so sources removed + # from the new provenance cannot remain live invisibly. + self.conn.execute( + "UPDATE edge_supports SET valid_to=? WHERE edge_id=? " + "AND valid_to IS NULL AND expired_at IS NULL", + (now_ts(), eid), + ) self.conn.execute( - "INSERT OR REPLACE INTO edges(id, workspace_id, repo_id, src, dst, relation, layer, " + "INSERT INTO edges(id, workspace_id, repo_id, src, dst, relation, layer, " "weight, valid_from, valid_to, ingested_at, expired_at, provenance) " - "VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?)", - (eid, edge.workspace_id, edge.repo_id, edge.src, edge.dst, edge.relation, layer, + "VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?) " + "ON CONFLICT(id) DO UPDATE SET workspace_id=excluded.workspace_id, " + "repo_id=excluded.repo_id, src=excluded.src, dst=excluded.dst, " + "relation=excluded.relation, layer=excluded.layer, weight=excluded.weight, " + "valid_from=excluded.valid_from, valid_to=excluded.valid_to, " + "ingested_at=excluded.ingested_at, expired_at=excluded.expired_at, " + "provenance=excluded.provenance", + (eid, edge.workspace_id, edge.repo_id, source, target, edge.relation, layer, edge.weight, edge.valid_from if edge.valid_from is not None else now_ts(), edge.valid_to, edge.ingested_at or now_ts(), edge.expired_at, - _dumps(edge.provenance)), + _dumps(incoming_provenance)), ) - self.conn.commit() + self._write_edge_supports( + eid, edge.relation, incoming_provenance, + valid_from=edge.valid_from, valid_to=edge.valid_to, + ingested_at=edge.ingested_at, expired_at=edge.expired_at, + ) + if commit: + self.conn.commit() return eid def invalidate_edge(self, edge_id: str, at: Optional[float] = None) -> None: + ts = now_ts() if at is None else at self.conn.execute("UPDATE edges SET valid_to=? WHERE id=? AND valid_to IS NULL", - (now_ts() if at is None else at, edge_id)) + (ts, edge_id)) + self.conn.execute( + "UPDATE edge_supports SET valid_to=? WHERE edge_id=? " + "AND valid_to IS NULL AND expired_at IS NULL", (ts, edge_id) + ) self.conn.commit() - def add_edge_support(self, edge_id: str, provenance: dict) -> None: + def _write_edge_supports(self, edge_id: str, relation: str, provenance: dict, + *, valid_from: Optional[float] = None, + valid_to: Optional[float] = None, + ingested_at: Optional[float] = None, + expired_at: Optional[float] = None) -> None: + source_kind = _edge_source_kind(provenance, relation) + confidence = _edge_support_confidence(provenance, source_kind) + support_provenance = _merge_edge_provenance([provenance]) + support_provenance["confidence"] = confidence + timestamp = now_ts() + support_valid_from = valid_from if valid_from is not None else timestamp + support_ingested_at = ingested_at if ingested_at is not None else timestamp + for memory_id in _provenance_memory_ids(provenance): + if valid_to is None and expired_at is None: + current = self.conn.execute( + "SELECT id, confidence, valid_from, ingested_at, provenance " + "FROM edge_supports WHERE edge_id=? AND memory_id=? AND source_kind=? " + "AND valid_to IS NULL AND expired_at IS NULL", + (edge_id, memory_id, source_kind), + ).fetchone() + if current is not None: + current_provenance = _loads(current["provenance"], {}) + merged_provenance = _merge_edge_provenance( + [current_provenance, support_provenance] + ) + desired_confidence = max( + float(current["confidence"] or 0.0), confidence + ) + merged_provenance["confidence"] = desired_confidence + desired_valid_from = min( + value for value in (current["valid_from"], support_valid_from) + if value is not None + ) + desired_ingested_at = min( + value for value in (current["ingested_at"], support_ingested_at) + if value is not None + ) + serialized = _dumps(merged_provenance) + if desired_confidence != float(current["confidence"] or 0.0) \ + or desired_valid_from != current["valid_from"] \ + or desired_ingested_at != current["ingested_at"] \ + or serialized != (current["provenance"] or "{}"): + self.conn.execute( + "UPDATE edge_supports SET confidence=?, valid_from=?, " + "ingested_at=?, provenance=? WHERE id=?", + (desired_confidence, desired_valid_from, + desired_ingested_at, serialized, current["id"]), + ) + continue + self.conn.execute( + "INSERT OR IGNORE INTO edge_supports " + "(edge_id, memory_id, source_kind, confidence, valid_from, valid_to, " + "ingested_at, expired_at, provenance) VALUES (?,?,?,?,?,?,?,?,?)", + (edge_id, memory_id, source_kind, confidence, + support_valid_from, valid_to, support_ingested_at, expired_at, + _dumps(support_provenance)), + ) + + def add_edge_support(self, edge_id: str, provenance: dict, *, + commit: bool = True) -> None: """Record another source memory supporting an existing graph edge.""" incoming = _provenance_memory_ids(provenance) if not incoming: @@ -850,15 +1420,22 @@ def add_edge_support(self, edge_id: str, provenance: dict) -> None: stored = _loads(row["provenance"], {}) if not isinstance(stored, dict): stored = {} - supports = _provenance_memory_ids(stored) - merged = supports + [mid for mid in incoming if mid not in supports] - if merged == supports: - return - stored["memory_id"] = merged[0] - stored["memory_ids"] = merged - self.conn.execute("UPDATE edges SET provenance=? WHERE id=?", - (_dumps(stored), edge_id)) - self.conn.commit() + merged_provenance = _merge_edge_provenance([stored, provenance]) + if _dumps(merged_provenance) != _dumps(stored): + self.conn.execute("UPDATE edges SET provenance=? WHERE id=?", + (_dumps(merged_provenance), edge_id)) + edge_row = self.conn.execute( + "SELECT relation, valid_from, valid_to, ingested_at, expired_at " + "FROM edges WHERE id=?", (edge_id,) + ).fetchone() + if edge_row: + self._write_edge_supports( + edge_id, edge_row["relation"] or "", provenance, + valid_from=edge_row["valid_from"], valid_to=edge_row["valid_to"], + ingested_at=edge_row["ingested_at"], expired_at=edge_row["expired_at"], + ) + if commit: + self.conn.commit() def invalidate_edges_for_memory(self, memory_id: str, *, at: Optional[float] = None, commit: bool = True) -> None: @@ -883,22 +1460,43 @@ def invalidate_edges_for_memory(self, memory_id: str, *, at: Optional[float] = N owner = self.conn.fetchall( "SELECT workspace_id FROM memories WHERE id=?", (memory_id,)) workspace_id = owner[0]["workspace_id"] if owner else None - sql = ("SELECT id, provenance FROM edges " - "WHERE valid_to IS NULL AND provenance LIKE ? ESCAPE '\\'") - params: list[Any] = [f"%{_escape_like(memory_id)}%"] + indexed_sql = ( + "SELECT DISTINCT e.id, e.provenance FROM edge_supports s " + "JOIN edges e ON e.id=s.edge_id WHERE s.memory_id=? " + "AND s.valid_to IS NULL AND s.expired_at IS NULL AND e.valid_to IS NULL" + ) + indexed_params: list[Any] = [memory_id] if workspace_id is not None: - # NULL workspace_id = a global/unscoped edge; it stays in scope so this keeps - # closing exactly the edges it closed before, minus other tenants' rows. - sql += " AND (workspace_id=? OR workspace_id IS NULL)" - params.append(workspace_id) - rows = self.conn.fetchall(sql, params) + indexed_sql += " AND (e.workspace_id=? OR e.workspace_id IS NULL)" + indexed_params.append(workspace_id) + rows = self.conn.fetchall(indexed_sql, indexed_params) + if not rows: + # Compatibility fallback for a direct legacy SQL writer. Canonical write + # paths populate edge_supports, so normal invalidation is indexed. + sql = ("SELECT id, provenance FROM edges " + "WHERE valid_to IS NULL AND provenance LIKE ? ESCAPE '\\'") + params: list[Any] = [f"%{_escape_like(memory_id)}%"] + if workspace_id is not None: + sql += " AND (workspace_id=? OR workspace_id IS NULL)" + params.append(workspace_id) + rows = self.conn.fetchall(sql, params) ids_to_close: list[str] = [] for row in rows: prov = _loads(row["provenance"], {}) supports = _provenance_memory_ids(prov) if memory_id not in supports: continue - remaining = [mid for mid in supports if mid != memory_id] + self.conn.execute( + "UPDATE edge_supports SET valid_to=? WHERE edge_id=? AND memory_id=? " + "AND valid_to IS NULL AND expired_at IS NULL", + (ts, row["id"], memory_id), + ) + normalized_remaining = [r["memory_id"] for r in self.conn.execute( + "SELECT DISTINCT memory_id FROM edge_supports WHERE edge_id=? " + "AND valid_to IS NULL AND expired_at IS NULL ORDER BY memory_id", + (row["id"],), + ).fetchall()] + remaining = normalized_remaining or [mid for mid in supports if mid != memory_id] if not remaining: ids_to_close.append(row["id"]) continue @@ -910,10 +1508,58 @@ def invalidate_edges_for_memory(self, memory_id: str, *, at: Optional[float] = N marks = ",".join("?" for _ in ids_to_close) self.conn.execute(f"UPDATE edges SET valid_to=? WHERE id IN ({marks})", (ts, *ids_to_close)) + self.conn.execute( + f"UPDATE edge_supports SET valid_to=? WHERE edge_id IN ({marks}) " + "AND valid_to IS NULL AND expired_at IS NULL", + (ts, *ids_to_close), + ) if commit: self.conn.commit() # ── memory-to-memory links (A-MEM style) ──────────────────────────────────── + def edge_supports_in_scope(self, edge_ids: Optional[list[str]] = None, *, + at: Optional[float] = None, + limit: Optional[int] = None) -> list[dict]: + """Return live normalized evidence rows for graph inspection/scene scoring.""" + t = at if at is not None else now_ts() + row_cap = None if limit is None else max(0, int(limit)) + if row_cap == 0: + return [] + sql = ( + "SELECT id, edge_id, memory_id, source_kind, confidence, valid_from, " + "valid_to, ingested_at, expired_at, provenance FROM edge_supports " + "WHERE (valid_from IS NULL OR valid_from<=?) " + "AND (valid_to IS NULL OR ?= row_cap: + break + chunk = edge_ids[start:start + IN_CLAUSE_CHUNK] + marks = ",".join("?" for _ in chunk) + statement = sql + f" AND edge_id IN ({marks}) ORDER BY edge_id, memory_id, id" + statement_params: tuple[Any, ...] = (*params, *chunk) + if row_cap is not None: + statement += " LIMIT ?" + statement_params = (*statement_params, row_cap - len(rows)) + found = self.conn.execute( + statement, statement_params, + ).fetchall() + rows.extend(dict(row) for row in found) + return rows + statement = sql + " ORDER BY edge_id, memory_id, id" + statement_params: tuple[Any, ...] = tuple(params) + if row_cap is not None: + statement += " LIMIT ?" + statement_params = (*statement_params, row_cap) + return [dict(row) for row in self.conn.execute( + statement, statement_params + ).fetchall()] + def add_link(self, a: str, b: str, relation: str = "related", layer: Optional[GraphLayer] = None, reason: str = "", *, commit: bool = True) -> None: diff --git a/engraphis/core/sync.py b/engraphis/core/sync.py index 249f9ba..0cde5de 100644 --- a/engraphis/core/sync.py +++ b/engraphis/core/sync.py @@ -42,6 +42,7 @@ import hashlib import json +import logging import math import re from typing import Any, Optional @@ -50,6 +51,9 @@ from engraphis.core.interfaces import MemoryRecord, MemoryType, Scope, SearchFilter from engraphis.core.store import Store, now_ts + +logger = logging.getLogger("engraphis.sync") + # ── bundle format ───────────────────────────────────────────────────────────── SYNC_FORMAT = "engraphis-sync" SYNC_VERSION = 1 @@ -446,16 +450,7 @@ def __init__(self, store: Store, *, embedder=None, vector_index=None, self.embedder = embedder self.index = vector_index self.device_id = device_id or store.device_id() - - # Default allowed_workspaces to global settings to prevent accidental bypass of ENGRAPHIS_WORKSPACES - if allowed_workspaces is None: - try: - from engraphis.config import settings - if settings.allowed_workspaces: - allowed_workspaces = settings.allowed_workspaces - except (ImportError, AttributeError): - pass - + # Same hard boundary MemoryService enforces (SECURITY.md §3): when set, a bundle # may only be applied into one of these workspaces, so the folder transport can # never be steered into writing a workspace the operator never authorized. @@ -779,7 +774,7 @@ def _write(self, rec: MemoryRecord, *, commit: bool = True) -> None: # ── one round-trip over a transport ───────────────────────────────────────── def sync(self, transport, workspace_id: str, *, repo_id: Optional[str] = None, - dry_run: bool = False) -> dict: + dry_run: bool = False, push: bool = True) -> dict: """Push this device's snapshot, then pull and apply every *other* device's. Full-state and idempotent, so it is safe to run on any cadence (cron, a @@ -789,7 +784,7 @@ def sync(self, transport, workspace_id: str, *, repo_id: Optional[str] = None, own_name = "bundle-%s.json" % self.device_id pushed = False - if not dry_run: + if not dry_run and push: transport.push(own_name, json.dumps(bundle).encode("utf-8")) pushed = True @@ -813,7 +808,9 @@ def sync(self, transport, workspace_id: str, *, repo_id: Optional[str] = None, except StopIteration: break except Exception as exc: # noqa: BLE001 — transport failure, not a bad bundle - applied.append({"bundle": "?", "error": "transport: %s" % exc}) + logger.warning("sync transport pull failed (%s)", type(exc).__name__) + applied.append({"bundle": "?", "error": "transport failure", + "error_type": type(exc).__name__}) # A generator that raised is closed and cannot be resumed; a list-backed # transport keeps going. Either way we stop here rather than abort the run. break @@ -830,7 +827,9 @@ def sync(self, transport, workspace_id: str, *, repo_id: Optional[str] = None, rep = self.apply_bundle(remote, into_workspace=ws_name, only_repo_id=repo_id, dry_run=dry_run) except Exception as exc: # one hostile bundle must never abort the whole sync - applied.append({"bundle": name, "error": str(exc)}) + logger.warning("sync bundle rejected (%s)", type(exc).__name__) + applied.append({"bundle": name, "error": "bundle rejected", + "error_type": type(exc).__name__}) continue rep["from_device"] = remote.get("device_id", "?") applied.append(rep) @@ -840,6 +839,7 @@ def sync(self, transport, workspace_id: str, *, repo_id: Optional[str] = None, errors = [a for a in applied if "error" in a] return {"pushed": own_name if pushed else None, "workspace": ws_name, "device_id": self.device_id, "exported_memories": len(bundle["memories"]), + "read_only": bool(not push and not dry_run), "peers_applied": len(applied) - len(errors), # Explicit: the round must NOT read as a success when bundles were dropped # (refused for signature/authorization, unreadable, or never delivered). diff --git a/engraphis/dashboard_app.py b/engraphis/dashboard_app.py index d90abbf..bcb2618 100644 --- a/engraphis/dashboard_app.py +++ b/engraphis/dashboard_app.py @@ -27,10 +27,10 @@ # Reachable without any session/token in every mode: the page shell, liveness, and # auth endpoints needed while logged out. First-admin setup is handled separately below: -# it is safe from loopback, or remotely when the deployment API token authenticates it. +# it is safe from loopback, or remotely when the deployment bootstrap token authenticates it. _PUBLIC = {"/", "/api/health", "/api/ready", "/api/auth/state", "/api/auth/login", "/api/auth/logout", "/api/auth/forgot", "/api/auth/reset", - "/webhooks/polar"} + "/api/auth/invitations/accept", "/webhooks/polar"} # A zero-user Team install must be able to inspect entitlement before its first admin # exists. Trial creation is deliberately NOT public: on a hosted instance the deployment @@ -38,6 +38,7 @@ # This route stops being public as soon as any user is created. _TEAM_BOOTSTRAP_PUBLIC = { "/api/license", + "/api/license/trials", } @@ -79,6 +80,13 @@ def _mcp_transport_security(mcp): def create_app() -> FastAPI: + from engraphis.observability import configure_structured_logging + configure_structured_logging() + from engraphis.commercial import service_mode + mode = service_mode() + if mode == "vendor": + from engraphis.vendor_app import create_app as create_vendor_app + return create_vendor_app() # MCP-over-HTTP agent connect: build the streamable-http ASGI app up front so we can # give the dashboard a lifespan that initializes its session manager (a mounted # sub-app's own lifespan does NOT run in Starlette - only the root app's does - @@ -117,7 +125,9 @@ def create_app() -> FastAPI: # A server-only install intentionally has no MCP SDK; that expected shape stays # silent. If an installed SDK fails to mount, retain a warning for operators. _level = _logging.INFO if importlib.util.find_spec("mcp") is None else _logging.WARNING - _logging.getLogger("engraphis").log(_level, "MCP /mcp mount skipped: %s", _exc) + _logging.getLogger("engraphis").log( + _level, "MCP /mcp mount skipped (%s)", type(_exc).__name__ + ) @_contextlib.asynccontextmanager async def _lifespan(app: FastAPI): @@ -159,6 +169,10 @@ async def _license_error(_request: Request, exc: licensing.LicenseError): settings.db_path, embed_model=settings.embed_model, embed_dim=settings.embed_dim or 384, allowed_workspaces=settings.allowed_workspaces) + # The customer-side sync relay uses this same service to enforce the folder boundary + # for scoped Team tokens. In particular, personal and unknown workspaces must never be + # addressable through the account-wide relay namespace merely by guessing a name. + app.state.service = svc try: import sys as _sys _ed = svc.engine.embedder @@ -175,6 +189,8 @@ async def _license_error(_request: Request, exc: licensing.LicenseError): # after the standalone Inspector was retired. Route lives in engraphis.billing so all # entrypoints share identical signature-verification + idempotency. try: + if not settings.vendor_service: + raise ImportError("billing webhook disabled in customer service mode") from engraphis.billing import router as billing_router app.include_router(billing_router) except Exception: # noqa: BLE001 - billing stays optional (e.g. minimal installs) @@ -185,7 +201,14 @@ async def _license_error(_request: Request, exc: licensing.LicenseError): # revocation and serve Pro sync. Endpoints live outside /api (license-key auth), # so the _auth_gate below (which only guards /api/*) leaves them alone. from engraphis.inspector.cloud_mount import mount_cloud_endpoints - mount_cloud_endpoints(app) + mount_cloud_endpoints( + app, include_license=settings.vendor_service, + include_sync=settings.customer_service) + # Pre-split keys have team.engraphis.com/license/v1/* signed into them. Customer mode + # keeps that public surface as a bounded 90-day proxy to license.engraphis.com; + # combined development mode continues serving the local license router above. + from engraphis.inspector.license_compat_proxy import mount_license_compat_proxy + mount_license_compat_proxy(app) # Team auth plumbing is mounted whenever team mode is configured. The request gate # activates for a live Team license or an already-provisioned user database, so a @@ -194,7 +217,7 @@ async def _license_error(_request: Request, exc: licensing.LicenseError): try: from engraphis.routes import v2_team team_enabled, auth_store = v2_team.attach(app, svc) - except Exception: # noqa: BLE001 - team stays optional on minimal installs + except Exception as exc: # noqa: BLE001 - team stays optional on minimal installs # Swallowing this silently was an auth downgrade, not a graceful degradation: # with team_enabled False and auth_store None, _auth_gate below skips the ENTIRE # role layer AND personal-folder isolation (set_current_user is never called), and @@ -207,19 +230,32 @@ async def _license_error(_request: Request, exc: licensing.LicenseError): # as a LOCAL of this function, so a module-level alias would be shadowed and # unbound on the (normal) path where that block never runs. import logging as _log - _log.getLogger("engraphis").exception( + _log.getLogger("engraphis").error( "team auth failed to mount — role enforcement and personal-folder isolation " - "are NOT active; /api/* will answer 503 until this is repaired") + "are NOT active; /api/* will answer 503 until this is repaired (%s)", + type(exc).__name__, + ) team_auth_broken = settings.team_mode # Streamable HTTP sessions are process-local in the MCP SDK. Bind each one to the # authenticated user that initialized it so another valid member cannot replay a # stolen session id with their own bearer token. _mcp_session_users: dict[str, str] = {} - def _bearer_ok(request: Request) -> bool: + def _api_bearer_ok(request: Request) -> bool: from engraphis.inspector.auth import bearer_ok return bearer_ok(request.headers.get("Authorization"), settings.api_token) + def _bootstrap_bearer_ok(request: Request) -> bool: + """Accept either bootstrap secret, but only for the explicit bootstrap paths. + + ``ENGRAPHIS_DEPLOYMENT_TOKEN`` proves ownership during hosted onboarding; it is + not a second unrestricted service-account token. + """ + from engraphis.inspector.auth import bearer_ok + deployment = _os.environ.get("ENGRAPHIS_DEPLOYMENT_TOKEN", "").strip() + return (_api_bearer_ok(request) + or bearer_ok(request.headers.get("Authorization"), deployment)) + def _bearer_token(request: Request) -> str: header = request.headers.get("Authorization") or "" return header[7:].strip() if header[:7].lower() == "bearer " else "" @@ -242,7 +278,8 @@ async def _auth_gate(request: Request, call_next): if request.method == "OPTIONS": return await call_next(request) team_bootstrap_public = ( - path in _TEAM_BOOTSTRAP_PUBLIC + (path in _TEAM_BOOTSTRAP_PUBLIC + or (request.method == "GET" and path.startswith("/api/license/trials/"))) and team_enabled and auth_store is not None and auth_store.count_users() == 0 @@ -253,7 +290,7 @@ async def _auth_gate(request: Request, call_next): and auth_store is not None and auth_store.count_users() == 0 and (is_local_request(request) - or bool(settings.api_token and _bearer_ok(request))) + or _bootstrap_bearer_ok(request)) ) # The OpenAPI schema publishes the full route map and therefore passes through # the same wall as every other /api path below. @@ -287,6 +324,8 @@ async def _auth_gate(request: Request, call_next): if mu is None: return JSONResponse({"error": "authentication required", "auth": "team"}, status_code=401) + if "agent" not in set(mu.get("token_scopes") or ()): + return JSONResponse({"error": "token lacks agent scope"}, status_code=403) if not app.state.mcp_over_http: return JSONResponse({"error": "MCP-over-HTTP is unavailable"}, status_code=404) @@ -333,7 +372,7 @@ async def _auth_gate(request: Request, call_next): # Service-account bearer token bypass — skips team auth entirely, # allowing CI/CD scripts and automation to use the same ENGRAPHIS_API_TOKEN # regardless of whether team mode is enabled. - if settings.api_token and _bearer_ok(request): + if settings.api_token and _api_bearer_ok(request): return await call_next(request) # A new, unlicensed instance with no users remains open for solo use. Once a paid # license (Pro or Team) activates the wall—or any users have been provisioned—the @@ -353,6 +392,8 @@ async def _auth_gate(request: Request, call_next): # workspace-scoped read/write. supplied = _bearer_token(request) user = auth_store.resolve_api_token(supplied) if supplied else None + if user is not None and "agent" not in set(user.get("token_scopes") or ()): + return JSONResponse({"error": "token lacks agent scope"}, status_code=403) if user is None: user = auth_store.resolve_session(request.cookies.get(_COOKIE, "")) if user is None: @@ -369,7 +410,7 @@ async def _auth_gate(request: Request, call_next): return await call_next(request) # Single-user modes: optional bearer token, exactly as before team mode existed. if settings.api_token: - if not _bearer_ok(request): + if not _api_bearer_ok(request): return JSONResponse({"error": "unauthorized"}, status_code=401) return await call_next(request) @@ -383,15 +424,17 @@ async def _auth_gate(request: Request, call_next): # caller-supplied DSN), /api/code/index and /api/workspaces/import-folder # (server-local file reads), /api/workspaces/delete. # - # Hosted first-admin setup requires ENGRAPHIS_API_TOKEN; otherwise any remote - # caller could win a deployment race and make themselves the first admin. + # Hosted first-admin setup requires ENGRAPHIS_DEPLOYMENT_TOKEN; otherwise any + # remote caller could win a deployment race and make themselves the first admin. + # The same ownership proof starts the deployment-bound trial from the public + # onboarding screen, but it never grants access to ordinary data routes. if not is_local_request(request): return JSONResponse( {"error": "this instance has no authentication configured, so remote API " "access is refused. For a hosted first boot, set " - "ENGRAPHIS_API_TOKEN (and ENGRAPHIS_LICENSE_KEY for first-admin " - "setup) in the deployment environment, restart, " - "then create the first admin account.", + "ENGRAPHIS_DEPLOYMENT_TOKEN and ENGRAPHIS_DASHBOARD_URL in the " + "deployment environment, then use the hosted setup screen to " + "activate a trial and create the first admin account.", "auth": "unconfigured"}, status_code=403) return await call_next(request) @@ -426,6 +469,7 @@ def index(): _maybe_start_autosync() _maybe_start_dreaming() _maybe_start_license_revalidation() + _maybe_start_email_outbox() return app @@ -433,6 +477,55 @@ def index(): _AUTOSYNC_STARTED = False _DREAMING_STARTED = False _REVALIDATE_STARTED = False +_EMAIL_OUTBOX_STARTED = False + + +def _process_due_email() -> dict: + """Run one customer-local outbox pass with the configured mail provider.""" + from engraphis import email_outbox + from engraphis.inspector.webhooks import _deliver_text_email + return email_outbox.process_due(_deliver_text_email, limit=20) + + +def _maybe_start_email_outbox() -> None: + """Retry locally configured invitation/reset email without blocking requests. + + Production Railway customers normally relay mail to the vendor control plane, whose + ASGI lifespan owns its worker. A self-hosted customer may configure Resend/SMTP + locally; immediate sends already persist failures in the durable outbox, so this loop + makes the bounded retry policy effective there too. + """ + global _EMAIL_OUTBOX_STARTED + if _EMAIL_OUTBOX_STARTED: + return + import sys + if "pytest" in sys.modules or _os.environ.get("PYTEST_CURRENT_TEST"): + return + if _os.environ.get("ENGRAPHIS_EMAIL_OUTBOX_LOOP", "1").strip().lower() in ( + "0", "false", "no", "off"): + return + from engraphis.inspector.webhooks import email_configured + if not email_configured(): + return + import logging + import threading + import time + + logger = logging.getLogger("engraphis.email_outbox") + + def _loop() -> None: + time.sleep(10) + while True: + try: + _process_due_email() + except Exception as exc: # noqa: BLE001 - retry loop must survive one bad iteration + logger.error( + "customer email outbox iteration failed (%s)", type(exc).__name__ + ) + time.sleep(30) + + threading.Thread(target=_loop, name="engraphis-email-outbox", daemon=True).start() + _EMAIL_OUTBOX_STARTED = True def _maybe_start_autosync() -> None: diff --git a/engraphis/email_outbox.py b/engraphis/email_outbox.py new file mode 100644 index 0000000..ea72d2b --- /dev/null +++ b/engraphis/email_outbox.py @@ -0,0 +1,443 @@ +"""Durable transactional-email outbox shared by all commercial workflows. + +Message bodies and recipients stay in the vendor database because they are required to +retry delivery. Operations endpoints return only redacted metadata. Provider payloads +are reduced to stable event identifiers and states; raw webhook bodies are never stored. +""" +from __future__ import annotations + +import hashlib +import math +import os +import secrets +import sqlite3 +import time +from typing import Callable, Optional + +from engraphis.inspector import license_registry + +MAX_ATTEMPTS = 5 +CLAIM_LEASE_SECONDS = 300 +MAX_IDEMPOTENCY_KEY_CHARS = 256 +MAX_KIND_CHARS = 48 +MAX_RECIPIENT_CHARS = 384 +MAX_SUBJECT_CHARS = 240 +MAX_TEXT_BODY_BYTES = 256 * 1024 + +_SCHEMA = """ +CREATE TABLE IF NOT EXISTS email_outbox ( + id TEXT PRIMARY KEY, + idempotency_key TEXT UNIQUE, + kind TEXT NOT NULL, + recipient TEXT NOT NULL, + subject TEXT NOT NULL, + text_body TEXT NOT NULL, + reply_to TEXT, + status TEXT NOT NULL DEFAULT 'pending', + attempts INTEGER NOT NULL DEFAULT 0, + max_attempts INTEGER NOT NULL DEFAULT 5, + next_attempt_at REAL NOT NULL, + provider TEXT, + provider_message_id TEXT, + last_error TEXT NOT NULL DEFAULT '', + created_at REAL NOT NULL, + updated_at REAL NOT NULL, + sent_at REAL +); +CREATE INDEX IF NOT EXISTS email_outbox_due_idx + ON email_outbox(status, next_attempt_at); +CREATE INDEX IF NOT EXISTS email_outbox_provider_idx + ON email_outbox(provider_message_id); +CREATE TABLE IF NOT EXISTS email_delivery_events ( + provider_event_id TEXT PRIMARY KEY, + provider_message_id TEXT NOT NULL, + event_type TEXT NOT NULL, + occurred_at REAL NOT NULL, + recorded_at REAL NOT NULL +); +""" + + +def _connect() -> sqlite3.Connection: + conn = license_registry.connect() + conn.executescript(_SCHEMA) + return conn + + +def _bounded_header(value: str, *, name: str, maximum: int, + required: bool = False) -> str: + """Validate a value that may later become an email or provider header.""" + if not isinstance(value, str): + raise ValueError("%s must be text" % name) + cleaned = value.strip() + if required and not cleaned: + raise ValueError("%s is required" % name) + if len(cleaned) > maximum: + raise ValueError("%s is too long" % name) + if any(ord(char) < 32 or ord(char) == 127 for char in cleaned): + raise ValueError("%s contains control characters" % name) + return cleaned + + +def enqueue(kind: str, recipient: str, subject: str, text_body: str, *, + reply_to: Optional[str] = None, idempotency_key: str = "", + max_attempts: int = MAX_ATTEMPTS) -> str: + """Persist a message and return its stable id. + + Supplying an idempotency key makes repeated webhook/request delivery return the + original message rather than enqueueing duplicates. + """ + clean_kind = _bounded_header( + kind or "transactional", name="kind", maximum=MAX_KIND_CHARS, required=True) + clean_recipient = _bounded_header( + recipient, name="recipient", maximum=MAX_RECIPIENT_CHARS, required=True).lower() + clean_subject = _bounded_header( + subject, name="subject", maximum=MAX_SUBJECT_CHARS, required=True) + clean_reply_to = _bounded_header( + reply_to or "", name="reply_to", maximum=MAX_RECIPIENT_CHARS) or None + clean_idem = _bounded_header( + idempotency_key or "", name="idempotency_key", + maximum=MAX_IDEMPOTENCY_KEY_CHARS) or None + if not isinstance(text_body, str): + raise ValueError("text_body must be text") + if not text_body or len(text_body.encode("utf-8")) > MAX_TEXT_BODY_BYTES: + raise ValueError("text_body is empty or too large") + try: + attempts_limit = max(1, min(10, int(max_attempts))) + except (OverflowError, TypeError, ValueError) as exc: + raise ValueError("max_attempts must be a finite integer") from exc + now = time.time() + msg_id = "eml_" + secrets.token_hex(12) + conn = _connect() + try: + if clean_idem: + row = conn.execute( + "SELECT id FROM email_outbox WHERE idempotency_key=?", + (clean_idem,)).fetchone() + if row: + return str(row["id"]) + try: + conn.execute( + "INSERT INTO email_outbox(id,idempotency_key,kind,recipient,subject," + "text_body,reply_to,max_attempts,next_attempt_at,created_at,updated_at) " + "VALUES (?,?,?,?,?,?,?,?,?,?,?)", + (msg_id, clean_idem, clean_kind, clean_recipient, clean_subject, + text_body, clean_reply_to, attempts_limit, now, now, now)) + conn.commit() + return msg_id + except sqlite3.IntegrityError: + if not clean_idem: + raise + row = conn.execute( + "SELECT id FROM email_outbox WHERE idempotency_key=?", + (clean_idem,)).fetchone() + if row: + return str(row["id"]) + raise + finally: + conn.close() + + +def _claim(message_id: str) -> Optional[dict]: + conn = _connect() + previous = conn.isolation_level + conn.isolation_level = None + try: + conn.execute("BEGIN IMMEDIATE") + now = time.time() + row = conn.execute( + "SELECT * FROM email_outbox WHERE id=?", (message_id,)).fetchone() + claimable = row is not None and ( + (row["status"] in ("pending", "retry") + and float(row["next_attempt_at"]) <= now) + or (row["status"] == "sending" and float(row["next_attempt_at"]) <= now) + ) + if not claimable: + conn.execute("COMMIT") + return None + if int(row["attempts"]) >= int(row["max_attempts"]): + if row["status"] == "sending": + conn.execute( + "UPDATE email_outbox SET status='failed',last_error=?,updated_at=? " + "WHERE id=? AND status='sending'", + ("DeliveryLeaseExpired", now, message_id)) + conn.execute("COMMIT") + return None + attempts = int(row["attempts"]) + 1 + conn.execute( + "UPDATE email_outbox SET status='sending',attempts=?,next_attempt_at=?," + "updated_at=? WHERE id=?", + (attempts, now + CLAIM_LEASE_SECONDS, now, message_id)) + conn.execute("COMMIT") + out = dict(row) + out["attempts"] = attempts + return out + except BaseException: + try: + conn.execute("ROLLBACK") + except sqlite3.Error: + pass + raise + finally: + conn.isolation_level = previous + conn.close() + + +def _provider_state(conn: sqlite3.Connection, provider_message_id: str) -> str: + """Return the strongest recorded terminal state for a provider message.""" + if not provider_message_id: + return "sent" + states = { + str(row[0]).strip().lower() + for row in conn.execute( + "SELECT event_type FROM email_delivery_events WHERE provider_message_id=?", + (provider_message_id,)).fetchall() + } + if states & {"email.complained", "complained"}: + return "complained" + if states & {"email.bounced", "bounced"}: + return "bounced" + if states & {"email.delivered", "delivered"}: + return "delivered" + return "sent" + + +def deliver_now(message_id: str, + deliverer: Callable[ + [str, str, str, Optional[str], str], tuple[str, str]]) -> bool: + """Attempt one claimed message and persist the outcome. + + ``deliverer`` receives recipient, subject, body, reply-to, and the stable outbox + message id (for provider-side idempotency), then returns ``(provider, + provider_message_id)``. Delivery exceptions are re-raised after the retry state is + safely persisted so existing callers keep their fail/rollback policy. + """ + message = _claim(message_id) + if message is None: + return False + try: + provider, provider_id = deliverer( + message["recipient"], message["subject"], message["text_body"], + message.get("reply_to"), message_id) + except Exception as exc: + attempts = int(message["attempts"]) + terminal = attempts >= int(message["max_attempts"]) + delay = min(3600, 60 * (2 ** max(0, attempts - 1))) + conn = _connect() + try: + conn.execute( + "UPDATE email_outbox SET status=?,next_attempt_at=?,last_error=?," + "updated_at=? WHERE id=? AND status='sending' AND attempts=?", + ("failed" if terminal else "retry", time.time() + delay, + type(exc).__name__[:80], time.time(), message_id, attempts)) + conn.commit() + finally: + conn.close() + raise + conn = _connect() + previous = conn.isolation_level + conn.isolation_level = None + try: + provider_name = (provider or "unknown")[:40] + provider_message_id = (provider_id or "")[:160] + conn.execute("BEGIN IMMEDIATE") + updated = conn.execute( + "UPDATE email_outbox SET status='sent',provider=?,provider_message_id=?," + "last_error='',sent_at=?,updated_at=? " + "WHERE id=? AND status='sending' AND attempts=?", + (provider_name, provider_message_id, time.time(), time.time(), + message_id, int(message["attempts"]))) + if updated.rowcount == 1: + state = _provider_state(conn, provider_message_id) + if state != "sent": + conn.execute( + "UPDATE email_outbox SET status=?,updated_at=? WHERE id=?", + (state, time.time(), message_id)) + conn.commit() + was_updated = updated.rowcount == 1 + except BaseException: + try: + conn.execute("ROLLBACK") + except sqlite3.Error: + pass + raise + finally: + conn.isolation_level = previous + conn.close() + return was_updated + + +def process_due(deliverer: Callable[ + [str, str, str, Optional[str], str], tuple[str, str]], + *, limit: int = 20) -> dict: + conn = _connect() + try: + rows = conn.execute( + "SELECT id FROM email_outbox WHERE status IN ('pending','retry','sending') " + "AND next_attempt_at<=? ORDER BY next_attempt_at LIMIT ?", + (time.time(), max(1, min(100, int(limit))))).fetchall() + finally: + conn.close() + sent = failed = 0 + for row in rows: + try: + sent += int(deliver_now(str(row["id"]), deliverer)) + except Exception: + failed += 1 + return {"processed": len(rows), "sent": sent, "failed": failed} + + +def requeue_failed(*, limit: int = 100) -> int: + """Make terminal failures explicitly retryable after an operator requests it.""" + conn = _connect() + previous = conn.isolation_level + conn.isolation_level = None + try: + conn.execute("BEGIN IMMEDIATE") + rows = conn.execute( + "SELECT id FROM email_outbox WHERE status='failed' " + "ORDER BY updated_at LIMIT ?", (max(1, min(500, int(limit))),)).fetchall() + now = time.time() + for row in rows: + conn.execute( + "UPDATE email_outbox SET status='retry',attempts=0,next_attempt_at=?," + "last_error='',updated_at=? WHERE id=? AND status='failed'", + (now, now, str(row["id"]))) + conn.execute("COMMIT") + return len(rows) + except BaseException: + try: + conn.execute("ROLLBACK") + except sqlite3.Error: + pass + raise + finally: + conn.isolation_level = previous + conn.close() + + +def record_provider_event(provider_event_id: str, provider_message_id: str, + event_type: str, *, occurred_at: Optional[float] = None) -> bool: + """Idempotently reduce a verified provider event to an outbox state transition.""" + event_id = (provider_event_id or "").strip() + message_id = (provider_message_id or "").strip() + normalized = (event_type or "").strip().lower() + if not event_id or len(event_id) > 255 \ + or not message_id or len(message_id) > 160 \ + or not normalized or len(normalized) > 80 \ + or any(ord(char) < 33 or ord(char) == 127 + for value in (event_id, message_id, normalized) for char in value): + return False + mapping = { + "email.delivered": "delivered", + "email.bounced": "bounced", + "email.complained": "complained", + "delivered": "delivered", + "bounced": "bounced", + "complained": "complained", + } + state = mapping.get(normalized) + conn = _connect() + try: + happened = time.time() if occurred_at is None else float(occurred_at) + if not math.isfinite(happened): + return False + inserted = conn.execute( + "INSERT OR IGNORE INTO email_delivery_events(provider_event_id," + "provider_message_id,event_type,occurred_at,recorded_at) VALUES (?,?,?,?,?)", + (event_id, message_id, normalized, happened, time.time())) + if inserted.rowcount == 0: + # Svix/Resend is at-least-once. The stable provider event ID is the + # idempotency boundary: a replay must never reinterpret a different body and + # mutate another message after the original event was already reduced. + conn.commit() + return True + if state: + protected = { + "delivered": ("bounced", "complained"), + "bounced": ("complained",), + "complained": (), + }[state] + placeholders = ",".join("?" for _value in protected) + suffix = (" AND status NOT IN (%s)" % placeholders) if protected else "" + conn.execute( + "UPDATE email_outbox SET status=?,updated_at=? " + "WHERE provider_message_id=?" + suffix, + (state, time.time(), message_id, *protected)) + conn.commit() + return True + finally: + conn.close() + + +def health() -> dict: + now = time.time() + window_start = now - 86400 + conn = _connect() + try: + row = conn.execute( + "SELECT COUNT(*) AS backlog, MIN(created_at) AS oldest FROM email_outbox " + "WHERE status IN ('pending','retry','sending')").fetchone() + terminal = int(conn.execute( + "SELECT COUNT(*) FROM email_outbox WHERE status='failed'").fetchone()[0]) + bounced = int(conn.execute( + "SELECT COUNT(*) FROM email_outbox WHERE status IN ('bounced','complained')" + ).fetchone()[0]) + sent_recent = int(conn.execute( + "SELECT COUNT(*) FROM email_outbox WHERE sent_at>=?", (window_start,) + ).fetchone()[0]) + bounced_recent = int(conn.execute( + "SELECT COUNT(DISTINCT o.id) FROM email_delivery_events e " + "JOIN email_outbox o ON o.provider_message_id=e.provider_message_id " + "WHERE e.occurred_at>=? AND o.sent_at>=? " + "AND e.event_type IN ('email.bounced','email.complained'," + "'bounced','complained')", (window_start, window_start)).fetchone()[0]) + finally: + conn.close() + backlog = int(row["backlog"] or 0) + age = max(0, int(time.time() - row["oldest"])) if row["oldest"] else 0 + config_valid = True + try: + maximum = int(os.environ.get( + "ENGRAPHIS_EMAIL_MAX_BACKLOG_AGE_SECONDS", "900")) + maximum_bounce_rate = float(os.environ.get( + "ENGRAPHIS_EMAIL_MAX_BOUNCE_RATE", "0.05")) + minimum_sample = int(os.environ.get( + "ENGRAPHIS_EMAIL_BOUNCE_MIN_SAMPLE", "20")) + if maximum < 60 or minimum_sample < 1 or not ( + math.isfinite(maximum_bounce_rate) + and 0.0 <= maximum_bounce_rate <= 1.0): + raise ValueError("out-of-range email health configuration") + except (OverflowError, ValueError): + maximum, maximum_bounce_rate, minimum_sample = 900, 0.05, 20 + config_valid = False + bounce_rate = bounced_recent / max(1, sent_recent) + bounce_ok = sent_recent < minimum_sample or bounce_rate <= maximum_bounce_rate + healthy = config_valid and (backlog == 0 or age <= maximum) \ + and terminal == 0 and bounce_ok + return {"healthy": healthy, "backlog": backlog, + "oldest_age_seconds": age, "failed": terminal, + "bounced_or_complained": bounced, "sent_24h": sent_recent, + "bounced_or_complained_24h": bounced_recent, + "bounce_rate_24h": round(bounce_rate, 4), + "configuration_valid": config_valid} + + +def recent_redacted(limit: int = 100) -> list[dict]: + """Admin view with no recipient, subject, body, or provider error payload.""" + conn = _connect() + try: + rows = conn.execute( + "SELECT id,kind,status,attempts,max_attempts,provider,provider_message_id," + "created_at,updated_at,sent_at FROM email_outbox ORDER BY created_at DESC LIMIT ?", + (max(1, min(500, int(limit))),)).fetchall() + finally: + conn.close() + out = [] + for row in rows: + item = dict(row) + provider_id = item.pop("provider_message_id") or "" + item["provider_message_fingerprint"] = ( + hashlib.sha256(provider_id.encode()).hexdigest()[:12] if provider_id else "") + out.append(item) + return out diff --git a/engraphis/engines/intelligence.py b/engraphis/engines/intelligence.py index 2cf6eb7..ba1fcce 100644 --- a/engraphis/engines/intelligence.py +++ b/engraphis/engines/intelligence.py @@ -93,13 +93,13 @@ def auto_categorize(content: str, title: str = "", if "reason" not in result: result["reason"] = "" return result - except Exception as e: - logger.debug("Auto-categorize skipped: %s", e) + except Exception as exc: + logger.debug("Auto-categorize skipped (%s)", type(exc).__name__) return { "memory_type": suggested_type, "confidence": 0.0, "should_split": False, - "reason": f"LLM unavailable: {e}", + "reason": "LLM unavailable", "splits": [], } @@ -151,11 +151,11 @@ def check_conflicts(content: str, namespace: str, if "explanation" not in result: result["explanation"] = "" return result - except Exception as e: - logger.debug("Conflict check skipped: %s", e) + except Exception as exc: + logger.debug("Conflict check skipped (%s)", type(exc).__name__) return {"has_conflict": False, "conflict_type": "none", "conflicting_memory_id": None, "resolution": "keep_both", - "explanation": f"LLM unavailable: {e}"} + "explanation": "LLM unavailable"} def _parse_json(raw: str) -> dict[str, Any]: diff --git a/engraphis/engines/thoughts.py b/engraphis/engines/thoughts.py index 463d651..459a834 100644 --- a/engraphis/engines/thoughts.py +++ b/engraphis/engines/thoughts.py @@ -45,9 +45,18 @@ def synthesize_thoughts( temperature=temperature, thought_prompt=thought_prompt, ) - except Exception as e: - logger.error("Thought synthesis failed: %s", e) - return {"thought": None, "source_count": len(chunks), "persisted": False, "error": str(e)} + except Exception as exc: + # Provider exceptions may embed request URLs, credentials, or excerpts. Keep + # both logs and the returned status content-free while retaining a useful class. + error_type = type(exc).__name__ + logger.error("Thought synthesis failed (%s)", error_type) + return { + "thought": None, + "source_count": len(chunks), + "persisted": False, + "error": "LLM synthesis failed", + "error_type": error_type, + } persisted_id = None if persist and thought: diff --git a/engraphis/http_security.py b/engraphis/http_security.py index 9649403..51c7462 100644 --- a/engraphis/http_security.py +++ b/engraphis/http_security.py @@ -16,14 +16,14 @@ ``127.0.0.1`` to HTTPS in the developer's browser for a year and break every other local project on that origin. The proxy case is handled by honouring the forwarded proto, which is what Railway presents. -* **CSP is tuned to the dashboard as it actually is**, not to an ideal: ``static/index.html`` - carries inline ``""" def _reserve_trial(mid: str, email: str, plan: str) -> Optional[str]: @@ -1000,7 +1318,8 @@ def _reserve_trial(mid: str, email: str, plan: str) -> Optional[str]: try: conn.execute("BEGIN IMMEDIATE") existing = conn.execute( - "SELECT 1 FROM trial_grants WHERE machine_id=?", (mid,)).fetchone() + "SELECT 1 FROM trial_grants WHERE machine_id=? OR lower(email)=?", + (mid, email)).fetchone() if existing: conn.execute("COMMIT") return None @@ -1050,6 +1369,23 @@ async def start_team_trial(request: Request): default 5/hour); 400 for a missing machine_id, an unknown plan, or a missing/ malformed email; 409 if this device already holds a trial grant; 502 if the verification email could not be sent.""" + # The v1 route remains executable only in development/combined mode so old tests and + # explicitly enabled customer migrations can finish. The production vendor service + # must use deployment-bound claims: the legacy flow eventually displayed a signed + # key in a browser and required a redeploy, which is not a GA-safe onboarding path. + from engraphis.commercial import service_mode + legacy_enabled = os.environ.get( + "ENGRAPHIS_ENABLE_LEGACY_TRIAL_FLOW", "").strip().lower() in ( + "1", "true", "yes", "on") + if service_mode() == "vendor" and not legacy_enabled: + return JSONResponse( + { + "error": "legacy trial issuance is disabled", + "replacement": "/license/v1/trial-claims", + }, + status_code=410, + headers={"Deprecation": "true"}, + ) try: body = await _bounded_json_object(request) except _JsonBodyError as exc: @@ -1097,7 +1433,9 @@ async def start_team_trial(request: Request): {"error": "the free trial has already been used on this device"}, status_code=409) - verify_url = "%s/license/v1/start-trial/verify?token=%s" % (public_base, token) + # Fragments never reach reverse-proxy, CDN, or application access logs. The static + # confirmation page clears it immediately and sends the token only in a bounded body. + verify_url = "%s/license/v1/start-trial/verify#token=%s" % (public_base, token) from engraphis.inspector.webhooks import send_trial_verification_email try: await asyncio.to_thread( @@ -1114,27 +1452,50 @@ async def start_team_trial(request: Request): "expires_in": _TRIAL_TOKEN_TTL_SECONDS} -#: Applied to EVERY /start-trial/verify response, not just the success page: the request -#: URL itself carries the one-time token, so even an error page must stay out of shared -#: caches and out of the Referer of anything the reader clicks from there. +_TRIAL_CONFIRM_SCRIPT = """(function(){ +"use strict"; +const status=document.getElementById("trial-status"); +const button=document.getElementById("trial-confirm"); +const token=new URLSearchParams(window.location.hash.slice(1)).get("token")||""; +window.history.replaceState(null,"",window.location.pathname); +if(!token){status.textContent="This confirmation link is missing its token.";return;} +button.disabled=false; +status.textContent="Nothing happens until you activate the trial."; +button.addEventListener("click",async function(){ +button.disabled=true;status.textContent="Activating your trial..."; +try{ +const response=await fetch(window.location.pathname,{method:"POST",headers:{ +"Accept":"text/html","Content-Type":"application/json"},body:JSON.stringify({token:token}), +credentials:"omit",cache:"no-store",redirect:"error"}); +const page=await response.text();document.open();document.write(page);document.close(); +}catch(_error){button.disabled=false;status.textContent="Activation failed. Please retry.";} +}); +})();""" +_TRIAL_CONFIRM_SCRIPT_HASH = base64.b64encode( + hashlib.sha256(_TRIAL_CONFIRM_SCRIPT.encode("utf-8")).digest()).decode("ascii") + + +#: Applied to every trial-confirmation response. Both compatibility and deployment-bound +#: links carry the secret in a URL fragment (never sent in HTTP/access logs), clear it +#: immediately in the browser, and submit it only in a bounded JSON body. A hash-pinned +#: inline script is the sole executable content; no caller value is interpolated here. _TRIAL_PAGE_HEADERS = { "Cache-Control": "no-store, no-cache, must-revalidate, private", "Pragma": "no-cache", "Referrer-Policy": "no-referrer", + "Content-Security-Policy": ( + "default-src 'none'; script-src 'sha256-%s'; connect-src 'self'; " + "style-src 'unsafe-inline'; base-uri 'none'; frame-ancestors 'none'; " + "form-action 'none'" % _TRIAL_CONFIRM_SCRIPT_HASH), } def _trial_verify_rate_limited(request: Request) -> Optional[HTMLResponse]: - """Shared burst gate for both /start-trial/verify handlers, or None when allowed. - - Both are unauthenticated and both hit SQLite before they can know whether the token - is even real: the GET runs connect + two executescripts + a PRAGMA + a SELECT, and - the POST additionally takes BEGIN IMMEDIATE — a RESERVED write lock on the same - relay.db that carries seat claims and sync bundles — BEFORE it discovers the token is - garbage. Flooding either one with junk tokens is therefore a bigger lever than the - single indexed SELECT behind /verify/{key_id}. Both are sync defs, so each request - also pins a threadpool worker. Answers HTML, not JSON: these routes are opened by a - human in a browser.""" + """Shared burst gate for both trial-verification POST routes, or None when allowed. + + Both are unauthenticated and take a write transaction on relay.db before a valid token + is known. GET pages are deliberately static and do not spend this budget or touch the + database. Answers HTML because these routes are opened by a human in a browser.""" if _register_rate_ok(_register_rate_key(request)): return None return HTMLResponse( @@ -1143,77 +1504,43 @@ def _trial_verify_rate_limited(request: Request) -> Optional[HTMLResponse]: @router.get("/start-trial/verify") -def confirm_team_trial(request: Request, token: str = ""): - """Render the confirmation page for a magic-link token — WITHOUT redeeming it. - - Redemption lives in the POST companion below, and this split is load-bearing rather - than cosmetic. Corporate mail gateways and antivirus link-prescanners (Outlook Safe - Links, Proofpoint URL Defense, and friends) routinely GET every URL they find in an - email body before the recipient ever sees it. When a bare GET redeemed the token, a - prescanner silently burned the one-time grant, and the human who then clicked the - link got "this link is invalid or has already been used" on a completely legitimate - first attempt — and it hit hardest at exactly the corporate mail estates most likely - to be buying Team. - - So: GET is safe and idempotent (it only reads), and the actual grant happens on a - POST that a human has to click a button to send. Prescanners do not submit forms. - The token still rides in the query string, which is what the form posts back to, so - no request body parsing — and therefore no multipart dependency — is involved.""" - limited = _trial_verify_rate_limited(request) - if limited is not None: - return limited - token = (token or "").strip() - if not token: - return HTMLResponse(_trial_verify_error_html("Missing token."), - status_code=400, headers=_TRIAL_PAGE_HEADERS) +def confirm_team_trial(): + """Render a scanner-safe confirmation page without receiving or redeeming a token. - conn = reg.connect() - try: - conn.executescript(_TRIAL_SCHEMA) - _ensure_trial_plan_column(conn) - conn.executescript(_TRIAL_PENDING_SCHEMA) - row = conn.execute( - "SELECT machine_id, email, plan, expires_at FROM trial_pending " - "WHERE token_hash=?", (_hash_token(token),)).fetchone() - finally: - conn.close() - - # Mirror the POST's diagnostics so a user learns the real problem before clicking, - # not after. Read-only: a lapsed row is left for the retention sweep in - # _reserve_trial, never deleted here (deleting on GET would hand a prescanner a way - # to destroy the row it cannot redeem). - if row is None: - return HTMLResponse( - _trial_verify_error_html("This link is invalid or has already been used."), - status_code=400, headers=_TRIAL_PAGE_HEADERS) - if row["expires_at"] < time.time(): - return HTMLResponse( - _trial_verify_error_html( - "This link has expired — request a new trial from the dashboard."), - status_code=400, headers=_TRIAL_PAGE_HEADERS) - - return HTMLResponse(_trial_confirm_html(token, row["plan"], row["email"]), - headers=_TRIAL_PAGE_HEADERS) + The secret remains in the URL fragment, which HTTP does not transmit. Corporate mail + scanners can GET this page repeatedly without seeing or consuming the grant.""" + return HTMLResponse(_trial_confirm_html(), headers=_TRIAL_PAGE_HEADERS) @router.post("/start-trial/verify") -def verify_team_trial(request: Request, token: str = ""): +async def verify_team_trial(request: Request): """Redeem a magic-link token from :func:`start_team_trial` — mints and displays the real signed trial key. Answers a small HTML page, not JSON: this is meant to be reached by a human clicking the confirm button on the GET page above, who needs to read and copy a key, not parse a response body. One-time: the token is deleted on first use (success OR a stale/losing race), so replaying it never mints twice. - Takes the token from the QUERY STRING, not a form body: the GET page posts back to - its own URL, so there is nothing to parse and no python-multipart dependency (which - has broken the [server] extra before).""" - limited = _trial_verify_rate_limited(request) + The browser submits a bounded JSON body after clearing the URL fragment, so access + logs and Referer headers never carry the one-time credential.""" + limited = await asyncio.to_thread(_trial_verify_rate_limited, request) if limited is not None: return limited - token = (token or "").strip() - if not token: - return HTMLResponse(_trial_verify_error_html("Missing token."), + try: + body = await _bounded_json_object(request) + except _JsonBodyError: + return HTMLResponse(_trial_verify_error_html("Invalid confirmation request."), status_code=400, headers=_TRIAL_PAGE_HEADERS) + raw_token = body.get("token") + token = raw_token.strip() if isinstance(raw_token, str) else "" + if not token or len(token) > 512 \ + or any(ord(char) < 33 or ord(char) == 127 for char in token): + return HTMLResponse(_trial_verify_error_html("Invalid confirmation link."), + status_code=400, headers=_TRIAL_PAGE_HEADERS) + return await asyncio.to_thread(_verify_team_trial_token, token) + + +def _verify_team_trial_token(token: str): + """Consume one validated legacy token and return its locked-down HTML response.""" conn = reg.connect() try: @@ -1244,7 +1571,8 @@ def verify_team_trial(request: Request, token: str = ""): status_code=400, headers=_TRIAL_PAGE_HEADERS) mid, email, plan = row["machine_id"], row["email"], row["plan"] existing = conn.execute( - "SELECT 1 FROM trial_grants WHERE machine_id=?", (mid,)).fetchone() + "SELECT 1 FROM trial_grants WHERE machine_id=? OR lower(email)=?", + (mid, email)).fetchone() if existing: conn.execute( "DELETE FROM trial_pending WHERE token_hash=?", (token_hash,)) @@ -1279,9 +1607,339 @@ def verify_team_trial(request: Request, token: str = ""): reg.record_issued(key) except Exception: pass - # This body contains the full signed license key and the URL still carries the - # one-time token, so keep both out of shared caches and out of Referer headers on - # any link the reader clicks from here. Uses the shared constant rather than an - # inline copy so the success page and the error pages can never drift apart. + # This body contains the full signed license key. Keep it out of shared caches and + # Referer headers using the same headers as every confirmation/error response. return HTMLResponse(_trial_verify_success_html(key, plan, TRIAL_DAYS), headers=_TRIAL_PAGE_HEADERS) + + +# ── v1.0 deployment-bound trial claims ─────────────────────────────────────── + +def _deployment_hash(token: str) -> str: + return hashlib.sha256(token.encode("utf-8")).hexdigest() + + +def _valid_trial_claim_id(value: str) -> bool: + return (value.startswith("clm_") and 8 <= len(value) <= 64 + and all(char.isascii() and (char.isalnum() or char in "_-") for char in value)) + + +def _claim_confirmation_html() -> str: + """Generic scanner-safe page; the fragment token never enters server-rendered HTML.""" + return f""" + + +Confirm Engraphis trial +

Confirm your Engraphis trial

+

The signed license will be delivered directly to your deployment. It will not be +shown in this browser or sent by email.

+ +

Preparing this one-time confirmation link...

+

If you did not request this trial, close this page.

+""" + + +def _claim_success_html(plan: str) -> str: + import html as _html + return f""" +Trial confirmed +

{_html.escape(plan.title())} trial confirmed

+

Return to your Engraphis deployment. It will retrieve and store the license +automatically; no key copying or redeploy is required.

""" + + +def _reserve_trial_claim(machine_id: str, email: str, plan: str, + deployment_token: str, dashboard_url: str + ) -> tuple[str, Optional[str], str]: + """Return ``(claim_id, confirmation_token, state)`` under one write lock.""" + conn = reg.connect() + conn.executescript(_TRIAL_SCHEMA) + _ensure_trial_plan_column(conn) + conn.executescript(_TRIAL_CLAIM_SCHEMA) + _ensure_trial_claim_columns(conn) + now = time.time() + deployment_hash = _deployment_hash(deployment_token) + confirmation = secrets.token_urlsafe(32) + claim_id = "clm_" + secrets.token_urlsafe(18) + window = time.strftime("%Y-%m-%d", time.gmtime(now)) + previous = conn.isolation_level + conn.isolation_level = None + try: + conn.execute("BEGIN IMMEDIATE") + # Expired, unconfirmed reservations must not permanently squat on an email, + # machine, or deployment token. Remove every stale collision before deciding + # whether the new tuple is already used; confirmed claims remain permanent. + conn.execute( + "DELETE FROM trial_claims WHERE confirmed_at IS NULL AND expires_at= now: + conn.execute("COMMIT") + return str(existing["claim_id"]), None, "pending" + claim_id = str(existing["claim_id"]) + prior_grant = conn.execute( + "SELECT 1 FROM trial_grants WHERE machine_id=? OR lower(email)=? LIMIT 1", + (machine_id, email)).fetchone() + if prior_grant: + conn.execute("COMMIT") + return "", None, "used" + + rate = conn.execute( + "SELECT count FROM trial_email_attempts WHERE email=? AND window=?", + (email, window)).fetchone() + if rate and int(rate["count"]) >= 3: + conn.execute("COMMIT") + return claim_id, None, "rate_limited" + conn.execute( + "INSERT INTO trial_email_attempts(email,window,count) VALUES (?,?,1) " + "ON CONFLICT(email,window) DO UPDATE SET count=count+1", (email, window)) + conn.execute("DELETE FROM trial_email_attempts WHERE window 128 or not deployment_token + or len(deployment_token) < 24 or len(deployment_token) > 512): + return JSONResponse({"error": "valid deployment credentials are required"}, + status_code=400) + if not _EMAIL_RE.match(email) or plan not in ("pro", "team"): + return JSONResponse({"error": "valid email and trial plan are required"}, + status_code=400) + try: + dashboard_url = cloud_license.validate_cloud_base_url(dashboard_url) + except ValueError: + return JSONResponse({"error": "valid dashboard URL is required"}, status_code=400) + public_base = _relay_public_base() + if not public_base: + return JSONResponse({"error": "trial signup is not configured"}, status_code=503) + if not await asyncio.to_thread(_bump_trial_rate, _client_ip(request)): + return JSONResponse({"error": "too many trial requests"}, status_code=429) + claim_id, confirmation, state = await asyncio.to_thread( + _reserve_trial_claim, machine_id, email, plan, deployment_token, dashboard_url) + if state == "used": + return JSONResponse({"error": "the free trial has already been used"}, + status_code=409) + if state == "rate_limited": + return JSONResponse({"error": "too many trial emails"}, status_code=429) + if state in ("pending", "confirmed"): + return {"claim_id": claim_id, "status": state, "pending": state == "pending"} + + # URL fragments are handled entirely by the browser and never reach reverse-proxy, + # CDN, or application access logs. The confirmation page clears the fragment before + # submitting this one-time token in a bounded JSON body. + verify_url = "%s/license/v1/trial-claims/verify#token=%s" % ( + public_base, confirmation) + from engraphis.inspector.webhooks import send_trial_claim_email + delivery = "sent" + try: + await asyncio.to_thread( + send_trial_claim_email, email, verify_url, plan, + minutes=_TRIAL_TOKEN_TTL_SECONDS // 60, + idempotency_key="trial-claim:%s:%s" % (claim_id, _hash_token(confirmation))) + except Exception as exc: # durable outbox retains the retry + logger.error("trial claim confirmation queued after delivery failure (%s)", + type(exc).__name__) + delivery = "retry" + conn = reg.connect() + try: + conn.executescript(_TRIAL_CLAIM_SCHEMA) + conn.execute("UPDATE trial_claims SET delivery_state=? WHERE claim_id=?", + (delivery, claim_id)) + conn.commit() + finally: + conn.close() + return {"claim_id": claim_id, "status": "pending", "pending": True, + "delivery_state": delivery, "expires_in": _TRIAL_TOKEN_TTL_SECONDS} + + +@router.get("/trial-claims/verify") +def confirm_trial_claim(): + # The token lives in the URL fragment, which HTTP never transmits. GET is therefore + # completely static, read-only, and safe for corporate link scanners. The hash-pinned + # script clears the fragment and waits for a deliberate human click. + return HTMLResponse(_claim_confirmation_html(), headers=_TRIAL_PAGE_HEADERS) + + +@router.post("/trial-claims/verify") +async def verify_trial_claim(request: Request): + limited = await asyncio.to_thread(_trial_verify_rate_limited, request) + if limited is not None: + return limited + try: + body = await _bounded_json_object(request) + except _JsonBodyError: + return HTMLResponse(_trial_verify_error_html("Invalid confirmation request."), + status_code=400, headers=_TRIAL_PAGE_HEADERS) + raw_token = body.get("token") + token = raw_token.strip() if isinstance(raw_token, str) else "" + if not token or len(token) > 512 \ + or any(ord(char) < 33 or ord(char) == 127 for char in token): + return HTMLResponse(_trial_verify_error_html("Invalid confirmation link."), + status_code=400, headers=_TRIAL_PAGE_HEADERS) + return await asyncio.to_thread(_verify_trial_claim_token, token) + + +def _verify_trial_claim_token(token: str): + conn = reg.connect() + conn.executescript(_TRIAL_CLAIM_SCHEMA) + previous = conn.isolation_level + conn.isolation_level = None + try: + conn.execute("BEGIN IMMEDIATE") + row = conn.execute( + "SELECT * FROM trial_claims WHERE confirmation_hash=?", + (_hash_token(token),)).fetchone() if token else None + if row is None: + conn.execute("COMMIT") + return HTMLResponse(_trial_verify_error_html("Invalid confirmation link."), + status_code=400, headers=_TRIAL_PAGE_HEADERS) + if row["confirmed_at"] is not None: + conn.execute("COMMIT") + return HTMLResponse(_claim_success_html(row["plan"]), + headers=_TRIAL_PAGE_HEADERS) + now = time.time() + if float(row["expires_at"]) < now: + conn.execute("COMMIT") + return HTMLResponse(_trial_verify_error_html( + "This confirmation link has expired."), status_code=400, + headers=_TRIAL_PAGE_HEADERS) + from engraphis.inspector.webhooks import issue_key + from engraphis.licensing import TRIAL_DAYS + seats = TEAM_TRIAL_SEATS if row["plan"] == "team" else 1 + key = issue_key( + row["email"], product_name=row["plan"], seats=seats, + days=TRIAL_DAYS, trial=True, record=False) + conn.execute( + "UPDATE trial_claims SET confirmed_at=?,license_key=?,delivery_state='confirmed' " + "WHERE claim_id=? AND confirmed_at IS NULL", (now, key, row["claim_id"])) + conn.execute( + "INSERT OR IGNORE INTO trial_grants(machine_id,email,plan,issued_at) " + "VALUES (?,?,?,?)", (row["machine_id"], row["email"], row["plan"], now)) + conn.execute("COMMIT") + return HTMLResponse(_claim_success_html(row["plan"]), + headers=_TRIAL_PAGE_HEADERS) + except BaseException: + try: + conn.execute("ROLLBACK") + except Exception: + pass + raise + finally: + conn.isolation_level = previous + conn.close() + + +@router.get("/trial-claims/{claim_id}") +def trial_claim_status(claim_id: str): + if not _valid_trial_claim_id(claim_id): + return JSONResponse({"error": "claim not found"}, status_code=404) + conn = reg.connect() + try: + conn.executescript(_TRIAL_CLAIM_SCHEMA) + row = conn.execute( + "SELECT plan,expires_at,confirmed_at,claimed_at,delivery_state " + "FROM trial_claims WHERE claim_id=?", (claim_id,)).fetchone() + finally: + conn.close() + if row is None: + return JSONResponse({"error": "claim not found"}, status_code=404) + status = ("active" if row["claimed_at"] is not None else + "confirmed" if row["confirmed_at"] is not None else + "expired" if float(row["expires_at"]) < time.time() else "pending") + return {"claim_id": claim_id, "plan": row["plan"], "status": status, + "confirmed": row["confirmed_at"] is not None, + "active": row["claimed_at"] is not None, + "delivery_state": row["delivery_state"]} + + +@router.post("/trial-claims/{claim_id}/claim") +async def claim_trial_license(claim_id: str, request: Request): + """Return key material only to the deployment that initiated this claim.""" + try: + body = await _bounded_json_object(request) + except _JsonBodyError as exc: + return _json_error(exc) + deployment_token = str(body.get("deployment_token") or "").strip() + machine_id = str(body.get("machine_id") or "").strip() + if not _valid_trial_claim_id(claim_id) \ + or not 24 <= len(deployment_token) <= 512 \ + or not 1 <= len(machine_id) <= 128 \ + or any(ord(char) < 32 or ord(char) == 127 for char in machine_id): + return JSONResponse({"error": "deployment credentials required"}, status_code=401) + conn = reg.connect() + try: + conn.executescript(_TRIAL_CLAIM_SCHEMA) + row = conn.execute( + "SELECT * FROM trial_claims WHERE claim_id=? AND deployment_hash=? " + "AND machine_id=?", (claim_id, _deployment_hash(deployment_token), + machine_id)).fetchone() + finally: + conn.close() + if row is None: + return JSONResponse({"error": "claim not found"}, status_code=404) + if row["confirmed_at"] is None or not row["license_key"]: + status = "expired" if float(row["expires_at"]) < time.time() else "pending" + return {"claim_id": claim_id, "status": status, "ready": False} + try: + await asyncio.to_thread(reg.record_issued, row["license_key"]) + except Exception: + return JSONResponse({"status": "recovery_pending", "ready": False}, + status_code=503) + conn = reg.connect() + try: + conn.execute("UPDATE trial_claims SET claimed_at=?,delivery_state='claimed' " + "WHERE claim_id=?", (time.time(), claim_id)) + conn.commit() + finally: + conn.close() + return {"claim_id": claim_id, "status": "ready", "ready": True, + "key": row["license_key"]} diff --git a/engraphis/inspector/license_compat_proxy.py b/engraphis/inspector/license_compat_proxy.py new file mode 100644 index 0000000..f78a548 --- /dev/null +++ b/engraphis/inspector/license_compat_proxy.py @@ -0,0 +1,140 @@ +"""Bounded v1.0 compatibility proxy for retired customer-host license routes. + +Keys issued before the control-plane split contain ``team.engraphis.com`` as their signed +server URL. Customer mode must therefore keep ``/license/v1/*`` reachable for the announced +90-day migration window even though issuance and leases now live on +``license.engraphis.com``. The proxy forwards only protocol headers required by license +clients: never cookies, customer API tokens, forwarded-client identity, or vendor secrets. +""" +from __future__ import annotations + +from urllib.parse import quote + +import httpx +from fastapi import APIRouter, FastAPI, Request +from fastapi.responses import JSONResponse, Response + +from engraphis import __version__ +from engraphis.config import resolve_license_server_url, settings + + +MAX_REQUEST_BYTES = 1024 * 1024 +MAX_RESPONSE_BYTES = 2 * 1024 * 1024 +COMPAT_SUNSET = "Sat, 17 Oct 2026 00:00:00 GMT" +_METHODS = ["GET", "HEAD", "POST", "PUT", "PATCH", "DELETE", "OPTIONS"] +_REQUEST_HEADERS = frozenset({"accept", "authorization", "content-type"}) +_RESPONSE_HEADERS = frozenset({ + "cache-control", "content-disposition", "content-type", "etag", "location", + "referrer-policy", "retry-after", "www-authenticate", +}) + +router = APIRouter(prefix="/license/v1", include_in_schema=False) + + +def _deprecation_headers() -> dict: + return { + "Deprecation": "true", + "Sunset": COMPAT_SUNSET, + "Link": '; rel="successor-version"', + } + + +async def _bounded_body(request: Request): + try: + declared = int(request.headers.get("content-length") or 0) + except ValueError: + return None, JSONResponse( + {"error": "invalid content length"}, status_code=400, + headers=_deprecation_headers()) + if declared < 0: + return None, JSONResponse( + {"error": "invalid content length"}, status_code=400, + headers=_deprecation_headers()) + if declared > MAX_REQUEST_BYTES: + return None, JSONResponse( + {"error": "license compatibility request is too large"}, status_code=413, + headers=_deprecation_headers()) + body = bytearray() + async for chunk in request.stream(): + if len(body) + len(chunk) > MAX_REQUEST_BYTES: + return None, JSONResponse( + {"error": "license compatibility request is too large"}, status_code=413, + headers=_deprecation_headers()) + body.extend(chunk) + return bytes(body), None + + +async def _send_upstream(method: str, url: str, headers: dict, body: bytes) -> httpx.Response: + timeout = httpx.Timeout(15.0, connect=5.0) + async with httpx.AsyncClient(timeout=timeout, follow_redirects=False) as client: + return await client.request(method, url, headers=headers, content=body) + + +def _target_url(compat_path: str, query: str) -> str: + base = resolve_license_server_url().rstrip("/") + path = quote(compat_path or "", safe="/-._~") + target = base + "/license/v1" + (("/" + path) if path else "") + if query: + target += "?" + query + return target + + +async def _proxy(request: Request, compat_path: str = ""): + if settings.service_mode != "customer": + return JSONResponse({"error": "compatibility proxy is unavailable"}, status_code=404) + if any(segment in (".", "..") for segment in compat_path.split("/")): + return JSONResponse( + {"error": "invalid license compatibility path"}, status_code=400, + headers=_deprecation_headers()) + body, error = await _bounded_body(request) + if error is not None: + return error + headers = { + name: value for name, value in request.headers.items() + if name.lower() in _REQUEST_HEADERS + } + headers["user-agent"] = "Engraphis/%s license-compat-v1" % __version__ + target = _target_url(compat_path, request.url.query) + try: + upstream = await _send_upstream(request.method, target, headers, body or b"") + except (httpx.TimeoutException, httpx.NetworkError): + return JSONResponse( + {"error": "license service is temporarily unavailable"}, status_code=503, + headers=_deprecation_headers()) + except httpx.HTTPError: + return JSONResponse( + {"error": "license compatibility request failed"}, status_code=502, + headers=_deprecation_headers()) + if len(upstream.content) > MAX_RESPONSE_BYTES: + return JSONResponse( + {"error": "license service response exceeded the compatibility limit"}, + status_code=502, headers=_deprecation_headers()) + response_headers = _deprecation_headers() + response_headers.update({ + name: value for name, value in upstream.headers.items() + if name.lower() in _RESPONSE_HEADERS + }) + return Response( + content=b"" if request.method == "HEAD" else upstream.content, + status_code=upstream.status_code, + headers=response_headers, + ) + + +@router.api_route("", methods=_METHODS) +async def proxy_license_root(request: Request): + return await _proxy(request) + + +@router.api_route("/{compat_path:path}", methods=_METHODS) +async def proxy_license_path(compat_path: str, request: Request): + return await _proxy(request, compat_path) + + +def mount_license_compat_proxy(app: FastAPI) -> bool: + """Mount only on the isolated customer service; combined keeps local v1 routes.""" + if settings.service_mode != "customer": + return False + app.include_router(router) + app.state._license_compat_proxy_mounted = True + return True diff --git a/engraphis/inspector/license_registry.py b/engraphis/inspector/license_registry.py index ffaf363..45cbeb3 100644 --- a/engraphis/inspector/license_registry.py +++ b/engraphis/inspector/license_registry.py @@ -19,6 +19,7 @@ from __future__ import annotations import hashlib +import math import os import sqlite3 import time @@ -57,10 +58,27 @@ def _state_dir() -> Path: expires REAL, subscription_id TEXT, order_id TEXT, + signing_key_id TEXT, status TEXT NOT NULL DEFAULT 'active', -- 'active' | 'revoked' created_at REAL NOT NULL, revoked_at REAL ); +CREATE TABLE IF NOT EXISTS signer_rotation_reissues ( + source_key_id TEXT NOT NULL, + replacement_key_id TEXT NOT NULL UNIQUE, + source_signing_key_id TEXT NOT NULL, + replacement_signing_key_id TEXT NOT NULL, + created_at REAL NOT NULL, + PRIMARY KEY (source_key_id, replacement_signing_key_id) +); +CREATE TABLE IF NOT EXISTS control_plane_events ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + kind TEXT NOT NULL, + occurred_at REAL NOT NULL +); +CREATE INDEX IF NOT EXISTS idx_control_plane_events_kind_time + ON control_plane_events(kind, occurred_at); + """ @@ -72,7 +90,18 @@ def connect(db_path: Optional[str] = None) -> sqlite3.Connection: """Open the relay DB (creating parent dir + schema). Callers close it.""" path = _db_path(db_path) if path != ":memory:": - Path(path).expanduser().parent.mkdir(parents=True, exist_ok=True) + database = Path(path).expanduser() + database.parent.mkdir(parents=True, exist_ok=True, mode=0o700) + # SQLite otherwise creates the PII-bearing registry using the process umask, + # which can yield 0644 on ordinary hosts. Pre-create it owner-only before SQLite + # opens it, and tighten an existing file after upgrades as defense in depth. + descriptor = os.open(str(database), os.O_RDWR | os.O_CREAT, 0o600) + os.close(descriptor) + try: + os.chmod(database, 0o600) + except OSError: + pass + path = str(database) conn = sqlite3.connect(path) conn.row_factory = sqlite3.Row # Wait (up to 5s) for a competing writer's lock rather than failing with @@ -103,15 +132,69 @@ def connect(db_path: Optional[str] = None) -> sqlite3.Connection: conn.execute("ALTER TABLE issued_licenses ADD COLUMN subscription_id TEXT") if "order_id" not in columns: conn.execute("ALTER TABLE issued_licenses ADD COLUMN order_id TEXT") + if "signing_key_id" not in columns: + # Pre-v1.0 rows cannot be backfilled from a public-key fingerprint because the + # registry deliberately stores no raw license material. They remain NULL and are + # reported as ``unknown`` by inventory(), which forces a conservative reissue. + conn.execute("ALTER TABLE issued_licenses ADD COLUMN signing_key_id TEXT") conn.execute( "CREATE INDEX IF NOT EXISTS idx_issued_subscription " "ON issued_licenses(subscription_id)") conn.execute( "CREATE INDEX IF NOT EXISTS idx_issued_order " "ON issued_licenses(order_id)") + conn.execute( + "CREATE INDEX IF NOT EXISTS idx_issued_signer " + "ON issued_licenses(signing_key_id)") return conn +def inventory(db_path: Optional[str] = None) -> dict: + """Return PII-free counts for the pre-rotation issued-key audit.""" + conn = connect(db_path) + try: + statuses = { + str(row["status"]): int(row["n"]) + for row in conn.execute( + "SELECT status, COUNT(*) AS n FROM issued_licenses GROUP BY status") + } + plans = { + str(row["plan"] or "unknown"): int(row["n"]) + for row in conn.execute( + "SELECT plan, COUNT(*) AS n FROM issued_licenses GROUP BY plan") + } + active_key_ids = [ + str(row["key_id"]) + for row in conn.execute( + "SELECT key_id FROM issued_licenses WHERE status='active' ORDER BY key_id") + ] + registrations = int(conn.execute( + "SELECT COUNT(*) AS n FROM registrations").fetchone()["n"]) + signing_key_ids = { + str(row["signing_key_id"] or "unknown"): int(row["n"]) + for row in conn.execute( + "SELECT signing_key_id, COUNT(*) AS n FROM issued_licenses " + "GROUP BY signing_key_id") + } + rotation_reissues = int(conn.execute( + "SELECT COUNT(*) AS n FROM signer_rotation_reissues").fetchone()["n"]) + finally: + conn.close() + total = sum(statuses.values()) + return { + "issued_total": total, + "active": statuses.get("active", 0), + "revoked": statuses.get("revoked", 0), + "other_status": total - statuses.get("active", 0) - statuses.get("revoked", 0), + "plans": plans, + "active_key_ids": active_key_ids, + "signing_key_ids": signing_key_ids, + "registered_machines": registrations, + "rotation_reissues": rotation_reissues, + "rotation_requires_migration": statuses.get("active", 0) > 0, + } + + def account_id_for(lic: License) -> str: """Stable, non-PII namespace for a customer's bundles. @@ -122,6 +205,24 @@ def account_id_for(lic: License) -> str: return hashlib.sha256(basis.encode("utf-8")).hexdigest()[:16] +def _record_issued_on_connection(conn: sqlite3.Connection, lic: License) -> None: + """Insert or refresh one verified license without changing a revocation tombstone.""" + conn.execute( + "INSERT INTO issued_licenses " + " (key_id, email, plan, seats, issued, expires, subscription_id, order_id, " + " signing_key_id, status, created_at) " + "VALUES (?,?,?,?,?,?,?,?,?, 'active', ?) " + "ON CONFLICT(key_id) DO UPDATE SET " + " email=excluded.email, plan=excluded.plan, seats=excluded.seats, " + " issued=excluded.issued, expires=excluded.expires, " + " subscription_id=excluded.subscription_id, order_id=excluded.order_id, " + " signing_key_id=excluded.signing_key_id", + (lic.key_id, lic.email, lic.plan, lic.seats, lic.issued, lic.expires, + lic.subscription_id or None, lic.order_id or None, + lic.signing_key_id or None, time.time()), + ) + + def record_issued(key: str, *, db_path: Optional[str] = None) -> str: """Record a freshly issued key in the registry (idempotent). Returns its key_id. @@ -131,24 +232,207 @@ def record_issued(key: str, *, db_path: Optional[str] = None) -> str: lic = parse_key(key) conn = connect(db_path) try: - conn.execute( - "INSERT INTO issued_licenses " - " (key_id, email, plan, seats, issued, expires, subscription_id, order_id, " - " status, created_at) " - "VALUES (?,?,?,?,?,?,?,?, 'active', ?) " - "ON CONFLICT(key_id) DO UPDATE SET " - " email=excluded.email, plan=excluded.plan, seats=excluded.seats, " - " issued=excluded.issued, expires=excluded.expires, " - " subscription_id=excluded.subscription_id, order_id=excluded.order_id", - (lic.key_id, lic.email, lic.plan, lic.seats, lic.issued, lic.expires, - lic.subscription_id or None, lic.order_id or None, time.time()), - ) + _record_issued_on_connection(conn, lic) conn.commit() finally: conn.close() return lic.key_id +def signer_rotation_state(replacement_signing_key_id: str, *, + db_path: Optional[str] = None) -> dict: + """Return registry state needed by the offline signer-reissue command. + + ``candidates`` contains customer PII and is vendor-admin data; unlike + :func:`inventory`, this result must never be exposed through an HTTP endpoint. + Completed audit rows let an interrupted command resume without duplicating keys. + """ + target = (replacement_signing_key_id or "").strip().lower() + if len(target) != 16 or any(char not in "0123456789abcdef" for char in target): + raise ValueError("replacement signing-key id must be 16 lowercase hex characters") + conn = connect(db_path) + try: + candidates = [ + dict(row) for row in conn.execute( + "SELECT issued.* FROM issued_licenses AS issued " + "WHERE issued.status='active' AND NOT EXISTS (" + " SELECT 1 FROM signer_rotation_reissues AS rotation " + " WHERE rotation.replacement_signing_key_id=? " + " AND (rotation.source_key_id=issued.key_id " + " OR rotation.replacement_key_id=issued.key_id)" + ") ORDER BY issued.key_id", + (target,), + ) + ] + completed = [ + dict(row) for row in conn.execute( + "SELECT source_key_id, replacement_key_id, source_signing_key_id, " + "replacement_signing_key_id, created_at " + "FROM signer_rotation_reissues WHERE replacement_signing_key_id=? " + "ORDER BY source_key_id", + (target,), + ) + ] + finally: + conn.close() + return {"candidates": candidates, "completed": completed} + + +def record_signer_rotation( + reissues: list[tuple[str, str, str]], *, replacement_signing_key_id: str, + db_path: Optional[str] = None, now: Optional[float] = None) -> int: + """Atomically record signer replacements while leaving every source key active. + + Each tuple is ``(source_key_id, source_signing_key_id, replacement_key)``. + Source revocation is deliberately separate and grace-period gated. + """ + target = (replacement_signing_key_id or "").strip().lower() + if len(target) != 16 or any(char not in "0123456789abcdef" for char in target): + raise ValueError("replacement signing-key id must be 16 lowercase hex characters") + + prepared = [] + source_ids = set() + replacement_ids = set() + for source_key_id, source_signing_key_id, replacement_key in reissues: + source_id = (source_key_id or "").strip() + source_signer = (source_signing_key_id or "").strip().lower() + if not source_id or len(source_signer) != 16 \ + or any(char not in "0123456789abcdef" for char in source_signer): + raise ValueError("source key id and 16-character hex signer id are required") + replacement = parse_key(replacement_key, now=0) + if replacement.signing_key_id != target: + raise ValueError("replacement key was not signed by the requested signer") + if source_id in source_ids or replacement.key_id in replacement_ids: + raise ValueError("duplicate source or replacement key in rotation batch") + if source_id == replacement.key_id: + raise ValueError("source and replacement key ids must differ") + source_ids.add(source_id) + replacement_ids.add(replacement.key_id) + prepared.append((source_id, source_signer, replacement_key, replacement)) + + if not prepared: + return 0 + + created_at = time.time() if now is None else float(now) + conn = connect(db_path) + inserted = 0 + try: + conn.execute("BEGIN IMMEDIATE") + for source_id, source_signer, _replacement_key, replacement in prepared: + existing = conn.execute( + "SELECT replacement_key_id, source_signing_key_id " + "FROM signer_rotation_reissues " + "WHERE source_key_id=? AND replacement_signing_key_id=?", + (source_id, target), + ).fetchone() + if existing is not None: + if existing["replacement_key_id"] != replacement.key_id \ + or existing["source_signing_key_id"] != source_signer: + raise ValueError("rotation audit conflicts with the requested replacement") + continue + + source = conn.execute( + "SELECT status, signing_key_id, email, plan, seats, issued, expires, " + "subscription_id, order_id FROM issued_licenses WHERE key_id=?", + (source_id,), + ).fetchone() + if source is None or source["status"] != "active": + raise ValueError("every signer-rotation source must still be active") + stored_signer = str(source["signing_key_id"] or "").strip().lower() + if stored_signer and stored_signer != source_signer: + raise ValueError("source signer does not match the registry") + source_entitlement = ( + str(source["email"] or ""), str(source["plan"] or ""), + max(1, int(source["seats"] or 1)), source["issued"], source["expires"], + str(source["subscription_id"] or ""), str(source["order_id"] or ""), + ) + replacement_entitlement = ( + replacement.email, replacement.plan, replacement.seats, + replacement.issued, replacement.expires, + replacement.subscription_id, replacement.order_id, + ) + if source_entitlement != replacement_entitlement: + raise ValueError("replacement key changes the source entitlement") + + _record_issued_on_connection(conn, replacement) + replacement_row = conn.execute( + "SELECT status FROM issued_licenses WHERE key_id=?", + (replacement.key_id,), + ).fetchone() + if replacement_row is None or replacement_row["status"] != "active": + raise ValueError("replacement key already has a revocation tombstone") + conn.execute( + "UPDATE issued_licenses SET signing_key_id=COALESCE(signing_key_id, ?) " + "WHERE key_id=?", + (source_signer, source_id), + ) + conn.execute( + "INSERT INTO signer_rotation_reissues " + "(source_key_id, replacement_key_id, source_signing_key_id, " + " replacement_signing_key_id, created_at) VALUES (?,?,?,?,?)", + (source_id, replacement.key_id, source_signer, target, created_at), + ) + inserted += 1 + conn.execute("COMMIT") + except BaseException: + if conn.in_transaction: + conn.execute("ROLLBACK") + raise + finally: + conn.close() + return inserted + + +def retire_signer_rotation_sources( + source_key_ids: list[str], *, replacement_signing_key_id: str, + db_path: Optional[str] = None, grace_seconds: float = 30 * 86400, + now: Optional[float] = None) -> int: + """Revoke audited source keys only after their replacements and grace period exist.""" + target = (replacement_signing_key_id or "").strip().lower() + if len(target) != 16 or any(char not in "0123456789abcdef" for char in target): + raise ValueError("replacement signing-key id must be 16 lowercase hex characters") + source_ids = sorted(set((key_id or "").strip() for key_id in source_key_ids)) + if not source_ids or any(not key_id for key_id in source_ids): + raise ValueError("at least one non-empty source key id is required") + + timestamp = time.time() if now is None else float(now) + minimum_age = max(30 * 86400, float(grace_seconds)) + conn = connect(db_path) + try: + conn.execute("BEGIN IMMEDIATE") + for source_id in source_ids: + audit = conn.execute( + "SELECT replacement_key_id, created_at FROM signer_rotation_reissues " + "WHERE source_key_id=? AND replacement_signing_key_id=?", + (source_id, target), + ).fetchone() + if audit is None: + raise ValueError("source key has no audited signer replacement") + if timestamp - float(audit["created_at"]) < minimum_age: + raise ValueError("the 30-day signer-rotation grace period has not elapsed") + replacement = conn.execute( + "SELECT status FROM issued_licenses WHERE key_id=?", + (audit["replacement_key_id"],), + ).fetchone() + if replacement is None or replacement["status"] != "active": + raise ValueError("an active replacement is required before source retirement") + + placeholders = ",".join("?" for _ in source_ids) + cursor = conn.execute( + "UPDATE issued_licenses SET status='revoked', revoked_at=? " + f"WHERE status='active' AND key_id IN ({placeholders})", + (timestamp, *source_ids), + ) + conn.execute("COMMIT") + return cursor.rowcount + except BaseException: + if conn.in_transaction: + conn.execute("ROLLBACK") + raise + finally: + conn.close() + + def revoke(key_id: str, *, db_path: Optional[str] = None) -> bool: """Persist a revocation tombstone. Returns True when state changed. @@ -282,6 +566,46 @@ def verify_for_feature(key: str, feature: str, *, db_path: Optional[str] = None, return lic +def record_control_plane_event(kind: str, *, db_path: Optional[str] = None, + now: Optional[float] = None) -> None: + """Record a content-free counter used by readiness alerts.""" + clean = (kind or "").strip().lower()[:48] + allowed = "abcdefghijklmnopqrstuvwxyz0123456789_.-" + if not clean or any(char not in allowed for char in clean): + raise ValueError("invalid control-plane event kind") + now = time.time() if now is None else float(now) + conn = connect(db_path) + try: + conn.execute( + "INSERT INTO control_plane_events(kind,occurred_at) VALUES (?,?)", (clean, now)) + conn.execute("DELETE FROM control_plane_events WHERE occurred_at bool: + """Fail readiness on a sustained recent lease/sync rejection rate.""" + now = time.time() if now is None else float(now) + try: + threshold = max(1, int(os.environ.get( + "ENGRAPHIS_REJECTED_LEASE_ALERT_THRESHOLD", "50"))) + window = max(60, int(os.environ.get( + "ENGRAPHIS_REJECTED_LEASE_ALERT_WINDOW_SECONDS", "3600"))) + conn = connect(db_path) + try: + count = int(conn.execute( + "SELECT COUNT(*) FROM control_plane_events " + "WHERE kind='lease_rejected' AND occurred_at>=?", (now - window,) + ).fetchone()[0]) + finally: + conn.close() + return count < threshold + except (OSError, TypeError, ValueError, sqlite3.Error): + return False + + # ── device registrations & seat accounting ───────────────────────────────────────────── # A "seat" is one concurrently-active device. The per-license cap (``License.seats``) is # enforced here, on vendor hardware, so it holds regardless of what a patched client does. @@ -305,7 +629,9 @@ def lease_ttl_seconds() -> int: try: hours = float(os.environ.get("ENGRAPHIS_LEASE_TTL_HOURS", "").strip() or LEASE_TTL_HOURS_DEFAULT) - except ValueError: + except (OverflowError, ValueError): + hours = LEASE_TTL_HOURS_DEFAULT + if not math.isfinite(hours): hours = LEASE_TTL_HOURS_DEFAULT return max(300, int(hours * 3600)) @@ -321,9 +647,12 @@ def seat_reclaim_seconds() -> int: never reclaim a live, about-to-renew device. Tunable via ``ENGRAPHIS_SEAT_RECLAIM_MULT`` (>= 1.0).""" try: - mult = max(1.0, float(os.environ.get("ENGRAPHIS_SEAT_RECLAIM_MULT", "").strip() or 2.0)) - except ValueError: + mult = float(os.environ.get("ENGRAPHIS_SEAT_RECLAIM_MULT", "").strip() or 2.0) + except (OverflowError, ValueError): + mult = 2.0 + if not math.isfinite(mult): mult = 2.0 + mult = max(1.0, mult) return int(lease_ttl_seconds() * mult) diff --git a/engraphis/inspector/sync_relay.py b/engraphis/inspector/sync_relay.py index dc66836..67cd72e 100644 --- a/engraphis/inspector/sync_relay.py +++ b/engraphis/inspector/sync_relay.py @@ -1,20 +1,20 @@ -"""Gated cloud-sync relay — the server-side half of the managed Pro sync transport. +"""Gated cloud-sync relay — the server-side half of managed Pro/Team sync. -Stores opaque per-account sync bundles and serves them only to a device that presents a -valid, unexpired, non-revoked license whose plan includes ``sync``. The gate -(:func:`license_registry.verify_for_feature`) runs here, on vendor hardware, so a client -that has patched its local feature check still cannot push or pull: no valid key, no -bundles. Bundles are namespaced by an account id derived from the license, so one -customer's devices never see another customer's data. +Stores opaque per-account sync bundles and serves them only to a device presenting an +unexpired, non-revoked, per-user token with the required sync scope. The customer server +also verifies its active Pro/Team entitlement before every request. A legacy license-key +path is retained only for the configured migration window. Bundles remain namespaced by +the account entitlement, so one customer never sees another customer's data. -Mounted OUTSIDE the ``/api/`` prefix so the dashboard's admin-token auth gate does not -apply — authentication here IS the license key, carried as ``Authorization: Bearer``. +Mounted OUTSIDE the ``/api/`` prefix because clients use scoped bearer tokens rather than +browser sessions; authentication is still enforced inside every relay handler. The bundle bytes are opaque to this layer (the sync engine treats every pulled bundle as untrusted anyway), so an end-to-end-encrypted client can push ciphertext unchanged. """ from __future__ import annotations import asyncio +import json import os import re import sqlite3 @@ -153,7 +153,7 @@ def _rate_limited(request: Request) -> bool: def _authorize(request: Request): - """Verify the caller's license server-side. Returns (license, account_id). + """Verify the caller's scoped token and account entitlement server-side. Raises :class:`LicenseError` (rendered as 402 by the app's exception handler) if the key is missing, malformed, expired, wrong-plan, or revoked, and @@ -177,7 +177,52 @@ def _authorize(request: Request): status_code=429, detail={"error": "too many sync attempts — try again shortly"}, headers={"Retry-After": "60"}) - lic = reg.verify_for_feature(_bearer_key(request), SYNC_FEATURE) + bearer = _bearer_key(request) + # Each Request owns its state, but initialize explicitly so the license-key path can + # never inherit the scoped-user marker if an adapter reuses a request-like object. + request.state.sync_user = None + + # Customer deployments authenticate Team members with their own hashed, revocable + # API token. The active account license remains the entitlement and namespace anchor; + # the user token supplies identity, role, and least-privilege sync scopes. + auth_store = getattr(request.app.state, "auth_store", None) + if auth_store is not None: + user = auth_store.resolve_api_token(bearer) + if user is not None: + write_request = request.method.upper() in ("POST", "PUT", "PATCH", "DELETE") + required_scope = "sync:write" if write_request else "sync:read" + if required_scope not in set(user.get("token_scopes") or ()): + raise HTTPException( + status_code=403, + detail={"error": "sync token lacks %s" % required_scope}) + # Roles are mutable after token issuance. Re-check the owner's current role on + # every write so downgrading a member to viewer immediately revokes write + # authority even if an older token still carries the sync:write scope. + if write_request and user.get("role") not in ("member", "admin"): + raise HTTPException( + status_code=403, + detail={"error": "the current account role is read-only"}) + from engraphis import licensing + lic = licensing.current_license() + if not lic.has(SYNC_FEATURE): + raise LicenseError("an active Pro or Team license is required for sync", + feature=SYNC_FEATURE) + request.state.sync_user = user + return lic, reg.account_id_for(lic) + if bearer.startswith("engr_ut_"): + raise HTTPException( + status_code=401, + detail={"error": "sync token is expired, revoked, or invalid"}, + ) + + lic = reg.verify_for_feature(bearer, SYNC_FEATURE) + from engraphis.config import settings + if lic.plan == "team" and settings.service_mode == "customer" \ + and os.environ.get("ENGRAPHIS_LEGACY_TEAM_KEY_SYNC", "0").strip().lower() \ + not in ("1", "true", "yes", "on"): + raise LicenseError( + "Team license-key sync is disabled; use a per-user scoped device token", + feature=SYNC_FEATURE) if lic.plan == "team": mid = _machine_id(request) if not mid: @@ -193,6 +238,54 @@ def _authorize(request: Request): return lic, reg.account_id_for(lic) +def _enforce_scoped_workspace(request: Request, workspace: str) -> None: + """Keep personal folders out of the account-wide relay namespace. + + A per-user token proves identity, scope, and current role, but the relay database is + intentionally shared by the whole licensed account. Therefore a scoped token may only + address a workspace that exists on this customer deployment and is explicitly shared + (or is a legacy workspace with no visibility flag). Personal, unknown, and malformed + workspace records all receive the same denial so names cannot be probed. + + Vendor relay calls authenticated by a license key retain their existing account-level + contract: the vendor process has no customer memory database with which to evaluate + local folder visibility. Official customer sync already refuses to upload personal + folders before using that legacy transport. + """ + request_state = getattr(request, "state", None) + scoped_user = getattr(request_state, "sync_user", None) + app_state = getattr(getattr(request, "app", None), "state", None) + svc = getattr(app_state, "service", None) + store = getattr(svc, "store", None) + conn = getattr(store, "conn", None) + if conn is None: + if scoped_user is None: + return + raise HTTPException(status_code=503, detail={ + "error": "workspace authorization is unavailable"}) + try: + rows = conn.fetchall( + "SELECT settings FROM workspaces WHERE name=?", (workspace,)) + if len(rows) != 1: + if scoped_user is None: + # A vendor relay has no copy of each customer's workspace catalog. Its + # license-key contract therefore remains account-scoped for unknown names. + return + raise ValueError("workspace is missing") + raw = rows[0]["settings"] + settings = {} if not raw else json.loads(raw) + visibility = settings.get("visibility") if isinstance(settings, dict) else None + if not isinstance(settings, dict) or visibility not in (None, "", "shared"): + raise ValueError("workspace is not shared") + except HTTPException: + raise + except Exception: # noqa: BLE001 - malformed/missing state must fail closed uniformly + raise HTTPException( + status_code=403, + detail={"error": "scoped sync tokens may access existing shared workspaces only"}, + ) from None + + router = APIRouter(prefix="/relay/v1", tags=["sync-relay"]) @@ -287,6 +380,7 @@ async def push_bundle(workspace_id: str, name: str, request: Request): safe = _safe_name(name) if not workspace or not safe: return JSONResponse({"error": "invalid workspace or bundle name"}, status_code=400) + await asyncio.to_thread(_enforce_scoped_workspace, request, workspace) declared = request.headers.get("Content-Length") if declared: try: @@ -329,6 +423,7 @@ async def delete_bundle(workspace_id: str, name: str, request: Request): safe = _safe_name(name) if not workspace or not safe: return JSONResponse({"error": "invalid workspace or bundle name"}, status_code=400) + await asyncio.to_thread(_enforce_scoped_workspace, request, workspace) deleted = await asyncio.to_thread( _delete_bundle, account_id, workspace, safe) return {"ok": True, "name": safe, "deleted": deleted} @@ -360,6 +455,7 @@ def pull_bundle(workspace_id: str, name: str, request: Request): safe = _safe_name(name) if not workspace or not safe: return JSONResponse({"error": "invalid workspace or bundle name"}, status_code=400) + _enforce_scoped_workspace(request, workspace) conn = _conn() try: row = conn.execute( @@ -392,6 +488,7 @@ def pull_bundles(workspace_id: str, request: Request): workspace = _safe_workspace_id(workspace_id) if not workspace: return JSONResponse({"error": "invalid workspace"}, status_code=400) + _enforce_scoped_workspace(request, workspace) conn = _conn() try: conn.execute("BEGIN") @@ -426,6 +523,7 @@ def list_names(workspace_id: str, request: Request): workspace = _safe_workspace_id(workspace_id) if not workspace: return JSONResponse({"error": "invalid workspace"}, status_code=400) + _enforce_scoped_workspace(request, workspace) conn = _conn() try: rows = conn.execute( diff --git a/engraphis/inspector/webhooks.py b/engraphis/inspector/webhooks.py index 6117e67..6c77b60 100644 --- a/engraphis/inspector/webhooks.py +++ b/engraphis/inspector/webhooks.py @@ -18,10 +18,12 @@ import email.message import hashlib +import hmac import logging import math import os import smtplib +import stat import time from pathlib import Path from typing import Optional @@ -35,6 +37,11 @@ logger = logging.getLogger("engraphis.webhooks") + +def _log_ref(value: object) -> str: + """Return a non-reversible correlation id for customer/provider values.""" + return hashlib.sha256(str(value or "").encode("utf-8", "replace")).hexdigest()[:12] + # Canonical public links used in outbound customer-facing email footers. Centralized # as constants so the URLs can't drift per-email — a previous invite shipped a # wrong repo URL (github.com/engraphis/engraphis) that 404'd for paying customers. @@ -58,12 +65,21 @@ def _hex_to_seed(value: str, *, source: str) -> bytes: def _read_seed_file(path: Path) -> bytes: try: - text = path.read_text(encoding="utf-8").strip() - except OSError: + resolved = path.expanduser().resolve(strict=True) + info = resolved.stat() + if not stat.S_ISREG(info.st_mode) or info.st_size <= 0 or info.st_size > 1024: + raise RuntimeError("signing key file must be a small regular file") + if os.name != "nt" and stat.S_IMODE(info.st_mode) & 0o077: + raise RuntimeError("signing key file must be owner-only (chmod 600)") + text = resolved.read_text(encoding="utf-8").strip() + except RuntimeError: + raise + except (OSError, UnicodeError): raise RuntimeError( - f"Signing key not found at {path} — set ENGRAPHIS_VENDOR_SIGNING_KEY to the " - f"vendor seed (64 hex chars) or run `python -m scripts.license_admin keygen`") - return _hex_to_seed(text, source=str(path)) + "Signing key file is unavailable — set ENGRAPHIS_VENDOR_SIGNING_KEY to the " + "vendor seed (64 hex chars) or run `python -m scripts.license_admin keygen`" + ) from None + return _hex_to_seed(text, source="signing key file") def _looks_like_hex_seed(value: str) -> bool: @@ -85,7 +101,7 @@ def _load_signing_secret() -> bytes: continue if _looks_like_hex_seed(val): return _hex_to_seed(val, source=env_name) - path = Path(val) + path = Path(val).expanduser() if path.exists(): return _read_seed_file(path) raise RuntimeError( @@ -122,7 +138,7 @@ def _product_plan_overrides() -> dict: return out -def _map_polar_product_to_plan(product_name: str) -> str: +def _map_polar_product_to_plan(product_name: str, product_id: str = "") -> str: """Map Polar product name to an Engraphis plan tier. An operator-configured exact-name override (``ENGRAPHIS_POLAR_PRODUCT_MAP``) wins so @@ -133,6 +149,12 @@ def _map_polar_product_to_plan(product_name: str) -> str: a useless free-tier key. Correct Pro-vs-Team routing depends on the Polar product name containing "pro"/"team" (or an explicit override) — keep them named so. """ + from engraphis.commercial import product_for_id, service_mode + configured = product_for_id(product_id) + if configured: + return configured["plan"] + if service_mode() == "vendor": + raise RuntimeError("unrecognized Polar product id") name = (product_name or "").lower() override = _product_plan_overrides().get(name.strip()) if override: @@ -142,24 +164,33 @@ def _map_polar_product_to_plan(product_name: str) -> str: if "pro" in name: return "pro" logger.warning( - "unrecognized paid product '%s' — defaulting to Pro so the buyer still gets a " + "unrecognized paid product ref=%s — defaulting to Pro so the buyer still gets a " "working key. Name your Polar products with 'Pro'/'Team' for correct tiering.", - product_name) + _log_ref(product_name)) return "pro" -def _key_days(product_name: str, metadata: dict) -> int: +def _key_days(product_name: str, metadata: dict, product_id: str = "") -> int: """How long an auto-issued key stays valid. - Precedence: explicit ``license_days`` metadata → annual detection (≈13 months so a - late renewal never locks a paying customer out) → monthly default with a 5-day - grace over the 30-day cycle (renewal fires a fresh ``order.paid`` each period).""" + Exact configured product id wins in the vendor service. Development compatibility + then uses explicit ``license_days`` metadata, name-based annual detection, or the + 35-day monthly default.""" + # In the isolated control plane, the exact manifest-backed product id is the + # authority. Editable Polar metadata must not turn a recognized monthly product into + # an arbitrarily long-lived entitlement. Combined-mode compatibility still honors + # the historical metadata override when there is no configured exact product id. + from engraphis.commercial import product_for_id + configured = product_for_id(product_id) + if configured: + return 395 if configured["interval"] == "annual" else 35 try: explicit = int(metadata.get("license_days") or 0) except (TypeError, ValueError): explicit = 0 if explicit > 0: return explicit + # Compatibility inference only: the production vendor path returned above. name = (product_name or "").lower() if "annual" in name or "year" in name or "yr" in name: return 395 @@ -175,7 +206,7 @@ def _key_days(product_name: str, metadata: dict) -> int: def _subscription_key_days(payload: dict, product_name: str, metadata: dict, - *, now: Optional[float] = None) -> int: + product_id: str = "", *, now: Optional[float] = None) -> int: """Validity (in days) for a key re-issued from a Subscription object mid-cycle. Bounds the key to the subscription's ``current_period_end`` (+ a small fixed grace) @@ -189,20 +220,29 @@ def _subscription_key_days(payload: dict, product_name: str, metadata: dict, end = _parse_ts(payload.get("current_period_end")) if end and end > now: return max(1, math.ceil((end - now) / 86400) + _KEY_PERIOD_GRACE_DAYS) - return _key_days(product_name, metadata) + return _key_days(product_name, metadata, product_id) -def _plan_label(product_name: str) -> str: +def _plan_label(product_name: str, product_id: str = "") -> str: """Clean tier label for customer-facing email copy ("Pro"/"Team").""" - return _map_polar_product_to_plan(product_name).title() + return _map_polar_product_to_plan(product_name, product_id).title() def _extract_email(data: dict) -> Optional[str]: """Pull the buyer email from an order OR subscription payload, defensively.""" cust = data.get("customer") or {} user = data.get("user") or {} - return (cust.get("email") or data.get("customer_email") - or data.get("email") or user.get("email")) + if not isinstance(cust, dict): + cust = {} + if not isinstance(user, dict): + user = {} + candidate = (cust.get("email") or data.get("customer_email") + or data.get("email") or user.get("email")) + if not isinstance(candidate, str): + return None + normalized = candidate.strip().lower() + from engraphis.inspector.auth import _EMAIL_RE + return normalized if _EMAIL_RE.match(normalized) else None def _extract_subscription_id(data: dict, *, object_is_subscription: bool = False) -> str: """Normalized Polar subscription id carried into the signed license payload.""" @@ -220,8 +260,14 @@ def _extract_order_id(data: dict) -> str: def _extract_product_name(data: dict) -> str: - product = data.get("product") or {} - return product.get("name") or data.get("product_name") or "Pro" + product = data.get("product") + if not isinstance(product, dict): + product = {} + raw = product.get("name") or data.get("product_name") or "Pro" + # Keep provider-controlled labels out of email/TSV control sequences while retaining + # ordinary Unicode product names. + clean = "".join(ch for ch in str(raw).strip() if ch.isprintable())[:80] + return clean or "Pro" def _extract_seats(data: dict) -> int: @@ -243,9 +289,15 @@ def _extract_seats(data: dict) -> int: ``quantity`` and ``metadata.seats`` are kept as defensive fallbacks for payload shapes Polar doesn't currently send for this product. """ - product = data.get("product") or {} + product = data.get("product") + if not isinstance(product, dict): + product = {} meta = product.get("metadata") or data.get("metadata") or {} + if not isinstance(meta, dict): + meta = {} subscription = data.get("subscription") or {} + if not isinstance(subscription, dict): + subscription = {} for candidate in ( data.get("seats"), subscription.get("seats"), @@ -256,7 +308,7 @@ def _extract_seats(data: dict) -> int: n = int(candidate) if n > 0: return n - except (TypeError, ValueError): + except (OverflowError, TypeError, ValueError): continue return 1 @@ -266,11 +318,13 @@ def _parse_ts(value) -> Optional[float]: if value in (None, ""): return None if isinstance(value, (int, float)): - return float(value) + parsed = float(value) + return parsed if math.isfinite(parsed) else None try: from datetime import datetime - return datetime.fromisoformat(str(value).replace("Z", "+00:00")).timestamp() - except (ValueError, TypeError): + parsed = datetime.fromisoformat(str(value).replace("Z", "+00:00")).timestamp() + return parsed if math.isfinite(parsed) else None + except (OverflowError, OSError, ValueError, TypeError): return None @@ -293,7 +347,8 @@ def _trial_days(period_end, *, now: Optional[float] = None) -> int: def issue_key(email_addr: str, product_name: str = "pro", seats: int = 1, days: Optional[int] = None, metadata: Optional[dict] = None, *, trial: bool = False, record: bool = True, - subscription_id: str = "", order_id: str = "") -> str: + subscription_id: str = "", order_id: str = "", + product_id: str = "") -> str: """Generate a signed ``ENGR1.xxx.yyy`` key for *email_addr*. Uses the pinned vendor signing key (``.secrets/vendor_signing.key`` or @@ -306,15 +361,16 @@ def issue_key(email_addr: str, product_name: str = "pro", seats: int = 1, pub = ed25519_public_key(secret).hex() if pub == _DEV_VENDOR_PUBKEY_HEX: logger.warning( - "signing with DEV keypair — keys are forgeable. Rotate with " - "`python -m scripts.license_admin keygen --force` before real sales." + "signing with DEV keypair — keys are forgeable. Generate a new signer with " + "`python -m scripts.license_admin keygen --key-file ` " + "before real sales." ) - plan = _map_polar_product_to_plan(product_name) + plan = _map_polar_product_to_plan(product_name, product_id) if plan not in PLAN_FEATURES: plan = "pro" if days is None: - days = _key_days(product_name, metadata or {}) + days = _key_days(product_name, metadata or {}, product_id) subscription_id = str(subscription_id or "").strip()[:128] order_id = str(order_id or "").strip()[:128] @@ -326,6 +382,7 @@ def issue_key(email_addr: str, product_name: str = "pro", seats: int = 1, "seats": max(1, int(seats)), "issued": int(now), "expires": int(now + days * 86400), + "signing_key_id": pub[:16], } if subscription_id: payload["subscription_id"] = subscription_id @@ -336,25 +393,36 @@ def issue_key(email_addr: str, product_name: str = "pro", seats: int = 1, # Server-side enforcement (online-only): every minted key carries a signed # ``enforce: "cloud"`` claim plus the license-server URL, so the client requires a live # lease (register/renew against that URL) and the key is useless offline or after - # revocation. The URL is ENGRAPHIS_KEY_CLOUD_URL if set, else the built-in vendor relay - # (settings.relay_url) — so keys are cloud-enforced by DEFAULT now, not opt-in. The + # revocation. The URL is ENGRAPHIS_KEY_CLOUD_URL if set, else the isolated license + # service — so keys are cloud-enforced by DEFAULT now, not opt-in. The # claim rides inside the Ed25519-signed payload, so customers cannot strip it. - from engraphis.config import canonicalize_relay_url, settings - enforce_url = canonicalize_relay_url( - os.environ.get("ENGRAPHIS_KEY_CLOUD_URL", "") or settings.relay_url) + from engraphis.config import ( + DEFAULT_LICENSE_SERVER_URL, + canonicalize_license_server_url, + ) + enforce_url = canonicalize_license_server_url( + os.environ.get("ENGRAPHIS_KEY_CLOUD_URL", "") or DEFAULT_LICENSE_SERVER_URL) if enforce_url: payload["enforce"] = "cloud" payload["cloud_url"] = enforce_url key = compose_key(payload, secret) - logger.info("issued %s key for %s (expires in %d days)", plan, email_addr, days) + logger.info( + "issued %s key for customer_ref=%s (expires in %d days)", + plan, _log_ref(email_addr), days) if record: - try: # Registry writes remain best-effort; never revoke before recording. + try: from engraphis.inspector.license_registry import ( record_issued, revoke_superseded) key_id = record_issued(key) if subscription_id: revoke_superseded(subscription_id, key_id) except Exception as exc: + from engraphis.commercial import service_mode + if service_mode() == "vendor": + # A cloud-enforced paid key that is absent from the registry cannot + # activate. Fail before enqueueing its email so Polar retries instead of + # recording a purchase-without-delivery. + raise RuntimeError("license registry unavailable") from exc logger.warning("could not record/reconcile issued key in registry (%s)", type(exc).__name__) return key @@ -407,7 +475,8 @@ def _resend_api_key() -> str: def _send_via_resend_api(to: str, subject: str, text_body: str, from_addr: str, - api_key: str, *, reply_to: Optional[str] = None) -> None: + api_key: str, *, reply_to: Optional[str] = None, + idempotency_key: str = "") -> str: """Send via Resend's HTTPS API (port 443). Works where outbound SMTP is blocked (Railway, Fly, many PaaS). Raises RuntimeError on any failure. @@ -427,19 +496,41 @@ def _send_via_resend_api(to: str, subject: str, text_body: str, from_addr: str, "Accept": "application/json", "User-Agent": "Engraphis/1.0 (+https://engraphis.com)", } + provider_key = str(idempotency_key or "").strip() + if provider_key: + # Resend retains POST /emails idempotency keys for 24 hours. The durable + # outbox's stable message id closes the crash window where Resend accepted a + # send but this process died before recording the provider response. + headers["Idempotency-Key"] = provider_key[:256] try: resp = httpx.post(_RESEND_API_URL, json=payload, headers=headers, timeout=20.0) - except httpx.HTTPError as exc: - raise RuntimeError("Resend API unreachable: %s" % exc) from exc + except httpx.HTTPError: + # httpx exceptions retain the request URL and can include provider details. + # Keep the delivery boundary useful without reflecting that data outward. + raise RuntimeError("Resend API is unreachable") from None if resp.status_code not in (200, 201): - raise RuntimeError( - "Resend API error HTTP %s: %s" % (resp.status_code, resp.text[:200])) + # The provider body is untrusted and can echo addresses, message content, or + # credentials. Status is sufficient for retry/operations diagnostics. + raise RuntimeError("Resend API rejected the request (HTTP %s)" % resp.status_code) + try: + response_body = resp.json() + provider_id = response_body.get("id") if isinstance(response_body, dict) else None + except (ValueError, TypeError, AttributeError): + provider_id = None + if not isinstance(provider_id, str) or not provider_id \ + or len(provider_id) > 160 \ + or any(ord(char) < 33 or ord(char) == 127 for char in provider_id): + # A successful send without its stable provider id cannot be correlated with + # delivery/bounce events. Keep the outbox retryable; its idempotency key makes a + # retry safe even when the provider accepted the first request. + raise RuntimeError("Resend API response omitted its message id") + return provider_id def email_configured() -> bool: """True if THIS process has its own outbound email delivery set up (Resend or SMTP). Callers use this to decide *before* attempting a send whether to use - local delivery or fall back to something else (e.g. the vendor relay's + local delivery or fall back to something else (e.g. the vendor control plane's ``/license/v1/team-invite`` for a self-hosted dashboard with no mail account of its own) — cheaper and clearer than attempting-and-catching.""" if _resend_api_key(): @@ -449,29 +540,20 @@ def email_configured() -> bool: and os.environ.get("ENGRAPHIS_SMTP_PASSWORD", "").strip()) -def _send_text_email(to: str, subject: str, text_body: str, *, - reply_to: Optional[str] = None) -> None: - """Deliver a plain-text email to *to*. Shared transport for every outbound - email this module sends (license keys, team invites, ...). - - Prefers the Resend HTTPS API (``ENGRAPHIS_RESEND_API_KEY`` or the Resend key - already in ``ENGRAPHIS_SMTP_PASSWORD``) because many hosts — Railway included — - block outbound SMTP ports, which makes ``smtplib`` hang until timeout. Falls - back to SMTP (``ENGRAPHIS_SMTP_*``). Raises RuntimeError if nothing is - configured, and raises on delivery failure — callers decide how to log/fall - back (see :func:`_issue_and_email`'s 0600 fallback file for the license-key - case; a team invite has no secret to lose, so it just logs and moves on). - ``reply_to``, when given, routes replies to a human (e.g. the admin who sent - a team invite) instead of the shared sending address. - """ +def _deliver_text_email(to: str, subject: str, text_body: str, + reply_to: Optional[str] = None, + idempotency_key: str = "") -> tuple[str, str]: + """Low-level provider call used only by the durable outbox worker.""" from_addr = os.environ.get("ENGRAPHIS_SMTP_FROM", "keys@engraphis.com").strip() api_key = _resend_api_key() if api_key: - logger.info("sending email to %s via Resend API", to) - _send_via_resend_api(to, subject, text_body, from_addr, api_key, reply_to=reply_to) - logger.info("email delivered to %s (Resend API)", to) - return + logger.info("sending transactional email via Resend API") + provider_id = _send_via_resend_api( + to, subject, text_body, from_addr, api_key, reply_to=reply_to, + idempotency_key=idempotency_key) or "" + logger.info("transactional email accepted by Resend API") + return "resend", provider_id smtp_host = os.environ.get("ENGRAPHIS_SMTP_HOST", "").strip() smtp_port = int(os.environ.get("ENGRAPHIS_SMTP_PORT", "587")) @@ -489,17 +571,45 @@ def _send_text_email(to: str, subject: str, text_body: str, *, msg["Subject"] = subject if reply_to: msg["Reply-To"] = reply_to + if idempotency_key: + # Resend documents this equivalent header for its SMTP transport. Other SMTP + # providers safely preserve or ignore the non-secret extension header. + msg["Resend-Idempotency-Key"] = str(idempotency_key)[:256] msg.set_content(text_body) - logger.info("sending email to %s via %s:%d", to, smtp_host, smtp_port) + logger.info("sending transactional email via configured SMTP provider") with smtplib.SMTP(smtp_host, smtp_port, timeout=30) as smtp: smtp.starttls() smtp.login(smtp_user, smtp_pass) smtp.send_message(msg) - logger.info("email delivered to %s (SMTP)", to) + logger.info("transactional email accepted by SMTP provider") + return "smtp", "" + + +def _send_text_email(to: str, subject: str, text_body: str, *, + reply_to: Optional[str] = None, kind: str = "transactional", + idempotency_key: str = "") -> str: + """Persist and immediately attempt a plain-text transactional email. + + Prefers the Resend HTTPS API (``ENGRAPHIS_RESEND_API_KEY`` or the Resend key + already in ``ENGRAPHIS_SMTP_PASSWORD``) because many hosts — Railway included — + block outbound SMTP ports, which makes ``smtplib`` hang until timeout. Falls + back to SMTP (``ENGRAPHIS_SMTP_*``). Raises RuntimeError if nothing is + configured, and raises on delivery failure — callers decide how to log/fall + back (see :func:`_issue_and_email`'s 0600 fallback file for the license-key + case; a team invite has no secret to lose, so it just logs and moves on). + ``reply_to``, when given, routes replies to a human (e.g. the admin who sent + a team invite) instead of the shared sending address. + """ + from engraphis import email_outbox + message_id = email_outbox.enqueue( + kind, to, subject, text_body, reply_to=reply_to, + idempotency_key=idempotency_key) + email_outbox.deliver_now(message_id, _deliver_text_email) + return message_id def send_license_email(to: str, key: str, product_name: str = "Pro", - is_trial: bool = False) -> None: + is_trial: bool = False, *, idempotency_key: str = "") -> None: """Deliver a license key to *to*. Raises RuntimeError if nothing is configured, and raises on delivery @@ -509,105 +619,9 @@ def send_license_email(to: str, key: str, product_name: str = "Pro", subject = ("Your Engraphis %s Trial License Key" if is_trial else "Your Engraphis %s License Key") % product_name text_body = _license_email_text(key, product_name, is_trial=is_trial) - _send_text_email(to, subject, text_body) - - -def _team_invite_email_text(name: str, role: str, dashboard_url: str, - invited_by: str = "", key: str = "", - to: str = "") -> str: - greeting = "Hi %s," % name if name else "Hi," - who = "%s has added you" % invited_by if invited_by else "You've been added" - where = (" %s\n\n" % dashboard_url if dashboard_url else - " Ask your admin for your team dashboard's web address.\n\n") - reply_note = ("\nQuestions? Just reply to this email — it goes straight to %s.\n" - % invited_by if invited_by else "") - login_email = to or "the address this email was sent to" - # When this instance is genuinely Team-licensed, the invite carries the *shared* - # team license key so the member can turn on Pro features (and cloud sync) on - # their OWN machine, taking one server-enforced seat. Built with %-formatting so - # the key's own characters are never re-evaluated by the outer .format() below. - # - # The email is deliberately structured as two clearly-separated options: - # 1. JOIN the team dashboard (the hosted/Railway instance) — sign in with email - # + password. NO license key is involved here, and pasting the key on the - # hosted instance is NOT how membership works (it just re-activates a license - # that is already active there). This was a real support confusion: members - # followed the concrete "paste the key" steps on the hosted dashboard and - # thought they'd joined, when joining = logging in with credentials. - # 2. OPTIONALLY run Engraphis locally and access the team's shared memories from - # your own machine — THAT is what the shared key is for. It unlocks Pro - # features AND cloud sync; turning on Cloud Sync then pulls the team's - # converged memory store down to your local SQLite file. The key goes into - # your LOCAL dashboard (http://127.0.0.1:8700), never the team dashboard. - activate = (""" -────────────────────────────────────────────────────────────────────────────── -OPTION 2 (optional): run Engraphis on your OWN computer and access the team's -memories locally -────────────────────────────────────────────────────────────────────────────── -Your team plan also unlocks Pro features (analytics, export, automation, and -cloud sync) on your own machine. The shared team license key below is for THIS — -activating Pro on a LOCAL copy of Engraphis you run yourself — NOT for the team -dashboard above (that instance is already licensed; please don't paste this key -there). - -Shared team license key: - - %s - -To activate on your own machine: - 1. Install and run Engraphis locally (the dashboard opens at - http://127.0.0.1:8700) - 2. Go to Settings -> License - 3. Paste the key above and click Activate - -(Or set the ENGRAPHIS_LICENSE_KEY environment variable, or save the key to -~/.engraphis/license.key.) The key verifies against our license server on first -use and takes one of your team's seats — please use it only on your own devices. - -To then access the team's shared memories on your machine: after activating, go -to Settings -> Cloud Sync and turn sync on for the same workspace your team -uses. Your local store converges with the team's (new memories and edits flow -both ways); you keep a full local copy that works offline. -""" % key if key else "") - # OPTION 2 only exists when this invite actually carries a key. Viewers never get one - # (they hold no seat), and neither does any instance without a shared Team license — - # so for them the "two ways… the second is optional" framing pointed at a section - # that was not in the email. Keyless invites get a single-path preamble instead, and - # the "OPTION 1" label drops with it (there is nothing to number against). - intro = ("""You have two ways to use Engraphis as part of this team. You only need the first -one to be a member; the second is optional.""" - if key else - "Here's everything you need to get started.") - option_heading = ("OPTION 1 (required to join the team): sign in to the team dashboard" - if key else "Sign in to the team dashboard") - return """{greeting} - -{who} to an Engraphis team dashboard as a {role}. - -{intro} - -────────────────────────────────────────────────────────────────────────────── -{option_heading} -────────────────────────────────────────────────────────────────────────────── -The team's shared memories live on a hosted dashboard. Open it and sign in: - -{where}Log in with this email address: {login_email} -Your admin sets your password directly and will share it with you separately — -this email does not contain it. - -No license key is needed here, and you should NOT paste a license key into this -dashboard — joining the team means signing in with your email and password, -nothing else. -{activate}{reply_note} -— The Engraphis team - -Learn more: {site_url} -Source & self-hosting: {repo_url} -""".format(greeting=greeting, who=who, role=role, where=where, intro=intro, - option_heading=option_heading, login_email=login_email, - activate=activate, reply_note=reply_note, - site_url=SITE_URL, repo_url=REPO_URL) - + _send_text_email(to, subject, text_body, + kind="trial_license" if is_trial else "purchase_license", + idempotency_key=idempotency_key) def _password_reset_email_text(name: str, reset_url: str) -> str: @@ -638,7 +652,23 @@ def send_password_reset_email(to: str, name: str, reset_url: str) -> None: """ subject = "Reset your Engraphis dashboard password" text_body = _password_reset_email_text(name, reset_url) - _send_text_email(to, subject, text_body) + _send_text_email(to, subject, text_body, kind="password_reset") + + +def queue_password_reset_email(to: str, name: str, reset_url: str, *, + idempotency_key: str) -> str: + """Durably queue a vendor-relayed password-reset email without sending inline. + + The control-plane endpoint uses this path so a temporary provider outage leaves a + recoverable pending operation. The outbox worker performs the bounded retries; the + caller receives neither the reset URL nor a provider message identifier. + """ + from engraphis import email_outbox + return email_outbox.enqueue( + "password_reset", to, "Reset your Engraphis dashboard password", + _password_reset_email_text(name, reset_url), + idempotency_key=idempotency_key, + ) def _trial_verify_email_text(verify_url: str, plan: str, minutes: int) -> str: @@ -680,7 +710,31 @@ def send_trial_verification_email(to: str, verify_url: str, plan: str = "team", """ subject = "Confirm your Engraphis %s trial" % plan.title() text_body = _trial_verify_email_text(verify_url, plan, minutes) - _send_text_email(to, subject, text_body) + _send_text_email(to, subject, text_body, kind="trial_confirmation") + + +def send_trial_claim_email(to: str, verify_url: str, plan: str = "team", *, + minutes: int = 30, idempotency_key: str = "") -> None: + """Send scanner-safe confirmation for a deployment-bound trial claim.""" + label = plan.title() + subject = "Confirm your Engraphis %s trial" % label + text_body = f"""Someone requested a free {label} trial for an Engraphis deployment. + +Review and confirm the request here. The link expires in {minutes} minutes: + + {verify_url} + +Opening the link only displays a confirmation page. The trial activates only after you +press the confirmation button. The signed license key is delivered directly to the +requesting deployment and is never displayed in your browser or sent by email. + +If you did not request this trial, ignore this message. + +— The Engraphis team +""" + _send_text_email( + to, subject, text_body, kind="trial_confirmation", + idempotency_key=idempotency_key) # The canonical hosted team dashboard (the Railway deployment members sign in @@ -692,39 +746,69 @@ def send_trial_verification_email(to: str, verify_url: str, plan: str = "team", DEFAULT_TEAM_DASHBOARD_URL = "https://team.engraphis.com/" +def _invitation_email_text(name: str, role: str, invite_url: str, + invited_by: str = "") -> str: + greeting = "Hi %s," % name if name else "Hi," + inviter = ("%s invited you" % invited_by if invited_by + else "You have been invited") + return """%s + +%s to join an Engraphis team as a %s. + +Choose your password and accept the invitation here. This one-time link expires in +72 hours and is replaced if your administrator resends the invitation: + + %s + +The invitation does not contain a temporary password or the team's account-wide +license key. After signing in, create your own scoped, revocable device token from +Settings -> Connect an agent if you want to use a local client. + +If you did not expect this invitation, ignore this message. + +— The Engraphis team + +Learn more: %s +Source & self-hosting: %s +""" % (greeting, inviter, role, invite_url, SITE_URL, REPO_URL) + + def send_team_invite_email(to: str, name: str, role: str, *, invited_by: str = "", - key: str = "", dashboard_url: Optional[str] = None) -> None: - """Notify a newly added dashboard team member (``/api/auth/users``) that - their account exists, and — when ``key`` is given — hand them the shared - Team license key so they can turn on Pro features (and cloud sync) on their - own machine, taking one of the team's server-enforced seats. - - Still deliberately carries NO password: the admin sets the initial dashboard - password directly in the "Add member" form, so it is already known only to the - admin. ``key`` is different in kind — it is the account's *shared* Team key - (the same one every member of this team uses; see ``docs/SYNC.md``), not a - per-user secret, and seat count is enforced server-side — so including it is - what actually makes a member a licensed member rather than just a dashboard - login. Callers pass it only for a genuinely Team-licensed instance (see - ``routes.v2_team._send_invite``). ``invited_by`` (the admin's own email) is - named in the body and set as Reply-To — important when this is sent through - the shared vendor relay (see ``inspector.license_cloud.team_invite``), where - the visible From address is the vendor's, not the actual team's. ``dashboard_url`` - overrides ``ENGRAPHIS_DASHBOARD_URL`` when not None (so a relay send can carry - the admin's own dashboard URL instead of the relay host's). Raises on delivery - failure (see :func:`_send_text_email`); the caller treats this as best-effort - and must not let it block account creation. - """ + invite_url: str = "", key: str = "", + dashboard_url: Optional[str] = None, + idempotency_key: str = "") -> None: + """Send a one-time invitation URL. The deprecated ``key`` argument is ignored.""" + from engraphis.inspector.auth import _EMAIL_RE + subject = "Accept your Engraphis team invitation" + if not invite_url: + base = (dashboard_url or os.environ.get("ENGRAPHIS_DASHBOARD_URL", "") + or DEFAULT_TEAM_DASHBOARD_URL).strip().rstrip("/") + invite_url = base + text_body = _invitation_email_text(name, role, invite_url, invited_by=invited_by) + reply_to = invited_by if invited_by and _EMAIL_RE.match(invited_by) else None + _send_text_email( + to, subject, text_body, reply_to=reply_to, kind="invitation", + idempotency_key=idempotency_key) + + +def queue_team_invite_email(to: str, name: str, role: str, *, invited_by: str = "", + invite_url: str = "", dashboard_url: Optional[str] = None, + idempotency_key: str) -> str: + """Durably queue a vendor-relayed invitation without a provider call inline.""" + from engraphis import email_outbox from engraphis.inspector.auth import _EMAIL_RE - subject = "You've been added to an Engraphis team" - if not (dashboard_url or "").strip(): - dashboard_url = os.environ.get("ENGRAPHIS_DASHBOARD_URL", "").strip() - if not dashboard_url: - dashboard_url = DEFAULT_TEAM_DASHBOARD_URL - text_body = _team_invite_email_text(name, role, dashboard_url, - invited_by=invited_by, key=key, to=to) + + subject = "Accept your Engraphis team invitation" + if not invite_url: + base = (dashboard_url or os.environ.get("ENGRAPHIS_DASHBOARD_URL", "") + or DEFAULT_TEAM_DASHBOARD_URL).strip().rstrip("/") + invite_url = base reply_to = invited_by if invited_by and _EMAIL_RE.match(invited_by) else None - _send_text_email(to, subject, text_body, reply_to=reply_to) + return email_outbox.enqueue( + "invitation", to, subject, + _invitation_email_text(name, role, invite_url, invited_by=invited_by), + reply_to=reply_to, idempotency_key=idempotency_key, + ) def _fallback_dir() -> Path: @@ -738,22 +822,55 @@ def _fallback_dir() -> Path: return Path(db).expanduser().resolve().parent except Exception: pass + from engraphis.commercial import service_mode + if service_mode() == "vendor": + # Match billing._dedup_path's vendor default so paid-key recovery, the Polar + # ledger, and commercial_backup all resolve the same managed directory. + relay = os.environ.get("ENGRAPHIS_RELAY_DB", "").strip() + if relay and relay != ":memory:": + return Path(relay).expanduser().resolve().parent + state_dir = os.environ.get("ENGRAPHIS_STATE_DIR", "").strip() + return (Path(state_dir).expanduser().resolve() if state_dir + else (Path.home() / ".engraphis").resolve()) return Path.cwd() +UNDELIVERED_LICENSE_KEYS_NAME = "undelivered_license_keys.tsv" + + def _persist_fallback_key(email_addr: str, key: str, product_name: str) -> Optional[Path]: """Write an undelivered key to a 0600 operator-only file (NEVER the app log). Returns the file path, or None if it couldn't be written. The raw key must not hit application logs — log aggregation is usually less protected than this file. """ - path = _fallback_dir() / "undelivered_license_keys.tsv" + path = _fallback_dir() / UNDELIVERED_LICENSE_KEYS_NAME try: + safe_key = str(key) + if not safe_key or len(safe_key) > 8192 \ + or any(ord(char) < 33 or ord(char) == 127 for char in safe_key): + return None path.parent.mkdir(parents=True, exist_ok=True) + if path.is_symlink(): + return None # Create with 0600 before writing any key material. - fd = os.open(str(path), os.O_WRONLY | os.O_CREAT | os.O_APPEND, 0o600) + flags = os.O_WRONLY | os.O_CREAT | os.O_APPEND | getattr(os, "O_NOFOLLOW", 0) + fd = os.open(str(path), flags, 0o600) with os.fdopen(fd, "a", encoding="utf-8") as fh: - fh.write("%d\t%s\t%s\t%s\n" % (int(time.time()), email_addr, product_name, key)) + info = os.fstat(fh.fileno()) + linked = os.lstat(path) + if not stat.S_ISREG(info.st_mode) or info.st_nlink != 1 \ + or not stat.S_ISREG(linked.st_mode) \ + or not os.path.samestat(info, linked): + return None + # Both fields originated outside the process. Keep one record per line even + # if a future caller bypasses the normal email/product validation helpers. + safe_email = " ".join(str(email_addr).split())[:320] + safe_product = " ".join(str(product_name).split())[:240] + fh.write("%d\t%s\t%s\t%s\n" % ( + int(time.time()), safe_email, safe_product, safe_key)) + fh.flush() + os.fsync(fh.fileno()) try: os.chmod(path, 0o600) except OSError: @@ -763,31 +880,119 @@ def _persist_fallback_key(email_addr: str, key: str, product_name: str) -> Optio return None +def manual_fulfillment_clear() -> bool: + """Return whether no operator-only undelivered-key record awaits reconciliation.""" + path = _fallback_dir() / UNDELIVERED_LICENSE_KEYS_NAME + try: + if path.is_symlink(): + return False + if not path.exists(): + return True + # Operators remove or encrypted-archive the reconciled file. Treat even an empty + # leftover as unresolved so backup cannot later fail its non-empty private-file + # invariant while operational readiness incorrectly reports green. + return False + except OSError: + return False + + +def _existing_license_delivery(idempotency_key: str) -> Optional[str]: + """Return the key already held by a durable purchase-email operation. + + This closes the retry window after a key/email was persisted but before the Polar + delivery claims were finalized. The outbox body is the recoverable source of truth; + a retry reuses that same signed key instead of minting and potentially emailing a + second entitlement for one order. + """ + if not idempotency_key: + return None + from engraphis import email_outbox + conn = email_outbox._connect() + try: + row = conn.execute( + "SELECT text_body FROM email_outbox WHERE idempotency_key=?", + (idempotency_key,)).fetchone() + finally: + conn.close() + if row is None: + return None + for line in str(row["text_body"] or "").splitlines(): + candidate = line.strip() + if candidate.startswith("ENGR1.") and candidate.count(".") == 2: + return candidate + raise RuntimeError("durable purchase email is missing its signed key") + + def _issue_and_email(email_addr: str, product_name: str, seats: int, days: Optional[int], *, is_trial: bool = False, - subscription_id: str = "", order_id: str = "") -> str: - """Mint a signed key and email it. On ANY delivery failure, persist the key to - the 0600 fallback file (never the log) and still return it, so a paid or trial - key is never lost and the webhook can 202 without a Polar retry-storm.""" + subscription_id: str = "", order_id: str = "", + product_id: str = "", fulfillment_id: str = "") -> str: + """Mint a signed key and create a recoverable delivery operation. + + Provider failures remain in the durable outbox. If the enqueue itself failed, persist + the only raw key in the encrypted-backup-covered 0600 operator fallback instead. + """ + email_idempotency_key = "" + if order_id: + # A renewal has a fresh order id. Reusing this stable key makes a retry converge + # on the original durable outbox message rather than enqueueing a second email. + email_idempotency_key = "purchase-license:" + order_id + existing_key = _existing_license_delivery(email_idempotency_key) + if existing_key: + return existing_key + elif fulfillment_id: + # Seat changes and legacy trial lifecycle events have no order id. Hash the + # route's server-derived fulfillment identity so a crash after durable outbox + # enqueue but before webhook finalization reuses the original key/email instead + # of minting a second entitlement on retry. + digest = hashlib.sha256(fulfillment_id.encode("utf-8")).hexdigest() + email_idempotency_key = "license-fulfillment:" + digest + existing_key = _existing_license_delivery(email_idempotency_key) + if existing_key: + return existing_key + + # Resolve every mapping before minting. In vendor mode a missing product id must not + # mint an un-emailable key, release the webhook claim, and mint another on each retry. + label = _plan_label(product_name, product_id) key = issue_key( email_addr, product_name=product_name, seats=seats, days=days, - trial=is_trial, subscription_id=subscription_id, order_id=order_id) - label = _plan_label(product_name) + trial=is_trial, subscription_id=subscription_id, order_id=order_id, + product_id=product_id) try: - send_license_email(email_addr, key, product_name=label, is_trial=is_trial) + send_license_email( + email_addr, key, product_name=label, is_trial=is_trial, + idempotency_key=email_idempotency_key) except Exception as exc: # noqa: BLE001 — a delivery failure must not lose a key fp = hashlib.sha256(key.encode("ascii")).hexdigest()[:12] + # A provider outage happens *after* enqueue; the durable outbox already contains + # the exact key and will retry it, so creating a second plaintext copy would only + # expand secret exposure. The 0600 fallback is reserved for the rarer case where + # the durable enqueue itself failed and no recoverable delivery operation exists. + try: + queued_key = _existing_license_delivery(email_idempotency_key) + except Exception: # noqa: BLE001 - the outbox may be the failing dependency + queued_key = None + if queued_key and hmac.compare_digest(queued_key, key): + logger.warning( + "email delivery deferred (%s) — key %s remains in the durable outbox", + type(exc).__name__, fp) + return key saved = _persist_fallback_key(email_addr, key, product_name) if saved: logger.warning( - "email delivery failed (%s) — key %s for %s saved to the private " + "email delivery failed (%s) — key %s for customer_ref=%s saved to the private " "fallback file (deliver manually)", - type(exc).__name__, fp, email_addr) + type(exc).__name__, fp, _log_ref(email_addr)) else: logger.error( - "email delivery failed (%s) AND could not persist key %s for %s — " - "reissue via `python -m scripts.license_admin issue`", - type(exc).__name__, fp, email_addr) + "email delivery failed (%s) AND could not persist key %s for " + "customer_ref=%s — Polar delivery remains retryable; use " + "`python -m scripts.license_admin issue` only if recovery stays down", + type(exc).__name__, fp, _log_ref(email_addr)) + # No provider delivery, durable outbox row, or operator recovery file exists. + # Never acknowledge the purchase in this state: let Polar redeliver so a + # later healthy attempt can mint and persist a recoverable entitlement. + raise RuntimeError("license delivery could not be persisted") from exc return key @@ -803,13 +1008,18 @@ def handle_order_paid(payload: dict) -> Optional[str]: logger.warning("order.paid missing customer email — cannot issue key") return None product = payload.get("product") or {} + if not isinstance(product, dict): + product = {} product_name = _extract_product_name(payload) + from engraphis.commercial import extract_product_id + product_id = extract_product_id(payload) seats = _extract_seats(payload) - days = _key_days(product_name, product.get("metadata") or {}) + days = _key_days(product_name, product.get("metadata") or {}, product_id) return _issue_and_email( email_addr, product_name, seats, days, subscription_id=_extract_subscription_id(payload), - order_id=_extract_order_id(payload)) + order_id=_extract_order_id(payload), product_id=product_id, + fulfillment_id=str(payload.get("_engraphis_fulfillment_id") or "")) def handle_subscription_updated(payload: dict) -> Optional[str]: @@ -829,17 +1039,25 @@ def handle_subscription_updated(payload: dict) -> Optional[str]: logger.warning("subscription.updated missing customer email — cannot re-issue key") return None product = payload.get("product") or {} + if not isinstance(product, dict): + product = {} product_name = _extract_product_name(payload) + from engraphis.commercial import extract_product_id + product_id = extract_product_id(payload) seats = _extract_seats(payload) # Bound the replacement key to the subscription's current paid period, NOT a fresh # full window from now — a mid-cycle seat change must not extend entitlement past the # period the customer has actually paid through. - days = _subscription_key_days(payload, product_name, product.get("metadata") or {}) - logger.info("seat count changed for %s (%s) -> %d seats, re-issuing key", - email_addr, product_name, seats) + days = _subscription_key_days( + payload, product_name, product.get("metadata") or {}, product_id) + logger.info( + "seat count changed for customer_ref=%s product_ref=%s -> %d seats, re-issuing key", + _log_ref(email_addr), _log_ref(product_name), seats) return _issue_and_email( email_addr, product_name, seats, days, - subscription_id=_extract_subscription_id(payload, object_is_subscription=True)) + subscription_id=_extract_subscription_id(payload, object_is_subscription=True), + product_id=product_id, + fulfillment_id=str(payload.get("_engraphis_fulfillment_id") or "")) def handle_subscription_created(payload: dict) -> Optional[str]: @@ -861,10 +1079,15 @@ def handle_subscription_created(payload: dict) -> Optional[str]: logger.warning("subscription.created (trial) missing customer email") return None product_name = _extract_product_name(payload) + from engraphis.commercial import extract_product_id + product_id = extract_product_id(payload) seats = _extract_seats(payload) days = _trial_days(payload.get("current_period_end")) - logger.info("trial started for %s (%s) — issuing %d-day key", - email_addr, product_name, days) + logger.info( + "trial started for customer_ref=%s product_ref=%s — issuing %d-day key", + _log_ref(email_addr), _log_ref(product_name), days) return _issue_and_email( email_addr, product_name, seats, days, is_trial=True, - subscription_id=_extract_subscription_id(payload, object_is_subscription=True)) + subscription_id=_extract_subscription_id(payload, object_is_subscription=True), + product_id=product_id, + fulfillment_id=str(payload.get("_engraphis_fulfillment_id") or "")) diff --git a/engraphis/licensing.py b/engraphis/licensing.py index 92d1936..2e5161e 100644 --- a/engraphis/licensing.py +++ b/engraphis/licensing.py @@ -19,6 +19,7 @@ import hashlib import hmac import json +import math import os import re import threading @@ -189,17 +190,26 @@ def upgrade_url(plan: Optional[str] = None) -> str: or DEFAULT_PRO_UPGRADE_URL) _KEY_PREFIX = "ENGR1" -# Pinned **production** Ed25519 verify key (32-byte public half). Rotated 2026-07-11 -# to a fresh CSPRNG keypair, retiring BOTH the original dev keypair (see -# _DEV_VENDOR_PUBKEY_HEX below) and the interim 2026-07-08 key d3520482…9e08, which is -# now treated as retired out of caution. Any license signed by an older seed no longer -# verifies. The private seed lives ONLY in the gitignored `.secrets/vendor_signing.key` -# on the issuance machine — it never ships in this repo, never in .env, never in any agent -# session. Anyone with only this repo CANNOT forge a valid key. Re-generate on an -# offline/trusted machine before the first real sale. -# ROTATE BEFORE SELLING: run `python -m scripts.license_admin keygen --force` and replace -# this constant with the printed public key. +# Pinned Ed25519 verifier (32-byte public half). This pre-sale key was generated on a +# development machine and is not approved for production issuance. The private seed never +# ships in this repo; production keeps its replacement only in the vendor secret store and +# an encrypted recovery backup. Anyone with only this repository cannot forge a valid key. +# +# ROTATE BEFORE SELLING: inventory and back up the production registry, then generate into +# a NEW file on a trusted machine: +# python -m scripts.license_admin keygen --key-file /vendor_signing.key +# Pin the printed public key through the reviewed compatibility/reissue ceremony in +# docs/COMMERCIAL_OPERATIONS.md. Do not overwrite or discard the old seed first. _VENDOR_PUBKEY_HEX = "0f9ede880d65184f4615221d03e8127c38e1b7a8f8d789a050780ae50c36421d" +# Previous production verify keys live here only during an audited rotation window. +# New issuance always uses ``_VENDOR_PUBKEY_HEX``; remove retired entries after every +# customer has received a replacement and the announced grace period has elapsed. +_PREVIOUS_VENDOR_PUBKEY_HEXES = () + +# Readiness intentionally fails until an operator completes the trusted-machine ceremony, +# updates the verifier pin, validates production issuance, and flips this source-controlled +# release gate in a separately reviewed change. +VENDOR_SIGNER_RELEASE_READY = False # Frozen fingerprint of the OLD, known-compromised dev keypair. Kept as a sentinel so # is_default_vendor_key() / production_warnings() can flag it if anyone ever re-pins it. # Its private half does NOT ship in this repo (`.secrets/` is gitignored), but it was @@ -364,6 +374,10 @@ class License: expires: Optional[float] = None features: frozenset = field(default_factory=frozenset) key_id: str = "" # short fingerprint for support/display; never the key itself + #: Fingerprint of the Ed25519 public key that verified this license. New keys carry + #: the same value in their signed payload; legacy keys derive it from the verifier + #: that accepted them so a pre-rotation inventory can still identify their signer. + signing_key_id: str = "" is_trial: bool = False # True when this is a time-boxed server-issued trial key #: Historical signed policy marker. Every paid/trial key now requires a live #: server-side lease regardless of this value; it remains for key compatibility. @@ -454,6 +468,24 @@ def vendor_public_key() -> bytes: return raw +def vendor_public_keys() -> tuple: + """Current plus temporarily trusted rotation keys, current first.""" + current = vendor_public_key() + if _pubkey_override_allowed(): + return (current,) + out = [current] + for hexkey in _PREVIOUS_VENDOR_PUBKEY_HEXES: + try: + raw = bytes.fromhex(hexkey) + except ValueError: + raise LicenseError("previous vendor public key is not valid hex") + if len(raw) != 32: + raise LicenseError("previous vendor public key must be 32 bytes") + if raw not in out: + out.append(raw) + return tuple(out) + + def compose_key(payload: dict, secret: bytes) -> str: """Build a signed key from a payload dict (vendor-side; see license_admin).""" body = json.dumps(payload, separators=(",", ":"), sort_keys=True).encode("utf-8") @@ -474,7 +506,9 @@ def parse_key(key: str, *, now: Optional[float] = None) -> License: sig = _b64u_decode(parts[2]) except (ValueError, base64.binascii.Error): raise LicenseError("license key is not valid base64url") - if not ed25519_verify(vendor_public_key(), body, sig): + verified_with = next( + (pub for pub in vendor_public_keys() if ed25519_verify(pub, body, sig)), None) + if verified_with is None: raise LicenseError("license signature is invalid (tampered or wrong vendor key)") try: payload = json.loads(body.decode("utf-8")) @@ -482,6 +516,10 @@ def parse_key(key: str, *, now: Optional[float] = None) -> License: raise LicenseError("license payload is not valid JSON") if not isinstance(payload, dict) or payload.get("v") != 1: raise LicenseError("unsupported license payload version") + verified_key_id = verified_with.hex()[:16] + signing_key_id = str(payload.get("signing_key_id", "") or "").strip().lower() + if signing_key_id and signing_key_id != verified_key_id: + raise LicenseError("license signing-key id does not match its signature") plan = str(payload.get("plan", "")).lower() if plan not in PLAN_FEATURES: @@ -492,7 +530,12 @@ def parse_key(key: str, *, now: Optional[float] = None) -> License: expires = float(expires) except (TypeError, ValueError): raise LicenseError("invalid expiry in license") - if (now if now is not None else time.time()) > expires: + if not math.isfinite(expires): + raise LicenseError("invalid expiry in license") + current_time = time.time() if now is None else float(now) + if not math.isfinite(current_time): + raise LicenseError("invalid license validation time") + if current_time > expires: raise LicenseError("license expired on %s — renew at %s" % ( time.strftime("%Y-%m-%d", time.gmtime(expires)), upgrade_url())) extra = payload.get("features") or [] @@ -501,12 +544,13 @@ def parse_key(key: str, *, now: Optional[float] = None) -> License: features = frozenset(PLAN_FEATURES[plan]) | frozenset(str(f) for f in extra) try: seats = max(1, int(payload.get("seats", 1))) - except (TypeError, ValueError): + except (OverflowError, TypeError, ValueError): seats = 1 return License( plan=plan, email=str(payload.get("email", "")), seats=seats, issued=payload.get("issued"), expires=expires, features=features, key_id=hashlib.sha256(key.encode("ascii")).hexdigest()[:12], + signing_key_id=verified_key_id, is_trial=bool(payload.get("trial")), enforce=str(payload.get("enforce", "") or "").strip().lower(), cloud_url=str(payload.get("cloud_url", "") or "").strip().rstrip("/"), @@ -572,7 +616,7 @@ def _cloud_gate(lic: "License", material: str) -> tuple: Online-only by product policy (no offline mode): the verification server is ``ENGRAPHIS_CLOUD_URL`` if set, else the URL signed into the key at issuance (``lic.cloud_url`` — unforgeable, inside the signed payload), else the built-in - vendor relay (``settings.relay_url``). Unlike the previous opt-in design, a paid key + isolated license service. Unlike the previous opt-in design, a paid key is NEVER unlocked by local signature alone: it must register with the server and hold an unexpired lease. If no server URL resolves at all (someone blanked the relay URL), we DENY — there is deliberately no offline path to paid features. Revoked / expired / @@ -585,7 +629,7 @@ def _cloud_gate(lic: "License", material: str) -> tuple: if not base: return False, ("server-side license verification is required for paid features " "but no license server is configured (ENGRAPHIS_CLOUD_URL and the " - "vendor relay URL are both empty)") + "license service URL and signed server URL are both empty)") try: from engraphis import cloud_license return cloud_license.gate(lic, material, base_url=base) @@ -921,13 +965,18 @@ def production_warnings() -> list: at startup so an operator can't accidentally ship the dev signing key or bill against a placeholder checkout link.""" warns = [] + if not VENDOR_SIGNER_RELEASE_READY: + warns.append( + "vendor signer is still marked pre-sale. Audit issued keys, rotate the seed " + "on a trusted machine, update the pinned public key, and set " + "VENDOR_SIGNER_RELEASE_READY=True in the reviewed rotation release.") if is_default_vendor_key(): warns.append( "vendor signing key is the built-in DEV keypair, whose private half has been " "on dev boxes / in agent sessions and is treated as compromised. Anyone holding " - "that seed can forge Pro/Team keys. Rotate before selling: run " - "`python -m scripts.license_admin keygen --force`, then pin the printed public " - "key in engraphis/licensing.py (_VENDOR_PUBKEY_HEX).") + "that seed can forge Pro/Team keys. Generate a replacement into a new secure " + "path, then complete the reviewed compatibility/reissue ceremony in " + "docs/COMMERCIAL_OPERATIONS.md.") if "github.com" in upgrade_url(): warns.append( "upgrade link still points at the GitHub pricing anchor, not a real checkout. " diff --git a/engraphis/llm/client.py b/engraphis/llm/client.py index 00d2fd8..6fb2e27 100644 --- a/engraphis/llm/client.py +++ b/engraphis/llm/client.py @@ -42,6 +42,19 @@ ) +class _LLMProviderError(RuntimeError): + """Sanitized provider failure safe to expose outside this client boundary.""" + + def __init__(self, *, status: Optional[int] = None, unreachable: bool = False) -> None: + self.status = status + self.unreachable = unreachable + if status is not None: + message = "LLM provider rejected the request (HTTP %d)" % status + else: + message = "Could not reach the configured LLM provider" + super().__init__(message) + + class LLMClient: """Thin REST client for multiple LLM providers.""" @@ -85,7 +98,7 @@ def chat( if not self.api_key: raise ValueError( "No LLM API key configured. Set ENGRAPHIS_LLM_API_KEY in .env " - f"or pass api_key= when constructing LLMClient (provider={self.provider})." + "or pass api_key= when constructing LLMClient." ) if self.provider == "anthropic": return self._chat_anthropic(messages, system, temperature, max_tokens) @@ -159,11 +172,11 @@ def ping(self) -> dict[str, Any]: "error": "", "provider": self.provider, "model": self.model} except Exception as exc: # noqa: BLE001 - external-provider boundary logger.error("LLM connection test failed (%s)", type(exc).__name__) - if isinstance(exc, httpx.HTTPStatusError): - status = exc.response.status_code + if isinstance(exc, _LLMProviderError) and exc.status is not None: + status = exc.status error = ("Provider rejected the request (HTTP %d). Check the API key, " "model name, and provider settings." % status) - elif isinstance(exc, httpx.RequestError): + elif isinstance(exc, _LLMProviderError) and exc.unreachable: error = ("Could not reach the configured provider. Check the base URL " "and network connection.") else: @@ -189,14 +202,15 @@ def _chat_openai_compat(self, messages, system, temperature, max_tokens) -> str: headers.update(self.extra_headers) url = f"{self.base_url}/chat/completions" - logger.debug("LLM POST %s model=%s", url, self.model) - resp = self._http.post(url, json=body, headers=headers) - resp.raise_for_status() - data = resp.json() + # Custom provider URLs (and Google's URL below) may carry credentials in + # their query string. Keep debug logging useful without ever emitting the + # configured endpoint verbatim. + logger.debug("LLM provider request started") + data = self._post_json(url, body, headers) try: return data["choices"][0]["message"]["content"] - except (KeyError, IndexError, TypeError) as exc: - raise ValueError(f"Unexpected LLM response format: {data}") from exc + except (KeyError, IndexError, TypeError): + raise ValueError("Unexpected LLM response format") from None def _chat_anthropic(self, messages, system, temperature, max_tokens) -> str: """Anthropic Messages API.""" @@ -218,14 +232,12 @@ def _chat_anthropic(self, messages, system, temperature, max_tokens) -> str: headers.update(self.extra_headers) url = f"{self.base_url}/messages" - logger.debug("LLM POST %s model=%s", url, self.model) - resp = self._http.post(url, json=body, headers=headers) - resp.raise_for_status() - data = resp.json() + logger.debug("LLM provider request started") + data = self._post_json(url, body, headers) try: return data["content"][0]["text"] - except (KeyError, IndexError, TypeError) as exc: - raise ValueError(f"Unexpected Anthropic response format: {data}") from exc + except (KeyError, IndexError, TypeError): + raise ValueError("Unexpected Anthropic response format") from None def _chat_google(self, messages, system, temperature, max_tokens) -> str: """Google Gemini generateContent API.""" @@ -251,14 +263,26 @@ def _chat_google(self, messages, system, temperature, max_tokens) -> str: f"{self.base_url}/models/{self.model}:generateContent" f"?key={self.api_key}" ) - logger.debug("LLM POST %s model=%s", url, self.model) - resp = self._http.post(url, json=body, headers=headers) - resp.raise_for_status() - data = resp.json() + logger.debug("LLM provider request started") + data = self._post_json(url, body, headers) try: return data["candidates"][0]["content"]["parts"][0]["text"] - except (KeyError, IndexError, TypeError) as exc: - raise ValueError(f"Unexpected Google response format: {data}") from exc + except (KeyError, IndexError, TypeError): + raise ValueError("Unexpected Google response format") from None + + def _post_json(self, url: str, body: dict[str, Any], headers: dict[str, str]) -> Any: + """POST once and discard provider URLs/bodies before an error crosses the boundary.""" + try: + resp = self._http.post(url, json=body, headers=headers) + resp.raise_for_status() + except httpx.HTTPStatusError as exc: + raise _LLMProviderError(status=exc.response.status_code) from None + except httpx.RequestError: + raise _LLMProviderError(unreachable=True) from None + try: + return resp.json() + except (ValueError, TypeError, AttributeError): + raise ValueError("Unexpected LLM response format") from None def _anthropic_msg(m: dict[str, str]) -> dict[str, str]: diff --git a/engraphis/observability.py b/engraphis/observability.py new file mode 100644 index 0000000..5520d66 --- /dev/null +++ b/engraphis/observability.py @@ -0,0 +1,86 @@ +"""Redacted JSON logging for hosted customer and control-plane services.""" +from __future__ import annotations + +import json +import logging +import os +import re +from datetime import datetime, timezone + + +_EMAIL = re.compile(r"(?(?[?&#;]{_SENSITIVE_NAME}=)[^&#\s]*" +) +_URL_USERINFO = re.compile( + r"(?i)([a-z][a-z0-9+.-]*://)([^/@\s:]+):([^/@\s]+)@" +) + + +def redact(value: object) -> str: + """Remove common credential and customer-identifier shapes from one log field.""" + text = str(value) + text = _LICENSE.sub("[license]", text) + text = _BEARER.sub("Bearer [redacted]", text) + # Provider URLs can carry credentials in query parameters (Google's LLM API uses + # ``?key=...``), while ASGI access logs may contain invite/reset codes. Preserve the + # parameter name and all non-sensitive parameters so the request remains diagnosable. + text = _URL_SECRET.sub(lambda match: match.group("prefix") + "[redacted]", text) + text = _URL_USERINFO.sub(r"\1[redacted]:[redacted]@", text) + # Exceptions and provider SDKs commonly render dictionaries as either JSON or Python + # reprs. Assignment-only redaction did not cover ``{\"api_key\": \"...\"}`` or + # ``token: value`` forms, so a structured formatter could still serialize a secret. + text = _COLON_SECRET.sub( + lambda match: match.group("prefix") + '"[redacted]"', text) + text = _ASSIGNMENT.sub(r"\1=[redacted]", text) + return _EMAIL.sub("[email]", text) + + +class RedactedJsonFormatter(logging.Formatter): + def format(self, record: logging.LogRecord) -> str: + payload = { + "timestamp": datetime.fromtimestamp( + record.created, tz=timezone.utc).isoformat(timespec="milliseconds"), + "level": record.levelname.lower(), + "logger": record.name[:120], + "event": redact(record.getMessage())[:4000], + } + if record.exc_info and record.exc_info[0]: + payload["error_type"] = record.exc_info[0].__name__[:120] + return json.dumps(payload, separators=(",", ":"), ensure_ascii=True) + + +def configure_structured_logging() -> bool: + """Install the redacted formatter when ``ENGRAPHIS_JSON_LOGS`` is enabled.""" + if os.environ.get("ENGRAPHIS_JSON_LOGS", "").strip().lower() not in ( + "1", "true", "yes", "on"): + return False + formatter = RedactedJsonFormatter() + configured = False + for name in ("", "uvicorn", "uvicorn.error", "uvicorn.access"): + logger = logging.getLogger(name) + for handler in logger.handlers: + handler.setFormatter(formatter) + configured = True + if not configured: + handler = logging.StreamHandler() + handler.setFormatter(formatter) + logging.getLogger().addHandler(handler) + return True diff --git a/engraphis/resend_events.py b/engraphis/resend_events.py new file mode 100644 index 0000000..2175d0a --- /dev/null +++ b/engraphis/resend_events.py @@ -0,0 +1,101 @@ +"""Verified Resend delivery-event webhook.""" +from __future__ import annotations + +import base64 +import binascii +import hashlib +import hmac +import json +import math +import os +import time +from typing import Optional + +from fastapi import APIRouter, Request +from fastapi.responses import JSONResponse + +from engraphis import email_outbox + +router = APIRouter(prefix="/email/v1", tags=["email"]) +MAX_BODY_BYTES = 256 * 1024 + + +def _secret_bytes() -> bytes: + raw = os.environ.get("RESEND_WEBHOOK_SECRET", "").strip() + if not raw: + return b"" + if not raw.startswith("whsec_"): + return raw.encode("utf-8") + try: + encoded = raw[6:] + return base64.b64decode(encoded + "=" * (-len(encoded) % 4), validate=True) + except (binascii.Error, ValueError): + return b"" + + +def webhook_secret_ready() -> bool: + """Return whether the configured signing secret is usable without exposing it.""" + return len(_secret_bytes()) >= 16 + + +def verify_signature(body: bytes, event_id: str, timestamp: str, + signature_header: str, *, now: Optional[float] = None) -> bool: + secret = _secret_bytes() + if len(secret) < 16 or not event_id or not timestamp or not signature_header: + return False + try: + stamp = int(timestamp) + except ValueError: + return False + current_time = time.time() if now is None else now + if not math.isfinite(current_time) or abs(current_time - stamp) > 300: + return False + signed = event_id.encode() + b"." + timestamp.encode() + b"." + body + expected = base64.b64encode(hmac.new(secret, signed, hashlib.sha256).digest()).decode() + for candidate in signature_header.split(): + if candidate.startswith("v1,") and hmac.compare_digest(candidate[3:], expected): + return True + return False + + +@router.post("/resend-events") +async def resend_event(request: Request): + content_length = request.headers.get("content-length") + if content_length: + try: + declared_length = int(content_length) + if declared_length < 0: + return JSONResponse({"accepted": False}, status_code=400) + if declared_length > MAX_BODY_BYTES: + return JSONResponse({"accepted": False}, status_code=413) + except ValueError: + return JSONResponse({"accepted": False}, status_code=400) + chunks = bytearray() + async for chunk in request.stream(): + chunks.extend(chunk) + if len(chunks) > MAX_BODY_BYTES: + return JSONResponse({"accepted": False}, status_code=413) + body = bytes(chunks) + if len(body) > MAX_BODY_BYTES: + return JSONResponse({"accepted": False}, status_code=413) + event_id = request.headers.get("svix-id", "") + stamp = request.headers.get("svix-timestamp", "") + signature = request.headers.get("svix-signature", "") + if len(event_id) > 255 or len(stamp) > 32 or len(signature) > 4096: + return JSONResponse({"accepted": False}, status_code=400) + if not verify_signature(body, event_id, stamp, signature): + return JSONResponse({"accepted": False}, status_code=401) + try: + payload = json.loads(body.decode("utf-8")) + except (ValueError, UnicodeDecodeError, RecursionError): + return JSONResponse({"accepted": False}, status_code=400) + data = payload.get("data") if isinstance(payload, dict) else {} + if not isinstance(data, dict): + return JSONResponse({"accepted": False}, status_code=400) + message_id = data.get("email_id") or data.get("id") or "" + event_type = payload.get("type") if isinstance(payload, dict) else "" + if not isinstance(message_id, str) or not isinstance(event_type, str): + return JSONResponse({"accepted": False}, status_code=400) + if not email_outbox.record_provider_event(event_id, message_id, event_type): + return JSONResponse({"accepted": False}, status_code=400) + return {"accepted": True} diff --git a/engraphis/routes/memory.py b/engraphis/routes/memory.py index 4c4226e..d3d78bf 100644 --- a/engraphis/routes/memory.py +++ b/engraphis/routes/memory.py @@ -232,8 +232,10 @@ async def chat_memory_context(req: ChatRequest): temperature=req.temperature, max_tokens=req.maxTokens or req.max_tokens, ) - except Exception as e: - logger.warning("LLM chat error: %s", e) + except Exception as exc: + # Some provider errors include a credentialed request URL. The client already + # receives a generic response, so keep the log equally content-free. + logger.warning("LLM chat error (%s)", type(exc).__name__) raise HTTPException(500, "LLM service unavailable") return _ok({"answer": answer, "context": ctx.get("chunks", []), "context_count": ctx["count"]}) diff --git a/engraphis/routes/v2_api.py b/engraphis/routes/v2_api.py index 95f90b8..7e99c12 100644 --- a/engraphis/routes/v2_api.py +++ b/engraphis/routes/v2_api.py @@ -8,16 +8,26 @@ from __future__ import annotations import json +import hmac import logging +import os +import threading import time +import weakref from typing import Optional -from fastapi import APIRouter, File, Form, HTTPException, Query, UploadFile -from pydantic import BaseModel +from fastapi import APIRouter, File, Form, HTTPException, Query, Request, UploadFile +from pydantic import BaseModel, Field from engraphis import licensing from engraphis.config import DEFAULT_RELAY_URL, canonicalize_relay_url, settings -from engraphis.service import MemoryService, ValidationError +from engraphis.netutil import is_local_request +from engraphis.service import ( + GraphIndexRebuilding, + GraphSceneCapacityExceeded, + MemoryService, + ValidationError, +) router = APIRouter(prefix="/api", tags=["dashboard"]) logger = logging.getLogger("engraphis.api") @@ -66,6 +76,21 @@ def _run(fn, *a, **k): """Call a service method, mapping validation errors to 400 and the rest to 500.""" try: return fn(*a, **k) + except GraphIndexRebuilding as exc: + raise HTTPException(status_code=409, detail={ + "error": str(exc), "index_state": "rebuilding", "job_id": exc.job_id, + }) + except GraphSceneCapacityExceeded as exc: + raise HTTPException(status_code=413, detail={ + "error": str(exc), + "safety_state": "capacity_exceeded", + "degraded": True, + "truncated": False, + "resource": exc.resource, + "count": exc.count, + "limit": exc.limit, + "recommended_action": "narrow repository, time, type, or relation filters", + }) except ValidationError as exc: raise HTTPException(status_code=400, detail={"error": str(exc)}) except HTTPException: @@ -185,19 +210,35 @@ def health(): def bootstrap(): lic = licensing.current_license(refresh=True).to_public_dict() lic["error"] = licensing.license_error() - wss = _run(service().list_workspaces).get("workspaces") or [] + current_service = service() + wss = _run(current_service.list_workspaces).get("workspaces") or [] + # A workspace-bound server rejects global aggregate statistics. Bootstrap must + # first choose one of the already-authorized workspaces so the dashboard can + # establish WS instead of failing before it renders the workspace switcher. + scoped_stats_workspace = None + if current_service.allowed_workspaces is not None and wss: + scoped_stats_workspace = max( + wss, + key=lambda item: (int(item.get("memories") or 0), str(item.get("name") or "")), + ).get("name") emb = None try: from engraphis.backends import embedder_st as _est from engraphis.backends.embedder_deterministic import DeterministicEmbedder - e = service().engine.embedder + e = current_service.engine.embedder d = int(getattr(e, "dim", 0)) semantic = not isinstance(e, DeterministicEmbedder) emb = {"class": type(e).__name__, "dim": d, "semantic": semantic, "model": settings.embed_model, "error": getattr(_est, "LAST_EMBEDDER_ERROR", "")} except Exception: # noqa: BLE001 pass - return {"license": lic, "workspaces": wss, "stats": _run(service().stats), "embedder": emb} + return { + "license": lic, + "workspaces": wss, + "stats": _run(current_service.stats, workspace=scoped_stats_workspace), + "embedder": emb, + "features": {"graph_ui_v2": bool(settings.graph_ui_v2)}, + } # ── workspaces / stats ──────────────────────────────────────────────────────── @@ -217,6 +258,122 @@ def workspaces(): "openrouter": "openai/gpt-4o-mini", } +_llm_connection_state: dict = { + "ok": False, + "provider": "", + "model": "", + "tested_at": 0.0, +} +_llm_extractor_lock = threading.RLock() +_sync_token_state_lock = threading.RLock() + + +class _DisabledExtractor: + """A race-safe equivalent of ``extractor=None`` for live configuration changes. + + ``MemoryEngine.ingest`` checks the attribute and then reads it again for ``extract``. + Replacing a live extractor with ``None`` between those reads can raise AttributeError; + a stable no-op object instead returns no facts and lets the engine use its normal + passthrough fallback. + """ + + @staticmethod + def extract(text: str, *, context: str = "") -> list: + return [] + + +_DISABLED_EXTRACTOR = _DisabledExtractor() + + +def _extractor_enabled() -> bool: + from engraphis.backends.extractor import LLMExtractor, StructuredLLMExtractor + return isinstance(service().engine.extractor, (LLMExtractor, StructuredLLMExtractor)) + + +def _close_llm(llm) -> None: + if llm is not None and hasattr(llm, "close"): + try: + llm.close() + except Exception: # noqa: BLE001 - cleanup cannot block a settings change + pass + + +def _retire_extractor(extractor) -> None: + """Close an old extractor only after every in-flight request releases it.""" + llm = getattr(extractor, "llm", None) + if llm is None or not hasattr(llm, "close"): + return + try: + weakref.finalize(extractor, _close_llm, llm) + except TypeError: + # A non-weak-referenceable third-party extractor cannot be closed safely here: + # another request may still be using it. Prefer a bounded, rare resource leak to + # terminating an in-flight provider call. + logger.warning("retired LLM extractor could not be finalized safely") + + +def _set_llm_extractor(enabled: bool, *, persist: bool = True) -> dict: + """Apply the dashboard extractor switch immediately and, when possible, durably.""" + with _llm_extractor_lock: + return _set_llm_extractor_locked(enabled, persist=persist) + + +def _set_llm_extractor_locked(enabled: bool, *, persist: bool) -> dict: + old = service().engine.extractor + if enabled: + from engraphis.backends.extractor import PassthroughExtractor, get_extractor + new = get_extractor("llm_structured") + if isinstance(new, PassthroughExtractor): + raise RuntimeError("structured LLM extractor could not be initialized") + service().engine.extractor = new + extractor = "llm_structured" + else: + service().engine.extractor = _DISABLED_EXTRACTOR + extractor = "none" + if old is not service().engine.extractor: + _retire_extractor(old) + + settings.extractor = extractor + settings.llm_auto_extract = bool(enabled) + os.environ["ENGRAPHIS_EXTRACTOR"] = extractor + os.environ["ENGRAPHIS_LLM_AUTO_EXTRACT"] = "1" if enabled else "0" + persisted = False + if persist: + try: + from engraphis.config import persist_project_env + persist_project_env({ + "ENGRAPHIS_EXTRACTOR": extractor, + "ENGRAPHIS_LLM_AUTO_EXTRACT": "1" if enabled else "0", + }) + persisted = True + except (OSError, ValueError) as exc: + logger.warning("could not persist LLM extractor setting (%s)", type(exc).__name__) + return { + "extractor": extractor, + "extractor_enabled": bool(enabled), + "auto_extract": bool(enabled), + "persisted": persisted, + } + + +def _record_llm_test(result: dict) -> None: + with _llm_extractor_lock: + _llm_connection_state.update({ + "ok": bool(result.get("ok")), + "provider": str(result.get("provider") or settings.llm_provider), + "model": str(result.get("model") or settings.llm_model), + "tested_at": time.time(), + }) + + +def _llm_is_verified(provider: str, model: str) -> bool: + with _llm_extractor_lock: + return bool( + _llm_connection_state.get("ok") + and _llm_connection_state.get("provider") == provider + and _llm_connection_state.get("model") == model + ) + @router.get("/llm/status") def llm_status(): @@ -226,13 +383,18 @@ def llm_status(): provider = settings.llm_provider or "openai" model = settings.llm_model or _LLM_DEFAULT_MODELS.get(provider, "") key_set = bool(settings.llm_api_key) + verified = bool(key_set and _llm_is_verified(provider, model)) return { "provider": provider, "model": model, "key_set": key_set, "base_url": settings.llm_base_url or "", "extractor": settings.extractor, + "extractor_enabled": _extractor_enabled(), + "auto_extract": bool(settings.llm_auto_extract), "configured": key_set and bool(model), + "working": verified, + "tested_at": (_llm_connection_state.get("tested_at") if verified else 0.0), "default_models": _LLM_DEFAULT_MODELS, # A copy-paste .env block so the user doesn't have to memorise var names. "env_snippet": ( @@ -241,6 +403,7 @@ def llm_status(): f"ENGRAPHIS_LLM_API_KEY=\n" + (f"ENGRAPHIS_LLM_BASE_URL={settings.llm_base_url}\n" if settings.llm_base_url else "") + ("ENGRAPHIS_EXTRACTOR=llm_structured\n" if key_set else "# set ENGRAPHIS_EXTRACTOR=llm_structured to use it\n") + + "ENGRAPHIS_LLM_AUTO_EXTRACT=1\n" ), } @@ -252,19 +415,145 @@ def llm_test(): instance's API credit, so it's not a viewer action. Returns the ping result; never raises (the client's ping() already swallows every failure into ``ok=False``).""" if not settings.llm_api_key: + _record_llm_test({"ok": False}) return {"ok": False, "error": "No API key configured. Set ENGRAPHIS_LLM_API_KEY in your .env and restart.", "provider": settings.llm_provider, "model": settings.llm_model} try: from engraphis.llm.client import LLMClient with LLMClient() as llm: - return llm.ping() + result = llm.ping() + _record_llm_test(result) + if result.get("ok") and settings.llm_auto_extract: + result.update(_set_llm_extractor(True)) + result["auto_enabled"] = True + else: + result.update({ + "extractor": settings.extractor, + "extractor_enabled": _extractor_enabled(), + "auto_extract": bool(settings.llm_auto_extract), + "auto_enabled": False, + }) + return result except Exception as exc: # noqa: BLE001 + _record_llm_test({"ok": False}) logger.error("LLM connection test failed (%s)", type(exc).__name__) return {"ok": False, "error": "The provider test failed. Check the configured " "provider, model, and network connection.", "provider": settings.llm_provider, "model": settings.llm_model} +class _ExtractorToggleReq(BaseModel): + enabled: bool + + +@router.post("/llm/extractor") +def llm_extractor_toggle(req: _ExtractorToggleReq): + """Turn structured extraction on/off immediately; enabling requires a live provider.""" + if not req.enabled: + return {"ok": True, **_set_llm_extractor(False)} + if not settings.llm_api_key: + raise HTTPException(status_code=400, detail={ + "error": "Connect an LLM and set its API key before enabling extraction."}) + try: + from engraphis.llm.client import LLMClient + with LLMClient() as llm: + result = llm.ping() + except Exception as exc: # noqa: BLE001 - provider clients fail in many library-specific ways + _record_llm_test({"ok": False}) + logger.error("LLM extractor verification failed (%s)", type(exc).__name__) + raise HTTPException(status_code=400, detail={ + "error": "The configured LLM could not be verified. Check the provider, " + "model, API key, and network connection."}) from None + _record_llm_test(result) + if not result.get("ok"): + raise HTTPException(status_code=400, detail={ + "error": result.get("error") or "The configured LLM is not working."}) + return {"ok": True, "provider": result.get("provider"), + "model": result.get("model"), **_set_llm_extractor(True)} + + +def _metadata_object(raw) -> dict: + if isinstance(raw, dict): + return raw + try: + parsed = json.loads(raw or "{}") + return parsed if isinstance(parsed, dict) else {} + except (TypeError, ValueError, RecursionError): + return {} + + +@router.get("/llm/activity") +def llm_activity(workspace: Optional[str] = None, limit: int = 100): + """List memories the LLM extracted, consolidated, or retention-classified. + + This is intentionally a derived audit view: it exposes stored memory outcomes and + bounded metadata, never prompts, API keys, or raw provider responses. + """ + ws = workspace or _default_ws() + if not ws: + return {"workspace": "", "count": 0, "activities": []} + try: + ws = service()._clean_ws(ws) + except ValidationError as exc: + raise HTTPException(status_code=400, detail={"error": str(exc)}) + row = service().store.conn.execute( + "SELECT id FROM workspaces WHERE name=?", (ws,) + ).fetchone() + if row is None: + return {"workspace": ws, "count": 0, "activities": []} + rows = service().store.conn.execute( + "SELECT id, title, content, mtype, ingested_at, metadata FROM memories " + "WHERE workspace_id=? AND valid_to IS NULL AND expired_at IS NULL AND (" + "metadata LIKE '%\"llm_extraction\"%' OR " + "metadata LIKE '%\"structured_extraction\"%' OR " + "metadata LIKE '%\"structured_consolidation\"%' OR " + "metadata LIKE '%\"retention_supervision\"%') " + "ORDER BY ingested_at DESC LIMIT ?", + (row["id"], max(1, min(500, int(limit)))), + ).fetchall() + activities = [] + for record in rows: + metadata = _metadata_object(record["metadata"]) + extraction = metadata.get("llm_extraction") + consolidation = metadata.get("structured_consolidation") + retention = metadata.get("retention_supervision") + if isinstance(extraction, dict): + action = "extracted" + detail = extraction + elif isinstance(consolidation, dict): + action = "consolidated" + detail = consolidation + elif isinstance(retention, dict) and retention.get("source") == "llm": + action = "retention supervised" + detail = retention + elif isinstance(metadata.get("structured_extraction"), dict): + action = "extracted" + detail = {"mode": "llm_structured", "legacy": True} + else: + continue + structured = metadata.get("structured_extraction") or {} + entities = metadata.get("entities") or structured.get("entities") or [] + relations = metadata.get("relations") or structured.get("relations") or [] + activities.append({ + "id": record["id"], + "title": record["title"] or "", + "content": record["content"] or "", + "mtype": record["mtype"] or "semantic", + "ingested_at": record["ingested_at"], + "action": action, + "provider": detail.get("provider") or "", + "model": detail.get("model") or "", + "mode": detail.get("mode") or "", + "fact_index": detail.get("fact_index"), + "fact_count": detail.get("fact_count"), + "confidence": detail.get("confidence", structured.get("confidence")), + "entities": entities[:20] if isinstance(entities, list) else [], + "relations": relations[:10] if isinstance(relations, list) else [], + "source_count": detail.get("source_count"), + }) + return {"workspace": ws, "count": len(activities), "activities": activities} + + class _CreateWsReq(BaseModel): workspace: str description: str = "" @@ -1017,7 +1306,157 @@ def graph(workspace: Optional[str] = None, limit: int = 2000, ] return _run( service().graph, workspace=ws, limit=limit, layers=selected, - include_code=include_code, repo=repo, + include_code=include_code, repo=repo, backfill=False, + ) + + +def _graph_csv(value: Optional[str]) -> Optional[list[str]]: + if value is None: + return None + items = list(dict.fromkeys(item.strip() for item in value.split(",") if item.strip())) + if len(items) > 64 or any(len(item) > 200 for item in items): + raise HTTPException( + status_code=422, + detail={"error": "graph filters allow at most 64 values of 200 characters"}, + ) + return items + + +@router.get("/graph/scene") +def graph_scene(workspace: Optional[str] = None, level: str = "overview", + center_id: Optional[str] = None, system_id: Optional[str] = None, + seeds: Optional[str] = None, repo: Optional[str] = None, + layers: Optional[str] = None, relations: Optional[str] = None, + entity_types: Optional[str] = None, + memory_types: Optional[str] = None, + as_of: Optional[float] = None, + time_from: Optional[float] = None, + time_to: Optional[float] = None, + depth: int = Query(default=1, ge=0, le=2), + min_support: int = Query(default=1, ge=0, le=1_000_000), + min_confidence: float = Query(default=0.0, ge=0.0, le=1.0), + include_code: bool = False, code_overlay: Optional[bool] = None, + include_weak_co_occurs: Optional[bool] = None, + include_weak_cooccurrence: Optional[bool] = None, + node_limit: Optional[int] = Query(default=None, ge=1, le=300), + edge_limit: Optional[int] = Query(default=None, ge=0, le=900)): + """Complete or focused evidence-backed graph scene with deterministic identity.""" + ws = workspace or _require_ws() + weak_cooccurrence = ( + include_weak_cooccurrence + if include_weak_cooccurrence is not None else + include_weak_co_occurs + if include_weak_co_occurs is not None else + level.strip().lower() == "complete" + ) + code_enabled = include_code if code_overlay is None else code_overlay + return _run( + service().graph_scene, workspace=ws, level=level, + center_id=center_id, system_id=system_id, seeds=_graph_csv(seeds), + repo=repo, layers=_graph_csv(layers), relations=_graph_csv(relations), + entity_types=_graph_csv(entity_types), memory_types=_graph_csv(memory_types), + as_of=as_of, time_from=time_from, time_to=time_to, depth=depth, + min_support=min_support, min_confidence=min_confidence, + include_weak_cooccurrence=weak_cooccurrence, + include_code=code_enabled, node_limit=node_limit, edge_limit=edge_limit, + ) + + +@router.get("/graph/suggest") +def graph_suggest(q: str = "", query: Optional[str] = None, + workspace: Optional[str] = None, + repo: Optional[str] = None, + memory_types: Optional[str] = None, + as_of: Optional[float] = None, + time_from: Optional[float] = None, + time_to: Optional[float] = None, + include_weak_cooccurrence: bool = False, + limit: int = Query(default=8, ge=1, le=25)): + ws = workspace or _require_ws() + return _run( + service().graph_suggest, query if query is not None else q, + workspace=ws, repo=repo, memory_types=_graph_csv(memory_types), + as_of=as_of, time_from=time_from, time_to=time_to, + include_weak_cooccurrence=include_weak_cooccurrence, limit=limit, + ) + + +@router.get("/graph/entities/{canonical_id}") +def graph_entity(canonical_id: str, workspace: Optional[str] = None, + repo: Optional[str] = None, + memory_types: Optional[str] = None, + as_of: Optional[float] = None, + time_from: Optional[float] = None, + time_to: Optional[float] = None, + include_weak_cooccurrence: bool = True): + ws = workspace or _require_ws() + return _run( + service().graph_entity, canonical_id, workspace=ws, repo=repo, + memory_types=_graph_csv(memory_types), as_of=as_of, + time_from=time_from, time_to=time_to, + include_weak_cooccurrence=include_weak_cooccurrence, + ) + + +@router.get("/graph/path") +def graph_path(source: str, target: str, workspace: Optional[str] = None, + repo: Optional[str] = None, as_of: Optional[float] = None, + memory_types: Optional[str] = None, + time_from: Optional[float] = None, + time_to: Optional[float] = None, + max_hops: int = Query(default=8, ge=1, le=8), + max_visits: int = Query(default=10_000, ge=1, le=50_000), + include_weak_cooccurrence: bool = False): + ws = workspace or _require_ws() + return _run( + service().graph_path, source, target, workspace=ws, repo=repo, + as_of=as_of, memory_types=_graph_csv(memory_types), + time_from=time_from, time_to=time_to, + max_hops=max_hops, max_visits=max_visits, + include_weak_cooccurrence=include_weak_cooccurrence, + ) + + +class _GraphIndexReq(BaseModel): + workspace: str + repo: Optional[str] = None + dry_run: bool = True + extractor: str = Field(default="regex", pattern=r"^regex$") + + +class _GraphIndexCancelReq(BaseModel): + workspace: str + + +@router.get("/graph/index/status") +def graph_index_status(workspace: Optional[str] = None): + """Current generation and latest explicit graph-index job for a workspace.""" + return _run(service().graph_index_status, workspace=workspace or _require_ws()) + + +@router.post("/graph/index/jobs") +def graph_index_start(req: _GraphIndexReq): + """Start an idempotent, persisted graph-index job (dry-run by default).""" + return _run( + service().start_graph_index_job, + workspace=req.workspace, + repo=req.repo, + dry_run=req.dry_run, + extractor=req.extractor, + ) + + +@router.get("/graph/index/jobs/{job_id}") +def graph_index_job(job_id: str, workspace: Optional[str] = None): + return _run( + service().graph_index_job, job_id, workspace=workspace or _require_ws() + ) + + +@router.post("/graph/index/jobs/{job_id}/cancel") +def graph_index_cancel(job_id: str, req: _GraphIndexCancelReq): + return _run( + service().cancel_graph_index_job, job_id, workspace=req.workspace ) @@ -1080,11 +1519,19 @@ def code_export(workspace: str, repo: str): # ── license ─────────────────────────────────────────────────────────────────── class _KeyReq(BaseModel): - key: str + key: str = Field(..., min_length=1, max_length=8192) class _TrialReq(BaseModel): - email: str = "" + email: str = Field(default="", max_length=320) + + +class _TrialClaimReq(BaseModel): + email: str = Field(..., min_length=3, max_length=320) + plan: str = Field(..., min_length=3, max_length=16) + # Optional: a remote caller must still send the configured ownership token (enforced + # in the handler), but a trusted loopback caller may omit it — the local UI does. + deployment_token: str = Field(default="", max_length=8192) @router.get("/license") @@ -1114,31 +1561,170 @@ def activate_license(req: _KeyReq): @router.post("/license/trial") -def start_trial(req: _TrialReq): - """Begin the one-time self-serve free Pro trial. Requires ``email`` in the body — - since 2026-07-14 a key is no longer issued synchronously; a one-time confirmation - link is emailed to it instead (see ``licensing.start_trial``), so a normal - response here is ``{"pending": true, ...}``, not an activated license. 400 if the - email is missing/invalid, a paid license is active, or the trial is spent.""" - try: - return licensing.start_trial(email=req.email) - except licensing.LicenseError as exc: - raise HTTPException(status_code=400, detail={"error": str(exc)}) +def start_trial(req: _TrialReq, request: Request): + """Deprecated v1.0 wrapper for a deployment-bound Pro claim.""" + token, fallback = _trial_binding(request) + result = _start_bound_trial(req.email, "pro", token, dashboard_url_fallback=fallback) + result["deprecated"] = True + result["replacement"] = "/api/license/trials" + return result @router.post("/license/team-trial") -def start_team_trial(req: _TrialReq): - """Begin the one-time self-serve Team trial (unlocks Team mode, seats, and the - invite-email relay — a real signed key from the vendor, no purchase). Requires - ``email`` in the body; since 2026-07-14 a normal response is ``{"pending": true, - ...}`` — the key is only minted once a confirmation link emailed to it is opened, - not returned synchronously here. See ``licensing.start_team_trial``. 400 if the - email is missing/invalid, a paid license is active, this device's trial was - already claimed, or the relay is unreachable.""" +def start_team_trial(req: _TrialReq, request: Request): + """Deprecated v1.0 wrapper for a deployment-bound Team claim.""" + token, fallback = _trial_binding(request) + result = _start_bound_trial(req.email, "team", token, dashboard_url_fallback=fallback) + result["deprecated"] = True + result["replacement"] = "/api/license/trials" + return result + + +def _configured_deployment_token() -> str: + return os.environ.get("ENGRAPHIS_DEPLOYMENT_TOKEN", "").strip() + + +def _local_deployment_token() -> str: + """A stable, machine-bound trial token for a trusted loopback caller. + + Loopback requests are already fully trusted by the dashboard auth gate (they reach + every ``/api`` route without a bearer, and first-admin setup accepts them the same + way), so the deployment-token ownership proof adds no security on localhost — it only + stops the local operator from starting a trial with a secret that, on a self-hosted + box, nobody ever configured. Derive a deterministic value from the machine id so the + create -> email-confirm -> poll round-trip binds to a single ``deployment_hash`` even + across a process restart, without asking the operator to invent one. + """ + import hashlib + + from engraphis import cloud_license + digest = hashlib.sha256( + ("engraphis-local-trial:" + cloud_license.machine_id()).encode("utf-8")).hexdigest() + return "local-" + digest + + +def _effective_deployment_token(request: Request) -> str: + """The configured ownership token, or a machine-bound one for trusted loopback.""" + configured = _configured_deployment_token() + if configured: + return configured + return _local_deployment_token() if is_local_request(request) else "" + + +def _trial_binding(request: Request) -> "tuple[str, str]": + """``(deployment_token, dashboard_url_fallback)`` for a trial from *request*. + + On loopback with nothing configured, the fallback dashboard URL is the request's own + origin, so a purely local instance needs neither ``ENGRAPHIS_DEPLOYMENT_TOKEN`` nor + ``ENGRAPHIS_DASHBOARD_URL`` set to start a trial. A proxied internet request never + looks local (any ``X-Forwarded-*`` header disqualifies it), so this changes nothing + for a hosted deployment. + """ + local = is_local_request(request) + token = _configured_deployment_token() or (_local_deployment_token() if local else "") + fallback = str(request.base_url).rstrip("/") if local else "" + return token, fallback + + +def _start_bound_trial(email: str, plan: str, deployment_token: str, + *, dashboard_url_fallback: str = "") -> dict: + email = email.strip().lower() + if not email or "@" not in email or len(email) > 320: + raise HTTPException(status_code=400, detail={ + "error": "a valid email address is required to start a trial"}) + if not deployment_token: + raise HTTPException(status_code=503, detail={ + "error": "ENGRAPHIS_DEPLOYMENT_TOKEN is not configured"}) + dashboard_url = (os.environ.get("ENGRAPHIS_DASHBOARD_URL", "").strip() + or (dashboard_url_fallback or "").strip()) + if not dashboard_url: + raise HTTPException(status_code=503, detail={ + "error": "ENGRAPHIS_DASHBOARD_URL is required for hosted trials"}) + from engraphis import cloud_license + from engraphis.config import resolve_license_server_url try: - return licensing.start_team_trial(email=req.email) - except licensing.LicenseError as exc: + result = cloud_license.create_trial_claim( + resolve_license_server_url(), deployment_token, cloud_license.machine_id(), + email, plan, dashboard_url=dashboard_url) + except (RuntimeError, ValueError) as exc: raise HTTPException(status_code=400, detail={"error": str(exc)}) + result.pop("key", None) + return result + + +@router.post("/license/trials") +def create_deployment_trial(req: _TrialClaimReq, request: Request): + """Start a claim after proving ownership of this deployment. + + A remote caller must present the configured ``ENGRAPHIS_DEPLOYMENT_TOKEN`` as an + ownership proof. A loopback caller is already trusted (see + :func:`_local_deployment_token`), so on localhost the token and dashboard URL are + derived automatically and a self-hosted instance can trial with nothing configured. + """ + local = is_local_request(request) + configured = _configured_deployment_token() + if not configured and not local: + raise HTTPException(status_code=503, detail={ + "error": "ENGRAPHIS_DEPLOYMENT_TOKEN is not configured"}) + if configured and not local and not hmac.compare_digest( + configured, req.deployment_token or ""): + raise HTTPException(status_code=401, detail={"error": "invalid deployment token"}) + plan = req.plan.strip().lower() + if plan not in ("pro", "team"): + raise HTTPException(status_code=400, detail={"error": "plan must be pro or team"}) + token = configured or _local_deployment_token() + fallback = str(request.base_url).rstrip("/") if local else "" + return _start_bound_trial(req.email, plan, token, dashboard_url_fallback=fallback) + + +@router.get("/license/trials/{claim_id}") +def get_deployment_trial(claim_id: str, request: Request): + """Poll, retrieve, persist, and activate a confirmed claim without exposing its key.""" + token = _effective_deployment_token(request) + if not token: + raise HTTPException(status_code=503, detail={"error": "deployment token unavailable"}) + from engraphis import cloud_license + from engraphis.config import resolve_license_server_url + try: + result = cloud_license.claim_trial( + resolve_license_server_url(), claim_id, token, cloud_license.machine_id()) + except (RuntimeError, ValueError) as exc: + raise HTTPException(status_code=502, detail={"error": str(exc)}) + key = result.pop("key", None) + if key: + try: + license_public = licensing.activate(key).to_public_dict() + except licensing.LicenseError as exc: + raise HTTPException(status_code=502, detail={"error": str(exc)}) + result["license"] = license_public + result["active"] = license_public.get("plan") in ("pro", "team") + return result + + +@router.post("/ops/backup") +def run_customer_backup(): + """Run the configured off-volume backup; authentication is enforced by middleware.""" + try: + from engraphis.commercial import run_configured_backup + result = run_configured_backup() + except Exception as exc: # noqa: BLE001 - never expose storage paths or key detail + import logging + logging.getLogger("engraphis.backup").error( + "customer backup failed (%s)", type(exc).__name__) + raise HTTPException(status_code=503, detail={ + "ok": False, "verified": False}) + if not result["verified"]: + raise HTTPException(status_code=503, detail=result) + return result + + +@router.get("/ops/ready") +def customer_operations_ready(): + """Authenticated, boolean-only storage readiness for the managed customer service.""" + from engraphis.commercial import customer_operations_readiness + from fastapi.responses import JSONResponse + checks = customer_operations_readiness() + return JSONResponse(checks, status_code=200 if checks["ready"] else 503) # ── Cloud sync (Pro) — the dashboard's one-click "Sync now" button ──────────────────── @@ -1160,13 +1746,21 @@ def _relay_url() -> str: @router.get("/sync/status") def sync_status(): """Whether one-click cloud sync is ready, plus the last-sync summary for the button.""" + from engraphis.backends.sync_relay import has_sync_token, sync_read_only + has_token = has_sync_token() has_key = bool(licensing._read_key_material()) lic = licensing.current_license(refresh=False) return { # Ready only when the plan includes sync AND a key is configured. Purchased and # trial entitlements are both real server-issued keys. - "available": bool(licensing.has_feature("sync") and has_key), + "available": bool(has_token or (licensing.has_feature("sync") and has_key)), "has_key": has_key, + "has_user_token": has_token, + "read_only": sync_read_only(), + "token_managed_by_environment": bool( + os.environ.get("ENGRAPHIS_SYNC_TOKEN", "").strip()), + "read_only_managed_by_environment": bool( + os.environ.get("ENGRAPHIS_SYNC_READ_ONLY", "").strip()), "plan": lic.plan, "relay_url": _relay_url(), "tier_required": licensing.required_plan("sync"), @@ -1230,18 +1824,29 @@ def _sync_all(svc) -> dict: "shared relay (the folder could be marked personal)", }) continue - if raw_settings.get("visibility") == "personal": + visibility = raw_settings.get("visibility") + if visibility == "personal": + continue + if visibility not in (None, "", "shared"): + errors.append({ + "workspace": name, + "error": "workspace visibility is invalid; refusing to sync to the " + "shared relay", + }) continue try: transport = get_transport("relay", base_url=_relay_url(), workspace_id=name) - rep = syncer.sync(transport, row["id"]) + from engraphis.backends.sync_relay import sync_read_only + read_only = sync_read_only() + rep = syncer.sync(transport, row["id"], push=not read_only) except RelayError as exc: # Record the HTTP status (402 == relay rejected the key) instead of raising, so # one workspace can't abort the sweep; sync_run() promotes a 402 to the button. errors.append({"workspace": name, "error": str(exc), "status": exc.status}) continue except Exception as exc: # noqa: BLE001 — one bad workspace must not abort the rest - errors.append({"workspace": name, "error": str(exc)}) + logger.error("sync workspace failed (%s)", type(exc).__name__) + errors.append({"workspace": name, "error": "sync workspace failed"}) continue exported += int(rep.get("exported_memories", 0) or 0) for a in rep.get("applied") or []: @@ -1261,8 +1866,11 @@ def _sync_all(svc) -> dict: def sync_run(): """Push this device's memories to the relay and pull every other device's — for every workspace. Backs the dashboard 'Sync now' button. Pro/Team; needs a license key.""" - _paid("sync") # 402 if the plan doesn't include sync - if not licensing._read_key_material(): + from engraphis.backends.sync_relay import has_sync_token + has_token = has_sync_token() + if not has_token: + _paid("sync") # legacy paid-key migration path + if not has_token and not licensing._read_key_material(): raise HTTPException(status_code=402, detail={ "error": "Cloud sync needs your license key. Sign in with it above, then Sync.", "upgrade_url": licensing.upgrade_url()}) @@ -1283,6 +1891,56 @@ def sync_run(): return {"ok": True, "summary": summary} +class _SyncTokenReq(BaseModel): + token: str = Field(..., min_length=24, max_length=8192) + read_only: bool = False + + +@router.post("/sync/token") +def configure_sync_token(req: _SyncTokenReq): + from engraphis.backends.sync_relay import ( + save_sync_read_only, save_sync_token, sync_read_only) + env_token = os.environ.get("ENGRAPHIS_SYNC_TOKEN", "").strip() + if env_token and not hmac.compare_digest(env_token, req.token.strip()): + raise HTTPException(status_code=409, detail={ + "error": "sync token is managed by ENGRAPHIS_SYNC_TOKEN"}) + env_policy = os.environ.get("ENGRAPHIS_SYNC_READ_ONLY", "").strip() + if env_policy and sync_read_only() != bool(req.read_only): + raise HTTPException(status_code=409, detail={ + "error": "read-only policy is managed by ENGRAPHIS_SYNC_READ_ONLY"}) + try: + with _sync_token_state_lock: + # A partial update must fail toward no uploads. Persist a restrictive sentinel + # before replacing the token; relax it only after token persistence succeeds. + save_sync_read_only(True) + if not env_token: + save_sync_token(req.token) + if not req.read_only: + save_sync_read_only(False) + except ValueError as exc: + raise HTTPException(status_code=400, detail={"error": str(exc)}) + except OSError: + raise HTTPException(status_code=503, detail={ + "error": "sync token state could not be persisted"}) + return {"configured": True, "read_only": bool(req.read_only), + "token_managed_by_environment": bool(env_token), + "read_only_managed_by_environment": bool(env_policy)} + + +@router.delete("/sync/token") +def remove_sync_token(): + from engraphis.backends.sync_relay import clear_sync_token, has_sync_token, sync_read_only + try: + with _sync_token_state_lock: + clear_sync_token() + except OSError: + raise HTTPException(status_code=503, detail={ + "error": "sync token state could not be removed"}) + # An explicit deployment environment token cannot be removed by a dashboard file + # operation. Report the effective state instead of claiming it disappeared. + return {"configured": has_sync_token(), "read_only": sync_read_only()} + + class _AutoSyncReq(BaseModel): enabled: Optional[bool] = None # cadence (timer) sync cadence_minutes: Optional[int] = None @@ -1305,7 +1963,9 @@ def sync_auto_set(req: _AutoSyncReq): **admin-only** (``inspector/auth.min_role``): auto-sync is an account-wide control. The loop itself is licensed-gated too, so a stale toggle can never reach the relay after a plan lapses.""" - _paid("sync") + from engraphis.backends.sync_relay import has_sync_token + if not has_sync_token(): + _paid("sync") from engraphis import autosync cur = autosync.load_policy() merged = {k: (getattr(req, k) if getattr(req, k) is not None else cur.get(k)) diff --git a/engraphis/routes/v2_team.py b/engraphis/routes/v2_team.py index 0db9564..1fba6e5 100644 --- a/engraphis/routes/v2_team.py +++ b/engraphis/routes/v2_team.py @@ -21,10 +21,10 @@ from engraphis import licensing from engraphis.config import resolve_license_server_url, settings -from engraphis.netutil import client_ip +from engraphis.netutil import client_ip, is_local_request from engraphis.inspector.auth import ( - PBKDF2_ITERATIONS, SESSION_TTL_SECONDS, AccountLockedError, AuthError, AuthStore, - role_at_least) + API_TOKEN_SCOPES, PBKDF2_ITERATIONS, SESSION_TTL_SECONDS, AccountLockedError, + AuthError, AuthStore, role_at_least) logger = logging.getLogger("engraphis.team") @@ -43,82 +43,76 @@ def _csv_cell(value) -> str: return s -def _send_invite(u: dict, admin: dict) -> tuple: - """Best-effort onboarding notification for a newly added member. Returns - ``(invited, reason, pro_included)`` — never raises, so a delivery hiccup can - never fail the account-creation request that already succeeded. - - The email always points the member at the team dashboard to sign in. When this - instance is genuinely Team-licensed AND the new account can write, it ALSO carries - the shared Team license key (``pro_included``) so the member can activate their own - local Engraphis and get Pro features + cloud sync, taking one server-enforced seat — - that is what turns "you have a dashboard login" into "you are a licensed member of - the team". A ``viewer`` never receives the key (see below): for that role - ``pro_included`` is False by design, not by failure. - - Prefers THIS instance's own email delivery (``ENGRAPHIS_RESEND_API_KEY`` / - ``ENGRAPHIS_SMTP_*`` in its own env) when configured — the invite then comes - from the operator's own address/domain. Without local delivery configured, - falls back to the vendor relay (``/license/v1/team-invite``), gated by this - instance's own currently-active license key actually carrying the ``team`` - feature server-side; the relay echoes that same verified key into the email — - so self-hosters get a working, license-bearing "Add member" out of the box - without setting up their own mail account.""" +def _dashboard_base_url(request: Request) -> str: + """Return a canonical origin for credential-bearing email links. + + A remote request's ``Host`` header is attacker-controlled, so it must never become + the destination of a password-reset or invitation email. Hosted deployments must set + ``ENGRAPHIS_DASHBOARD_URL``; only a genuinely local, unproxied request may fall back + to its request origin for the zero-config loopback experience. + """ + configured = os.environ.get("ENGRAPHIS_DASHBOARD_URL", "").strip() + if configured: + from engraphis.cloud_license import validate_cloud_base_url + try: + return validate_cloud_base_url(configured).rstrip("/") + except ValueError as exc: + raise ValueError("ENGRAPHIS_DASHBOARD_URL is invalid") from exc + if not is_local_request(request): + raise ValueError("ENGRAPHIS_DASHBOARD_URL is required for remote email links") + + from urllib.parse import urlsplit, urlunsplit + parts = urlsplit(str(request.base_url).strip()) + if parts.scheme.lower() not in ("http", "https") or not parts.hostname: + raise ValueError("request origin is not an absolute HTTP URL") + try: + parts.port + except ValueError: + raise ValueError("request origin has an invalid port") from None + if parts.username is not None or parts.password is not None \ + or "\\" in parts.netloc or any(char.isspace() for char in parts.netloc): + raise ValueError("request origin contains an invalid authority") + return urlunsplit((parts.scheme.lower(), parts.netloc, + parts.path.rstrip("/"), "", "")) + + +def _send_invite(invitation: dict, admin: dict, request: Request) -> tuple: + """Deliver a one-time invitation URL without exposing the Team license key.""" if os.environ.get("ENGRAPHIS_TEAM_INVITES", "1").strip().lower() in ( "0", "false", "no", "off"): - return False, "team invite delivery is disabled", False + return False, "team invite delivery is disabled" + + try: + dashboard_url = _dashboard_base_url(request) + except ValueError as exc: + logger.error("team invite URL is unavailable (%s)", type(exc).__name__) + return False, "a trusted dashboard URL is not configured" + from urllib.parse import quote + # Keep the secret in the URL fragment: browsers never send fragments in HTTP + # requests, so Uvicorn/proxy access logs cannot capture the one-time credential. + invite_url = dashboard_url + "/#invite_token=" + quote(invitation["token"], safe="") from engraphis.inspector import webhooks - from engraphis.licensing import _read_key_material - - # Only embed a key when this instance really has the ``team`` feature, so we never - # email a member a free/absent or non-Team "key" that would not unlock anything. - # - # …and never for a ``viewer``. The shared Team key is NOT role-aware: the sync relay - # authenticates on the key alone (``inspector/sync_relay._authorize``) and knows - # nothing about dashboard roles, so any holder can push and delete bundles in the - # team's relay namespace. Mailing it to a read-only account would hand that account - # team-wide write access out of band of every check the dashboard makes — the - # dashboard refuses the viewer's every write while the relay accepts them all. The - # key follows the role: viewers get a dashboard-only invite. - can_hold_key = u.get("role") != "viewer" - team_key = (_read_key_material() - if (can_hold_key and licensing.has_feature("team")) else "") - dashboard_url = os.environ.get("ENGRAPHIS_DASHBOARD_URL", "").strip() - if webhooks.email_configured(): try: - webhooks.send_team_invite_email(u["email"], u["name"], u["role"], - invited_by=admin["email"], key=team_key, - dashboard_url=dashboard_url) - return True, "", bool(team_key) - except Exception as exc: # noqa: BLE001 — caller logs/audits, never raises further + webhooks.send_team_invite_email( + invitation["email"], invitation["name"], invitation["role"], + invited_by=admin["email"], invite_url=invite_url) + return True, "" + except Exception as exc: # noqa: BLE001 - audited by the caller logger.error("local team invite delivery failed (%s)", type(exc).__name__) - return False, "local email provider rejected delivery", False + return False, "local email provider rejected delivery" - from engraphis import cloud_license + from engraphis.licensing import _read_key_material key = _read_key_material() if not key: - return False, ("no local email delivery configured (ENGRAPHIS_RESEND_API_KEY / " - "ENGRAPHIS_SMTP_*) and no active license key to relay through"), False - # The relay accepts (and echoes into the email) only a key that verifies as Team - # server-side, so a successful relay send means a member got the activation key — - # viewers excepted, per the note below. - # Sync may target a customer-operated relay while trial issuance and mail delivery - # remain vendor services. Prefer the explicit/signed license-server URL so setting - # ENGRAPHIS_RELAY_URL to the customer's own deployment does not route invites back - # into a relay with no mail-provider credentials. - # - # ``key`` here is the caller's PROOF OF ENTITLEMENT to the relay, not (necessarily) - # the payload: the relay decides what to echo, and it withholds the key for a - # ``viewer`` for exactly the reason above (``license_cloud.team_invite``). So a - # viewer's relay-delivered invite is dashboard-only too, and ``pro_included`` must - # report that rather than assuming "sent == key delivered". - sent, reason = cloud_license.send_team_invite( + return False, ("no local email delivery configured and no active Team license " + "available for vendor-relayed delivery") + from engraphis import cloud_license + return cloud_license.send_team_invite( resolve_license_server_url(licensing.current_license().cloud_url), key, - u["email"], u["name"], u["role"], admin["email"], - dashboard_url=dashboard_url) - return sent, reason, bool(sent and can_hold_key) + invitation["email"], invitation["name"], invitation["role"], admin["email"], + invite_url=invite_url) class SetupReq(BaseModel): @@ -144,10 +138,21 @@ class ResetReq(BaseModel): class NewUserReq(BaseModel): email: str = Field(..., min_length=5, max_length=254) name: str = Field(default="", max_length=120) - password: str = Field(..., min_length=10, max_length=128) + password: Optional[str] = Field(default=None, min_length=10, max_length=128) + role: str = Field(default="member", pattern=r'^(viewer|member|admin)$') + + +class InvitationReq(BaseModel): + email: str = Field(..., min_length=5, max_length=254) + name: str = Field(default="", max_length=120) role: str = Field(default="member", pattern=r'^(viewer|member|admin)$') +class InvitationAcceptReq(BaseModel): + token: str = Field(..., min_length=10, max_length=256) + password: str = Field(..., min_length=10, max_length=128) + + class UpdUserReq(BaseModel): user_id: str role: Optional[str] = Field(default=None, pattern=r'^(viewer|member|admin)$') @@ -161,6 +166,7 @@ class DelUserReq(BaseModel): class TokenReq(BaseModel): label: str = Field(default="", max_length=120, description="A memorable name for this agent token (e.g. 'claude-code-laptop').") + scopes: Optional[list[str]] = Field(default=None, max_length=len(API_TOKEN_SCOPES)) def _enabled() -> bool: @@ -236,6 +242,7 @@ def state_off(): # never becomes public. Paid operations and seat growth continue to enforce the # live entitlement; adding seats beyond the first admin still requires Team. store = AuthStore(_users_db_path(settings.db_path), iterations=_auth_iterations()) + app.state.auth_store = store def _user(request: Request) -> Optional[dict]: tok = request.cookies.get(_COOKIE) @@ -313,20 +320,40 @@ def forgot(body: ForgotReq, request: Request): a failure is logged server-side and never surfaced to the caller, for the same anti-enumeration reason (see send_password_reset_email's docstring). """ + try: + base = _dashboard_base_url(request) + except ValueError as exc: + # Do not issue (and thereby invalidate) a reset token that cannot be sent. + # The public response remains identical to the unknown-user path. + logger.warning("password reset URL is unavailable (%s)", type(exc).__name__) + return {"ok": True} try: info = store.request_password_reset(body.email) except AuthError: info = None if info: - base = os.environ.get("ENGRAPHIS_DASHBOARD_URL", "").strip().rstrip("/") - reset_url = base + "/?reset_token=" + info["token"] try: - from engraphis.inspector.webhooks import send_password_reset_email - send_password_reset_email(info["email"], info["name"], reset_url) + reset_url = base + "/#reset_token=" + info["token"] + from engraphis.inspector import webhooks + if webhooks.email_configured(): + webhooks.send_password_reset_email( + info["email"], info["name"], reset_url) + else: + from engraphis import cloud_license + from engraphis.licensing import _read_key_material + key = _read_key_material() + if not key: + raise RuntimeError("no reset-email delivery credential") + queued, reason = cloud_license.send_password_reset( + resolve_license_server_url(licensing.current_license().cloud_url), + key, info["email"], info["name"], reset_url, + ) + if not queued: + raise RuntimeError(reason or "reset-email relay rejected delivery") except Exception as exc: # noqa: BLE001 — must never change the response - logger.warning( - "password reset email to %s failed (%s)", - info["email"], type(exc).__name__) + # Recipient and reset token are deliberately absent from logs. + logger.warning("password reset delivery failed (%s)", + type(exc).__name__) return {"ok": True} @router.post("/reset") @@ -349,6 +376,89 @@ def logout(request: Request, response: Response): response.delete_cookie(_COOKIE, path="/") return {"ok": True} + def _create_and_send_invitation(body, request: Request) -> dict: + admin = _require(request, "admin") + try: + invitation = store.create_invitation( + body.email, body.name, body.role, created_by=admin["id"], + seat_limit=licensing.current_license().seats) + except AuthError as exc: + raise HTTPException(status_code=400, detail={"error": str(exc)}) + sent, reason = _send_invite(invitation, admin, request) + store.set_invitation_delivery( + invitation["id"], "sent" if sent else "failed", reason) + ip = client_ip(request) + store.record_event( + "user.invited", actor_id=admin["id"], actor_email=admin["email"], + target=invitation["email"], detail="role=%s" % invitation["role"], ip=ip) + if not sent: + store.record_event( + "user.invite_email_failed", actor_id=admin["id"], + actor_email=admin["email"], target=invitation["email"], + detail=reason[:200], ip=ip) + public = dict(invitation) + public.pop("token", None) + public["delivery_state"] = "sent" if sent else "failed" + public["last_delivery_error"] = "" if sent else reason[:200] + return {"invitation": public, "invited": sent} + + @router.post("/invitations/accept") + def accept_invitation(body: InvitationAcceptReq, request: Request, response: Response): + try: + # Re-read the entitlement at acceptance time: an invitation may have been + # issued under a larger Team plan and must not oversubscribe a downgrade. + accepted = store.accept_invitation( + body.token, body.password, + seat_limit=licensing.current_license().seats, + ) + store.record_event( + "user.invitation_accepted", actor_id=accepted["id"], + actor_email=accepted["email"], target=accepted["email"], + detail="role=%s" % accepted["role"], ip=client_ip(request)) + # The one-time invitation token has already authenticated this recipient. + # Mint the session directly instead of feeding the new password through the + # public login throttle: an attacker must not be able to consume a valid + # invitation and then strand its recipient behind an IP lockout. + session_token = store.create_session(accepted["id"]) + except AuthError as exc: + raise HTTPException(status_code=400, detail={"error": str(exc)}) + _set_cookie(response, session_token, secure=_cookie_secure(request)) + return {"user": accepted} + + @router.post("/invitations") + def create_invitation(body: InvitationReq, request: Request): + return _create_and_send_invitation(body, request) + + @router.get("/invitations") + def invitations(request: Request): + _require(request, "admin") + return {"invitations": store.list_invitations()} + + @router.post("/invitations/{invite_id}/resend") + def resend_invitation(invite_id: str, request: Request): + admin = _require(request, "admin") + try: + invitation = store.resend_invitation(invite_id) + except AuthError as exc: + raise HTTPException(status_code=400, detail={"error": str(exc)}) + sent, reason = _send_invite(invitation, admin, request) + store.set_invitation_delivery(invite_id, "sent" if sent else "failed", reason) + store.record_event( + "user.invitation_resent", actor_id=admin["id"], actor_email=admin["email"], + target=invitation["email"], detail="sent=%s" % sent, ip=client_ip(request)) + return {"ok": sent, "delivery_state": "sent" if sent else "failed", + "error": "" if sent else reason} + + @router.delete("/invitations/{invite_id}") + def revoke_invitation(invite_id: str, request: Request): + admin = _require(request, "admin") + if not store.revoke_invitation(invite_id): + raise HTTPException(status_code=404, detail={"error": "invitation not found"}) + store.record_event( + "user.invitation_revoked", actor_id=admin["id"], + actor_email=admin["email"], target=invite_id, ip=client_ip(request)) + return {"ok": True} + @router.get("/users") def users(request: Request): # "admin", not "member": auth.min_role() maps every /api/auth/users* path to admin @@ -362,44 +472,14 @@ def users(request: Request): @router.post("/users") def add_user(body: NewUserReq, request: Request): - admin = _require(request, "admin") - seats = licensing.current_license().seats - try: - u = store.create_user(body.email, body.name, body.password, body.role, - seat_limit=seats) - except AuthError as exc: - raise HTTPException(status_code=400, detail={"error": str(exc)}) - ip = client_ip(request) - store.record_event("user.created", actor_id=admin["id"], actor_email=admin["email"], - target=u["email"], detail="role=%s" % body.role, ip=ip) - invited, fail_reason, pro_included = _send_invite(u, admin) - dashboard_configured = bool(os.environ.get("ENGRAPHIS_DASHBOARD_URL", "").strip()) - # A viewer is deliberately never mailed the shared Team key (see _send_invite), so - # "this invite carried no key" is the correct outcome for that role. Warning the - # admin about it would report a working, intentional policy as a licensing - # misconfiguration and push them to "fix" it. - key_expected = u["role"] != "viewer" - if not invited: - logger.warning("team invite email to %s failed: %s", u["email"], fail_reason) - store.record_event("user.invite_email_failed", actor_id=admin["id"], - actor_email=admin["email"], target=u["email"], - detail=fail_reason[:200], ip=ip) - elif not pro_included and key_expected: - # Delivered, but the member got a dashboard-only invite with no Pro/team key, - # because this instance is not Team-licensed. Tell the admin plainly rather - # than letting the invite look like it silently did nothing. - logger.warning("team invite to %s carried no Pro activation key: this instance " - "is not Team-licensed", u["email"]) - store.record_event("user.invite_no_license", actor_id=admin["id"], - actor_email=admin["email"], target=u["email"], - detail="dashboard access emailed, but no team license key " - "(instance not Team-licensed)", ip=ip) - return {"user": u, "invited": invited, - "pro_activation_sent": bool(pro_included), - # False for a viewer: no license key is expected in that invite, so a UI - # can distinguish "we couldn't send the key" from "this role never gets one". - "pro_activation_expected": key_expected, - "dashboard_url_configured": dashboard_configured} + # v1.0 compatibility alias. Password-bearing account creation is intentionally + # rejected; callers must send only email/name/role and let the recipient choose + # the password through the one-time invitation. + if body.password is not None: + raise HTTPException(status_code=400, detail={ + "error": "temporary passwords are no longer accepted; create an invitation" + }) + return _create_and_send_invitation(body, request) @router.post("/users/update") def upd_user(body: UpdUserReq, request: Request): @@ -494,7 +574,19 @@ def overview(request: Request): @router.post("/token") def create_token(body: TokenReq, request: Request): u = _require(request, "viewer") - row = store.create_api_token(u["id"], label=body.label) + allowed = {"agent", "sync:read"} + if role_at_least(u["role"], "member"): + allowed.add("sync:write") + requested = set(allowed if body.scopes is None else body.scopes) + if not requested: + raise HTTPException(status_code=400, detail={ + "error": "at least one token scope is required"}) + if not requested.issubset(allowed): + raise HTTPException(status_code=403, detail={"error": "scope not allowed for role"}) + try: + row = store.create_api_token(u["id"], label=body.label, scopes=requested) + except AuthError as exc: + raise HTTPException(status_code=400, detail={"error": str(exc)}) ip = client_ip(request) store.record_event("api_token.created", actor_id=u["id"], actor_email=u["email"], detail=row["label"] or "(unlabelled)", ip=ip) diff --git a/engraphis/service.py b/engraphis/service.py index a5112dd..2cf1556 100644 --- a/engraphis/service.py +++ b/engraphis/service.py @@ -21,13 +21,26 @@ import hashlib import contextvars import math +import copy +import time +import threading +from collections import Counter, OrderedDict +from functools import wraps from pathlib import Path from typing import Any, Optional from engraphis.backends.extractor import ChunkingExtractor from engraphis.core.engine import MemoryEngine +from engraphis.core.graph_scene import ( + build_canonical_graph, + build_graph_scene, + is_broad_search_fragment, + strongest_path, +) from engraphis.core.graph_layers import normalize_graph_layer +from engraphis.core.ids import new_id as make_id from engraphis.core.interfaces import Edge, GraphLayer, MemoryType, Node, Scope, SearchFilter +from engraphis.core.store import normalize_entity_name from engraphis.graphdata import build_graph_payload, empty_graph # ── validation limits (memory-poisoning / resource-exhaustion guards) ────────── @@ -47,6 +60,31 @@ MAX_IMPORT_FILE_BYTES = 2_000_000 MAX_IMPORT_RESOURCE_BYTES = 100_000_000 MAX_IMPORT_TOTAL_BYTES = 250_000_000 +# Analytical graph scenes rank the candidate graph before applying the much smaller +# browser scene budget. Keep that server-side candidate set finite as well: graph rows +# are user/sync writable, and an unbounded Louvain/PageRank request would otherwise be a +# straightforward authenticated resource-exhaustion path. +MAX_GRAPH_ANALYSIS_ENTITIES = 20_000 +MAX_GRAPH_ANALYSIS_EDGES = 100_000 +MAX_GRAPH_ANALYSIS_SUPPORTS = 250_000 +# Complete scenes are intentionally not representative samples. These are hard +# refusal ceilings, not render caps: callers receive an explicit capacity error rather +# than a silently incomplete chart. +MAX_GRAPH_COMPLETE_MEMORIES = 50_000 +MAX_GRAPH_COMPLETE_MEMORY_LINKS = 150_000 +MAX_GRAPH_COMPLETE_CODE_MEMORY_LINKS = 150_000 +MAX_GRAPH_COMPLETE_PAYLOAD_BYTES = 64 * 1024 * 1024 +MAX_GRAPH_INDEX_MEMORIES = 20_000 +MAX_GRAPH_INDEX_WORKERS = 2 +GRAPH_INDEX_BATCH_SIZE = 100 +GRAPH_INDEX_LEASE_SECONDS = 60.0 +GRAPH_INDEX_JOB_HISTORY = 100 +# Inspector payloads are deliberately smaller than analysis payloads. The endpoint +# reports complete counts, but bounds the returned detail so selecting a hub cannot +# produce a multi-megabyte response or lock the inspector's DOM. +GRAPH_ENTITY_RELATION_LIMIT = 200 +GRAPH_ENTITY_EVIDENCE_LIMIT = 100 +GRAPH_ENTITY_HISTORY_LIMIT = 50 # control characters except tab/newline/carriage-return _CONTROL_RE = re.compile(r"[\x00-\x08\x0b\x0c\x0e-\x1f\x7f]") @@ -57,6 +95,54 @@ class ValidationError(ValueError): """Raised when untrusted input fails a guard. Message is safe to surface.""" +class GraphSceneCapacityExceeded(ValidationError): + """A complete scene crossed a hard safety ceiling and was not sampled.""" + + def __init__(self, *, resource: str, count: int, limit: int) -> None: + self.resource = resource + self.count = int(count) + self.limit = int(limit) + super().__init__( + f"complete graph exceeds the {resource} safety limit " + f"({self.count} > {self.limit}); narrow the workspace filters" + ) + + +def _rollback_service_transaction(method): + """Roll back a failed multi-statement service mutation on the shared connection. + + The serialized SQLite wrapper pins its write lock until commit or rollback. Workspace + lifecycle operations contain many dependent statements, so an unexpected storage or + constraint failure must release both the partial transaction and that lock. + """ + @wraps(method) + def wrapped(self, *args, **kwargs): + started = not self.store.conn.in_transaction + try: + if started: + self.store.conn.execute("BEGIN IMMEDIATE") + result = method(self, *args, **kwargs) + if started and self.store.conn.in_transaction: + self.store.conn.commit() + return result + except BaseException: + if self.store.conn.in_transaction: + try: + self.store.conn.rollback() + except Exception: # noqa: BLE001 - preserve the original failure + pass + raise + return wrapped + + +class GraphIndexRebuilding(ValidationError): + """Raised when a graph read would observe a partially rebuilt derived index.""" + + def __init__(self, job_id: str) -> None: + self.job_id = job_id + super().__init__(f"graph index rebuilding (job {job_id})") + + # ── current dashboard user (request-scoped, team mode only) ──────────────────── # Set by the dashboard's team auth gate (engraphis/dashboard_app.py::_auth_gate) for the # duration of a request, and read at the workspace-authorization chokepoint below so a @@ -350,6 +436,43 @@ def __init__(self, engine: MemoryEngine, *, # ``graph()``. Guards against rescanning a workspace whose memories genuinely # yield no entities on every Graph-tab open. self._graph_backfilled: set = set() + # Graph scenes are expensive derived views over the full canonical graph. Cache + # a small number per service instance, keyed by both request parameters and the + # SQLite connection revision. ``total_changes`` catches writes performed through + # this connection; ``data_version`` catches commits from another connection. + # Consequently a memory/entity/edge mutation invalidates cached scenes without a + # stale TTL window, while repeated pan/filter/navigation reads stay comfortably + # inside the dashboard's warm-response budget. Each value also carries the next + # bi-temporal validity boundary: time passing can change a current-time scene even + # when no connection writes, so such an entry expires exactly at that boundary. + self._graph_scene_cache: "OrderedDict[tuple, tuple[float, dict]]" = OrderedDict() + self._graph_job_lock = threading.RLock() + self._graph_job_threads: dict[str, threading.Thread] = {} + self._graph_runner_id = make_id("device") + + def _graph_scene_revision(self) -> tuple[int, int, int]: + row = self.store.conn.execute("PRAGMA data_version").fetchone() + data_version = int(row[0]) if row is not None else 0 + return (int(self.store.conn.total_changes), data_version, + int(self.store.schema_version)) + + def _graph_scene_valid_until(self, workspace_id: str, at: float) -> float: + """Earliest future world-time boundary that can change a current graph scene.""" + row = self.store.conn.execute( + "SELECT MIN(boundary) FROM (" + "SELECT valid_from AS boundary FROM edges " + "WHERE workspace_id=? AND valid_from>? AND expired_at IS NULL " + "UNION ALL SELECT valid_to FROM edges " + "WHERE workspace_id=? AND valid_to>? AND expired_at IS NULL " + "UNION ALL SELECT s.valid_from FROM edge_supports s " + "JOIN edges e ON e.id=s.edge_id WHERE e.workspace_id=? " + "AND s.valid_from>? AND s.expired_at IS NULL " + "UNION ALL SELECT s.valid_to FROM edge_supports s " + "JOIN edges e ON e.id=s.edge_id WHERE e.workspace_id=? " + "AND s.valid_to>? AND s.expired_at IS NULL)", + (workspace_id, at, workspace_id, at, workspace_id, at, workspace_id, at), + ).fetchone() + return float(row[0]) if row is not None and row[0] is not None else math.inf @classmethod def create(cls, db_path: str = ":memory:", *, embed_model: Optional[str] = None, @@ -398,7 +521,9 @@ def _lookup_workspace(self, name: str) -> Optional[str]: def _lookup_repo(self, workspace_id: str, name: str) -> Optional[str]: row = self.store.conn.execute( - "SELECT id FROM repos WHERE workspace_id=? AND name=?", (workspace_id, name) + "SELECT id FROM repos WHERE workspace_id=? AND (name=? OR id=?) " + "ORDER BY CASE WHEN id=? THEN 0 ELSE 1 END LIMIT 1", + (workspace_id, name, name, name), ).fetchone() return row["id"] if row else None @@ -1926,6 +2051,7 @@ def set_workspace_description(self, workspace: str, description: str, self.store.conn.commit() return {"workspace": ws, "description": description} + @_rollback_service_transaction def delete_workspace(self, workspace: str, *, actor: str = "user") -> dict: """HARD-delete a workspace and everything scoped to it (memories, vectors, FTS rows, entities/edges, sessions, events, repos + their code graph). Unlike ``forget`` this is @@ -1935,11 +2061,30 @@ def delete_workspace(self, workspace: str, *, actor: str = "user") -> dict: wid = self._lookup_workspace(ws) if wid is None: raise ValidationError(f"no workspace named '{ws}' yet") + self._assert_no_active_graph_job(wid) c = self.store.conn - n_mem = c.execute("SELECT COUNT(*) AS n FROM memories WHERE workspace_id=?", (wid,)).fetchone()["n"] + memory_ids = [row["id"] for row in c.execute( + "SELECT id FROM memories WHERE workspace_id=?", (wid,) + ).fetchall()] + n_mem = len(memory_ids) msub = "(SELECT id FROM memories WHERE workspace_id=?)" rsub = "(SELECT id FROM repos WHERE workspace_id=?)" ssub = f"(SELECT id FROM symbols WHERE repo_id IN {rsub})" + # Retire each memory's evidence before the hard delete. This removes the + # memory from any global/legacy edge it supported and closes an edge whose last + # source is disappearing. Then delete the normalized evidence rows themselves: + # hard deletion must not leave orphaned provenance behind. + for memory_id in memory_ids: + self.store.invalidate_edges_for_memory(memory_id, commit=False) + c.execute( + "DELETE FROM edge_supports WHERE memory_id IN " + msub, + (wid,), + ) + c.execute( + "DELETE FROM edge_supports WHERE edge_id IN " + "(SELECT id FROM edges WHERE workspace_id=?)", + (wid,), + ) c.execute( f"DELETE FROM code_memory_links WHERE repo_id IN {rsub} " f"OR memory_id IN {msub} OR symbol_id IN {ssub}", @@ -1961,11 +2106,15 @@ def delete_workspace(self, workspace: str, *, actor: str = "user") -> dict: c.execute(f"DELETE FROM code_edges WHERE repo_id IN {rsub}", (wid,)) c.execute(f"DELETE FROM symbols WHERE repo_id IN {rsub}", (wid,)) c.execute("DELETE FROM repos WHERE workspace_id=?", (wid,)) + c.execute("DELETE FROM jobs WHERE workspace_id=?", (wid,)) + # Entity/edge delete triggers may have recreated this generation row. + c.execute("DELETE FROM graph_index_state WHERE workspace_id=?", (wid,)) c.execute("DELETE FROM workspaces WHERE id=?", (wid,)) self.store.audit(actor, "workspace_delete", wid, f"{ws} ({int(n_mem)} memories)") c.commit() return {"workspace": ws, "deleted": True, "memories_removed": int(n_mem)} + @_rollback_service_transaction def merge_workspaces(self, source: str, target: str, *, actor: str = "user") -> dict: """Fold ``source`` into ``target``, then remove the now-empty ``source`` workspace. This is the workspace-level counterpart to ``merge`` — and the @@ -1988,6 +2137,7 @@ def merge_workspaces(self, source: str, target: str, *, actor: str = "user") -> raise ValidationError(f"no workspace named '{src}' yet") if wid_dst is None: raise ValidationError(f"no workspace named '{dst}' yet") + self._assert_no_active_graph_job(wid_src, wid_dst) c = self.store.conn n_mem = c.execute("SELECT COUNT(*) AS n FROM memories WHERE workspace_id=?", (wid_src,)).fetchone()["n"] @@ -2105,19 +2255,38 @@ def _new_repo(old_repo_id): # 2) Entities: fold same name+type+repo together, else relabel. entity_remap: dict = {} src_entities = [dict(x) for x in c.execute( - "SELECT id, repo_id, name, etype FROM entities WHERE workspace_id=?", (wid_src,))] + "SELECT id, repo_id, name, etype, canonical_id, normalized_name " + "FROM entities WHERE workspace_id=?", (wid_src,))] for e in src_entities: nrid = _new_repo(e["repo_id"]) + normalized = e.get("normalized_name") or normalize_entity_name(e["name"]) existing = c.execute( - "SELECT id FROM entities WHERE workspace_id=? AND repo_id IS ? AND name=? AND etype IS ?", - (wid_dst, nrid, e["name"], e["etype"]) + "SELECT id, canonical_id FROM entities WHERE workspace_id=? AND repo_id IS ? " + "AND normalized_name=? AND etype IS ? ORDER BY id LIMIT 1", + (wid_dst, nrid, normalized, e["etype"]) ).fetchone() if existing: entity_remap[e["id"]] = existing["id"] c.execute("DELETE FROM entities WHERE id=?", (e["id"],)) else: - c.execute("UPDATE entities SET workspace_id=?, repo_id=? WHERE id=?", - (wid_dst, nrid, e["id"])) + canonical = c.execute( + "SELECT COALESCE(canonical_id, id) AS canonical_id FROM entities " + "WHERE workspace_id=? AND normalized_name=? AND etype IS ? " + "ORDER BY id LIMIT 1", + (wid_dst, normalized, e["etype"]), + ).fetchone() + c.execute( + "UPDATE entities SET workspace_id=?, repo_id=?, normalized_name=?, " + "canonical_id=?, canonical_method=? WHERE id=?", + (wid_dst, nrid, normalized, + canonical["canonical_id"] if canonical else (e["canonical_id"] or e["id"]), + "exact_normalized" if canonical else "exact", e["id"]), + ) + for old_id, new_id in entity_remap.items(): + c.execute( + "UPDATE entities SET canonical_id=? WHERE workspace_id=? AND canonical_id=?", + (new_id, wid_dst, old_id), + ) # 3) Edges: relabel workspace/repo, remapping any entity ids folded in step 2. src_edges = [dict(x) for x in c.execute( @@ -2131,7 +2300,7 @@ def _new_repo(old_repo_id): # 4) Memories / sessions / events: relabel workspace/repo per distinct repo_id # bucket (ids, content and history are untouched). - for table in ("memories", "sessions", "events"): + for table in ("memories", "sessions", "events", "jobs"): buckets = [dict(x) for x in c.execute( f"SELECT DISTINCT repo_id FROM {table} WHERE workspace_id=?", (wid_src,))] for b in buckets: @@ -2141,6 +2310,7 @@ def _new_repo(old_repo_id): (wid_dst, _new_repo(b["repo_id"]), wid_src, b["repo_id"])) # 5) The source workspace is now empty — drop it. + c.execute("DELETE FROM graph_index_state WHERE workspace_id=?", (wid_src,)) c.execute("DELETE FROM workspaces WHERE id=?", (wid_src,)) self.store.audit(actor, "workspace_merge", wid_dst, f"{src} ({int(n_mem)} memories) -> {dst}") c.commit() @@ -2160,6 +2330,7 @@ def _next_copy_name(self, base: str) -> str: return candidate n += 1 + @_rollback_service_transaction def copy_workspace(self, source: str, new_name: Optional[str] = None, *, actor: str = "user") -> dict: """Duplicate ``source`` into a brand-new workspace: repos (+ their code graph), @@ -2175,6 +2346,7 @@ def copy_workspace(self, source: str, new_name: Optional[str] = None, *, wid_src = self._lookup_workspace(src) if wid_src is None: raise ValidationError(f"no workspace named '{src}' yet") + self._assert_no_active_graph_job(wid_src) if new_name: dst = _clean_name(new_name, field="new_name") if self._lookup_workspace(dst) is not None: @@ -2244,25 +2416,42 @@ def _new_repo(old_repo_id): return repo_remap.get(old_repo_id, old_repo_id) if old_repo_id is not None else None # 2) Entities, cloned with fresh ids. - entity_remap: dict = {} - for e in [dict(x) for x in c.execute( - "SELECT * FROM entities WHERE workspace_id=?", (wid_src,))]: - neid = ids.new_id("entity") - entity_remap[e["id"]] = neid + source_entities = [dict(x) for x in c.execute( + "SELECT * FROM entities WHERE workspace_id=?", (wid_src,) + )] + entity_remap: dict = { + entity["id"]: ids.new_id("entity") for entity in source_entities + } + for e in source_entities: + neid = entity_remap[e["id"]] + old_canonical_id = e.get("canonical_id") + canonical_id = entity_remap.get(old_canonical_id, neid) + canonical_method = ( + (e.get("canonical_method") or "identity") + if old_canonical_id in entity_remap else "identity" + ) c.execute( "INSERT INTO entities(id, workspace_id, repo_id, name, etype, canonical_id, " - "created_at) VALUES (?,?,?,?,?,?,?)", + "normalized_name, canonical_method, canonical_confidence, created_at) " + "VALUES (?,?,?,?,?,?,?,?,?,?)", (neid, wid_dst, _new_repo(e["repo_id"]), e["name"], e["etype"], - e["canonical_id"], ts)) + canonical_id, e.get("normalized_name") or normalize_entity_name(e["name"]), + canonical_method, + e.get("canonical_confidence") or 1.0, ts)) # 3) Entity-graph edges, remapped onto the cloned entities/repos. - for ed in [dict(x) for x in c.execute( - "SELECT * FROM edges WHERE workspace_id=?", (wid_src,))]: + source_edges = [dict(x) for x in c.execute( + "SELECT * FROM edges WHERE workspace_id=?", (wid_src,) + )] + edge_remap: dict = {} + for ed in source_edges: + new_edge_id = ids.new_id("edge") + edge_remap[ed["id"]] = new_edge_id c.execute( "INSERT INTO edges(id, workspace_id, repo_id, src, dst, relation, layer, " "weight, valid_from, valid_to, ingested_at, expired_at, provenance) " "VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?)", - (ids.new_id("edge"), wid_dst, _new_repo(ed["repo_id"]), + (new_edge_id, wid_dst, _new_repo(ed["repo_id"]), entity_remap.get(ed["src"], ed["src"]), entity_remap.get(ed["dst"], ed["dst"]), ed["relation"], ed["layer"] or "semantic", ed["weight"], ed["valid_from"], ed["valid_to"], ed["ingested_at"], @@ -2284,11 +2473,48 @@ def _new_repo(old_repo_id): # 5) Memories, cloned with fresh ids — plus their full-text and vector mirrors, # which key off the memory id and so need the same new id. - memory_remap: dict = {} - for m in [dict(x) for x in c.execute( - "SELECT * FROM memories WHERE workspace_id=?", (wid_src,))]: - nmid = ids.new_id("memory") - memory_remap[m["id"]] = nmid + source_memories = [dict(x) for x in c.execute( + "SELECT * FROM memories WHERE workspace_id=?", (wid_src,) + )] + memory_remap = { + memory["id"]: ids.new_id("memory") for memory in source_memories + } + + def _remap_json_memory_ids(raw): + try: + value = json.loads(raw or "{}") + except (TypeError, ValueError): + return raw + + def walk(item): + if isinstance(item, dict): + remapped = {} + for key, child in item.items(): + if key in ("memory_id", "corrects"): + replacement = memory_remap.get(str(child or "")) + if replacement: + remapped[key] = replacement + continue + if key in ("memory_ids", "supersedes") and isinstance(child, list): + replacements = [ + memory_remap[str(old)] for old in child + if str(old) in memory_remap + ] + if replacements: + remapped[key] = list(dict.fromkeys(replacements)) + continue + remapped[key] = walk(child) + return remapped + if isinstance(item, list): + return [walk(child) for child in item] + if isinstance(item, str): + return memory_remap.get(item, item) + return item + + return json.dumps(walk(value), ensure_ascii=False, separators=(",", ":")) + + for m in source_memories: + nmid = memory_remap[m["id"]] c.execute( "INSERT INTO memories (id, workspace_id, repo_id, session_id, scope, mtype, " "title, content, summary, keywords, metadata, importance, surprise, stability, " @@ -2297,10 +2523,11 @@ def _new_repo(old_repo_id): "VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)", (nmid, wid_dst, _new_repo(m["repo_id"]), session_remap.get(m["session_id"]), m["scope"], m["mtype"], m["title"], m["content"], m["summary"], m["keywords"], - m["metadata"], m["importance"], m["surprise"], m["stability"], + _remap_json_memory_ids(m["metadata"]), m["importance"], + m["surprise"], m["stability"], m["access_count"], m["last_access"], m["valid_from"], m["valid_to"], m["ingested_at"], m["expired_at"], m["pinned"], m["sensitivity"], - m["provenance"], m["sort_order"])) + _remap_json_memory_ids(m["provenance"]), m["sort_order"])) fts_row = c.execute( "SELECT title, content, keywords FROM mem_fts WHERE id=?", (m["id"],)).fetchone() if fts_row: @@ -2322,6 +2549,50 @@ def _new_repo(old_repo_id): # 6) Cross-memory links where *both* endpoints were copied — a link to a memory # outside this workspace can't be meaningfully cloned, so those are dropped. + # Remap legacy provenance and normalized evidence only after memory ids exist. + # Opaque canonical/support ids never cross the workspace boundary unchanged. + for source_edge in source_edges: + new_edge_id = edge_remap[source_edge["id"]] + edge_provenance = _remap_json_memory_ids( + source_edge.get("provenance") or "{}" + ) + c.execute( + "UPDATE edges SET provenance=? WHERE id=?", + (edge_provenance, new_edge_id), + ) + source_supports = [dict(row) for row in c.execute( + "SELECT * FROM edge_supports WHERE edge_id=? ORDER BY id", + (source_edge["id"],), + )] + for support in source_supports: + new_memory_id = memory_remap.get(support["memory_id"]) + if new_memory_id is None: + continue + support_provenance = _remap_json_memory_ids( + support.get("provenance") or "{}" + ) + c.execute( + "INSERT INTO edge_supports(edge_id, memory_id, source_kind, confidence, " + "valid_from, valid_to, ingested_at, expired_at, provenance) " + "VALUES (?,?,?,?,?,?,?,?,?)", + (new_edge_id, new_memory_id, support["source_kind"], + support["confidence"], support["valid_from"], support["valid_to"], + support["ingested_at"], support["expired_at"], support_provenance), + ) + if not source_supports: + try: + fallback_provenance = json.loads(edge_provenance or "{}") + except (TypeError, ValueError): + fallback_provenance = {} + if isinstance(fallback_provenance, dict): + self.store._write_edge_supports( + new_edge_id, source_edge["relation"], fallback_provenance, + valid_from=source_edge["valid_from"], + valid_to=source_edge["valid_to"], + ingested_at=source_edge["ingested_at"], + expired_at=source_edge["expired_at"], + ) + if memory_remap: old_ids = list(memory_remap.keys()) marks = ",".join("?" for _ in old_ids) @@ -2366,7 +2637,8 @@ def _new_repo(old_repo_id): "INSERT INTO events(id, workspace_id, repo_id, session_id, kind, content, refs, " "interaction_level, ts) VALUES (?,?,?,?,?,?,?,?,?)", (ids.new_id("event"), wid_dst, _new_repo(ev["repo_id"]), - session_remap.get(ev["session_id"]), ev["kind"], ev["content"], ev["refs"], + session_remap.get(ev["session_id"]), ev["kind"], ev["content"], + _remap_json_memory_ids(ev["refs"]), ev["interaction_level"], ev["ts"])) self.store.audit(actor, "workspace_copy", wid_dst, @@ -2615,6 +2887,1705 @@ def export_workspace(self, *, workspace: str) -> dict: "memories": memories, "sessions": sessions, "audit": audit, "receipts": receipts} + def _recover_stale_graph_jobs(self, workspace_id: Optional[str] = None) -> int: + """Fail expired process-local workers and release their rebuilding gate. + + Jobs are persisted but Python threads are not. A process crash must therefore + become a bounded interruption rather than leaving graph reads and all future + jobs blocked forever. The heartbeat lease also keeps this safe when separate + service processes share the database. + """ + now = time.time() + cutoff = now - GRAPH_INDEX_LEASE_SECONDS + where = "state IN ('queued','running') AND COALESCE(heartbeat_at, created_at) None: + for workspace_id in dict.fromkeys(value for value in workspace_ids if value): + self._recover_stale_graph_jobs(workspace_id) + row = self.store.conn.execute( + "SELECT id FROM jobs WHERE workspace_id=? AND kind='graph_index' " + "AND state IN ('queued','running') LIMIT 1", + (workspace_id,), + ).fetchone() + if row is not None: + raise ValidationError( + f"workspace graph index job '{row['id']}' is still active" + ) + + def _graph_index_info(self, workspace_id: str) -> dict: + row = self.store.conn.execute( + "SELECT generation, state, active_job_id, updated_at, last_error " + "FROM graph_index_state WHERE workspace_id=?", + (workspace_id,), + ).fetchone() + if row is None: + return { + "generation": self.store.schema_version, + "state": "ready", + "active_job_id": None, + "updated_at": None, + "last_error": "", + } + return dict(row) + + def _assert_graph_index_ready(self, workspace_id: str) -> dict: + self._recover_stale_graph_jobs(workspace_id) + info = self._graph_index_info(workspace_id) + if info["state"] == "rebuilding" and info.get("active_job_id"): + raise GraphIndexRebuilding(str(info["active_job_id"])) + return info + + @staticmethod + def _graph_job_json(value: Any, fallback: Any) -> Any: + try: + parsed = json.loads(value or "") + except (TypeError, ValueError, RecursionError): + return fallback + return parsed if isinstance(parsed, type(fallback)) else fallback + + def _graph_job_dict(self, row: Any, *, reused: bool = False) -> dict: + data = dict(row) + total = int(data.get("total_items") or 0) + processed = int(data.get("processed_items") or 0) + return { + "id": data["id"], + "workspace_id": data["workspace_id"], + "repo_id": data.get("repo_id"), + "kind": data["kind"], + "state": data["state"], + "dry_run": bool(data.get("dry_run")), + "total_items": total, + "processed_items": processed, + "progress": ( + 1.0 if data["state"] == "completed" + else round(min(1.0, processed / total), 6) if total else 0.0 + ), + "counts": self._graph_job_json(data.get("counts"), {}), + "errors": self._graph_job_json(data.get("errors"), []), + "cancel_requested": bool(data.get("cancel_requested")), + "created_at": data.get("created_at"), + "started_at": data.get("started_at"), + "finished_at": data.get("finished_at"), + "reused": reused, + } + + def graph_index_job(self, job_id: str, *, workspace: str) -> dict: + wid, _rid = self._require_scope(workspace, None) + self._recover_stale_graph_jobs(wid) + clean_id = _clean_text(job_id, field="job_id", max_chars=MAX_NAME_CHARS) + row = self.store.conn.execute( + "SELECT * FROM jobs WHERE id=? AND workspace_id=? AND kind='graph_index'", + (clean_id, wid), + ).fetchone() + if row is None: + raise ValidationError(f"no graph index job '{clean_id}' in workspace '{workspace}'") + return self._graph_job_dict(row) + + def graph_index_status(self, *, workspace: str) -> dict: + wid, _rid = self._require_scope(workspace, None) + self._recover_stale_graph_jobs(wid) + owns_transaction = not self.store.conn.in_transaction + if owns_transaction: + self.store.conn.execute("BEGIN") + try: + info = self._graph_index_info(wid) + row = self.store.conn.execute( + "SELECT * FROM jobs WHERE workspace_id=? AND kind='graph_index' " + "ORDER BY created_at DESC, id DESC LIMIT 1", + (wid,), + ).fetchone() + result = { + "workspace": workspace, + "index": info, + "job": self._graph_job_dict(row) if row is not None else None, + } + if owns_transaction: + self.store.conn.commit() + return result + except BaseException: + if owns_transaction and self.store.conn.in_transaction: + self.store.conn.rollback() + raise + + def start_graph_index_job(self, *, workspace: str, repo: Optional[str] = None, + dry_run: bool = True, extractor: str = "regex") -> dict: + wid, rid = self._require_scope(workspace, repo) + clean_extractor = _clean_text( + extractor, field="extractor", max_chars=32 + ).lower() + if clean_extractor != "regex": + raise ValidationError("extractor must be 'regex'") + with self._graph_job_lock: + self._recover_stale_graph_jobs() + self._graph_job_threads = { + key: value for key, value in self._graph_job_threads.items() + if value.is_alive() + } + self.store.conn.execute("BEGIN IMMEDIATE") + try: + current_scope = self.store.conn.execute( + "SELECT 1 FROM workspaces WHERE id=?", (wid,) + ).fetchone() + if current_scope is None: + raise ValidationError("workspace was removed before the job could start") + if rid is not None and self.store.conn.execute( + "SELECT 1 FROM repos WHERE id=? AND workspace_id=?", (rid, wid) + ).fetchone() is None: + raise ValidationError("repository was removed before the job could start") + active = self.store.conn.execute( + "SELECT * FROM jobs WHERE workspace_id=? AND kind='graph_index' " + "AND state IN ('queued','running') ORDER BY created_at DESC LIMIT 1", + (wid,), + ).fetchone() + if active is not None: + self.store.conn.commit() + return self._graph_job_dict(active, reused=True) + global_active = int(self.store.conn.execute( + "SELECT COUNT(*) AS n FROM jobs WHERE kind='graph_index' " + "AND state IN ('queued','running')" + ).fetchone()["n"]) + if (global_active >= MAX_GRAPH_INDEX_WORKERS + or len(self._graph_job_threads) >= MAX_GRAPH_INDEX_WORKERS): + raise ValidationError( + "too many graph index jobs are active; retry after one finishes" + ) + live_where = ( + "workspace_id=? AND expired_at IS NULL AND valid_to IS NULL" + + (" AND repo_id=?" if rid else "") + ) + params: tuple[Any, ...] = (wid, rid) if rid else (wid,) + snapshot = self.store.conn.execute( + f"SELECT COUNT(*) AS n, MAX(id) AS upper_id FROM memories " + f"WHERE {live_where}", params, + ).fetchone() + total = int(snapshot["n"] or 0) + if total > MAX_GRAPH_INDEX_MEMORIES: + raise ValidationError( + "graph index job exceeds the memory candidate limit; filter by repository" + ) + entity_before = int(self.store.conn.execute( + "SELECT COUNT(*) AS n FROM entities WHERE workspace_id=?", (wid,) + ).fetchone()["n"]) + edge_before = int(self.store.conn.execute( + "SELECT COUNT(*) AS n FROM edges WHERE workspace_id=?", (wid,) + ).fetchone()["n"]) + # Bound maintenance metadata even if a client repeatedly starts dry runs. + self.store.conn.execute( + "DELETE FROM jobs WHERE id IN (SELECT id FROM jobs " + "WHERE workspace_id=? AND kind='graph_index' " + "AND state NOT IN ('queued','running') " + "ORDER BY created_at DESC, id DESC LIMIT -1 OFFSET ?)", + (wid, GRAPH_INDEX_JOB_HISTORY - 1), + ) + job_id = make_id("job") + now = time.time() + counts = { + "memories_scanned": 0, + "entity_mentions": 0, + "relation_mentions": 0, + "entities_before": entity_before, + "relations_before": edge_before, + "entities_after": entity_before, + "relations_after": edge_before, + "entities_added": 0, + "relations_added": 0, + "error_count": 0, + } + request = { + "workspace": workspace, + "repo": repo, + "extractor": clean_extractor, + "dry_run": bool(dry_run), + "upper_memory_id": snapshot["upper_id"] or "", + } + self.store.conn.execute( + "INSERT INTO jobs(id, workspace_id, repo_id, kind, state, dry_run, " + "total_items, processed_items, counts, errors, request, " + "cancel_requested, runner_id, heartbeat_at, created_at) " + "VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)", + ( + job_id, wid, rid, "graph_index", "queued", int(bool(dry_run)), + total, 0, json.dumps(counts, sort_keys=True), "[]", + json.dumps(request, sort_keys=True), 0, self._graph_runner_id, + now, now, + ), + ) + if not dry_run: + self.store.conn.execute( + "INSERT INTO graph_index_state " + "(workspace_id, generation, state, active_job_id, updated_at, " + "last_error) VALUES(?, 1, 'rebuilding', ?, ?, '') " + "ON CONFLICT(workspace_id) DO UPDATE SET " + "state='rebuilding', active_job_id=excluded.active_job_id, " + "updated_at=excluded.updated_at, last_error=''", + (wid, job_id, now), + ) + self.store.conn.commit() + except BaseException: + if self.store.conn.in_transaction: + self.store.conn.rollback() + raise + worker = threading.Thread( + target=self._run_graph_index_job, + args=(job_id,), + name=f"engraphis-graph-index-{job_id[-8:]}", + daemon=True, + ) + self._graph_job_threads[job_id] = worker + try: + worker.start() + except BaseException: + self._graph_job_threads.pop(job_id, None) + failed_at = time.time() + self.store.conn.execute( + "UPDATE jobs SET state='failed', finished_at=?, heartbeat_at=? " + "WHERE id=?", (failed_at, failed_at, job_id), + ) + self.store.conn.execute( + "UPDATE graph_index_state SET state='ready', active_job_id=NULL, " + "updated_at=?, last_error='worker_start_failed' " + "WHERE workspace_id=? AND active_job_id=?", + (failed_at, wid, job_id), + ) + self.store.conn.commit() + raise + row = self.store.conn.execute( + "SELECT * FROM jobs WHERE id=?", (job_id,) + ).fetchone() + return self._graph_job_dict(row) + + def cancel_graph_index_job(self, job_id: str, *, workspace: str) -> dict: + wid, _rid = self._require_scope(workspace, None) + self._recover_stale_graph_jobs(wid) + clean_id = _clean_text(job_id, field="job_id", max_chars=MAX_NAME_CHARS) + row = self.store.conn.execute( + "SELECT * FROM jobs WHERE id=? AND workspace_id=? AND kind='graph_index'", + (clean_id, wid), + ).fetchone() + if row is None: + raise ValidationError(f"no graph index job '{clean_id}' in workspace '{workspace}'") + if row["state"] in {"queued", "running"}: + self.store.conn.execute( + "UPDATE jobs SET cancel_requested=1 WHERE id=?", (clean_id,) + ) + self.store.conn.commit() + row = self.store.conn.execute( + "SELECT * FROM jobs WHERE id=?", (clean_id,) + ).fetchone() + return self._graph_job_dict(row) + + def _run_graph_index_job(self, job_id: str) -> None: + from engraphis.backends.graph_extractor import ( + StructuredMetadataGraphExtractor, + feed as graph_feed, + get_graph_extractor, + ) + + row = self.store.conn.execute( + "SELECT * FROM jobs WHERE id=? AND runner_id=?", + (job_id, self._graph_runner_id), + ).fetchone() + if row is None: + return + wid, rid, dry_run = row["workspace_id"], row["repo_id"], bool(row["dry_run"]) + request = self._graph_job_json(row["request"], {}) + counts = { + "memories_scanned": 0, + "entity_mentions": 0, + "relation_mentions": 0, + "entities_before": 0, + "relations_before": 0, + "entities_after": 0, + "relations_after": 0, + "entities_added": 0, + "relations_added": 0, + "error_count": 0, + **self._graph_job_json(row["counts"], {}), + } + errors: list[dict] = [] + final_state = "failed" + error_code = "" + try: + started = time.time() + claimed = self.store.conn.execute( + "UPDATE jobs SET state='running', started_at=?, heartbeat_at=? " + "WHERE id=? AND runner_id=? AND state='queued'", + (started, started, job_id, self._graph_runner_id), + ) + self.store.conn.commit() + if claimed.rowcount != 1: + return + regex_extractor = get_graph_extractor(str(request.get("extractor") or "regex")) + upper_memory_id = str(request.get("upper_memory_id") or "") + last_memory_id = "" + processed = 0 + stop = False + while not stop: + cancellation = self.store.conn.execute( + "SELECT cancel_requested, state, runner_id FROM jobs WHERE id=?", + (job_id,), + ).fetchone() + if (cancellation is None or bool(cancellation["cancel_requested"]) + or cancellation["state"] != "running" + or cancellation["runner_id"] != self._graph_runner_id): + final_state = "cancelled" + break + id_sql = ( + "SELECT id FROM memories WHERE workspace_id=? " + "AND expired_at IS NULL AND valid_to IS NULL AND id>?" + ) + id_params: list[Any] = [wid, last_memory_id] + if rid: + id_sql += " AND repo_id=?" + id_params.append(rid) + if upper_memory_id: + id_sql += " AND id<=?" + id_params.append(upper_memory_id) + id_sql += " ORDER BY id LIMIT ?" + id_params.append(GRAPH_INDEX_BATCH_SIZE) + memory_ids = [row["id"] for row in self.store.conn.execute( + id_sql, id_params + ).fetchall()] + if not memory_ids: + final_state = "completed" + break + for memory_id in memory_ids: + last_memory_id = memory_id + cancelled = self.store.conn.execute( + "SELECT cancel_requested, state, runner_id FROM jobs WHERE id=?", + (job_id,), + ).fetchone() + if (cancelled is None or bool(cancelled["cancel_requested"]) + or cancelled["state"] not in {"queued", "running"} + or cancelled["runner_id"] != self._graph_runner_id): + final_state = "cancelled" + stop = True + break + transaction_started = False + try: + if not dry_run: + self.store.conn.execute("BEGIN IMMEDIATE") + transaction_started = True + memory_sql = ( + "SELECT id, repo_id, title, content, metadata FROM memories " + "WHERE id=? AND workspace_id=? AND expired_at IS NULL " + "AND valid_to IS NULL" + ) + memory_params: list[Any] = [memory_id, wid] + if rid: + memory_sql += " AND repo_id=?" + memory_params.append(rid) + memory = self.store.conn.execute( + memory_sql, memory_params + ).fetchone() + if memory is not None: + try: + metadata = json.loads(memory["metadata"] or "{}") + except (TypeError, ValueError, RecursionError): + metadata = {} + extractors: list[tuple[str, Any]] = [] + if (isinstance(metadata, dict) + and self.engine._has_structured_graph_metadata(metadata)): + extractors.append(( + "structured_index", + StructuredMetadataGraphExtractor(metadata), + )) + extractors.append(("regex_index", regex_extractor)) + for source, selected_extractor in extractors: + extraction = selected_extractor.extract( + memory["content"] or "", title=memory["title"] or "" + ) + counts["entity_mentions"] += len(extraction.entities) + counts["relation_mentions"] += len(extraction.relations) + if not dry_run: + graph_feed( + self.store, + memory["content"] or "", + workspace_id=wid, + repo_id=memory["repo_id"], + title=memory["title"] or "", + extractor=selected_extractor, + extraction=extraction, + provenance={ + "source": source, + "memory_id": memory["id"], + "job_id": job_id, + }, + commit=False, + ) + processed += 1 + counts["memories_scanned"] = processed + heartbeat = time.time() + progress = self.store.conn.execute( + "UPDATE jobs SET processed_items=?, counts=?, errors=?, " + "heartbeat_at=? WHERE id=? AND runner_id=? AND state='running'", + ( + processed, + json.dumps(counts, sort_keys=True), + json.dumps(errors, sort_keys=True), + heartbeat, + job_id, + self._graph_runner_id, + ), + ) + self.store.conn.commit() + transaction_started = False + if progress.rowcount != 1: + final_state = "cancelled" + stop = True + break + except Exception as exc: # noqa: BLE001 - isolate one bad memory + if transaction_started or self.store.conn.in_transaction: + self.store.conn.rollback() + counts["error_count"] += 1 + if len(errors) < 25: + errors.append({ + "item": processed + 1, + "code": type(exc).__name__[:80], + }) + processed += 1 + counts["memories_scanned"] = processed + heartbeat = time.time() + self.store.conn.execute( + "UPDATE jobs SET processed_items=?, counts=?, errors=?, " + "heartbeat_at=? WHERE id=? AND runner_id=? AND state='running'", + ( + processed, + json.dumps(counts, sort_keys=True), + json.dumps(errors, sort_keys=True), + heartbeat, + job_id, + self._graph_runner_id, + ), + ) + self.store.conn.commit() + + entity_after = int(self.store.conn.execute( + "SELECT COUNT(*) AS n FROM entities WHERE workspace_id=?", (wid,) + ).fetchone()["n"]) + edge_after = int(self.store.conn.execute( + "SELECT COUNT(*) AS n FROM edges WHERE workspace_id=?", (wid,) + ).fetchone()["n"]) + counts.update({ + "entities_after": entity_after, + "relations_after": edge_after, + "entities_added": entity_after - int(counts["entities_before"]), + "relations_added": edge_after - int(counts["relations_before"]), + }) + except Exception as exc: # noqa: BLE001 - persist a safe terminal job state + error_code = type(exc).__name__[:80] + counts["error_count"] = int(counts.get("error_count") or 0) + 1 + errors.append({"item": int(counts.get("memories_scanned") or 0), + "code": error_code}) + final_state = "failed" + finally: + try: + status = "ok" if final_state == "completed" else final_state + self.store.audit( + "system", f"graph_index_{final_state}", wid, + f"job={job_id}; dry_run={int(dry_run)}; " + f"processed={int(counts.get('memories_scanned') or 0)}", + ) + self.store.record_receipt( + "graph_index", + workspace_id=wid, + repo_id=rid or "", + actor="system", + target_count=int(counts.get("memories_scanned") or 0), + status=status, + metadata={ + "dry_run": bool(dry_run), + "error_count": int(counts.get("error_count") or 0), + "entities_added": int(counts.get("entities_added") or 0), + "relations_added": int(counts.get("relations_added") or 0), + }, + ) + except Exception as exc: # noqa: BLE001 - terminal state must still persist + error_code = type(exc).__name__[:80] + counts["error_count"] = int(counts.get("error_count") or 0) + 1 + errors.append({"item": int(counts.get("memories_scanned") or 0), + "code": error_code}) + final_state = "failed" + finally: + finished = time.time() + terminal = self.store.conn.execute( + "UPDATE jobs SET state=?, processed_items=?, counts=?, errors=?, " + "finished_at=?, heartbeat_at=? WHERE id=? AND runner_id=? " + "AND state IN ('queued','running')", + ( + final_state, + int(counts.get("memories_scanned") or 0), + json.dumps(counts, sort_keys=True), + json.dumps(errors[:25], sort_keys=True), + finished, + finished, + job_id, + self._graph_runner_id, + ), + ) + if not dry_run and terminal.rowcount == 1: + self.store.conn.execute( + "UPDATE graph_index_state SET state='ready', active_job_id=NULL, " + "updated_at=?, last_error=? " + "WHERE workspace_id=? AND active_job_id=? " + "AND EXISTS(SELECT 1 FROM workspaces WHERE id=?)", + (finished, error_code, wid, job_id, wid), + ) + self.store.conn.commit() + self._graph_scene_cache.clear() + with self._graph_job_lock: + self._graph_job_threads.pop(job_id, None) + + def _graph_scene_rows(self, *, workspace: str, repo: Optional[str] = None, + as_of: Optional[float] = None, + entity_types: Optional[list[str]] = None, + memory_types: Optional[list[str]] = None, + time_from: Optional[float] = None, + time_to: Optional[float] = None, + include_weak_cooccurrence: bool = True, + include_code: bool = False, + include_complete_rows: bool = False) -> tuple: + """Load one transactionally consistent graph snapshot and generation state.""" + clean_workspace = self._clean_ws(workspace) + workspace_id = self._lookup_workspace(clean_workspace) + if workspace_id: + self._recover_stale_graph_jobs(workspace_id) + owns_transaction = not self.store.conn.in_transaction + if owns_transaction: + self.store.conn.execute("BEGIN") + try: + rows = self._graph_scene_rows_unlocked( + workspace=clean_workspace, + repo=repo, + as_of=as_of, + entity_types=entity_types, + memory_types=memory_types, + time_from=time_from, + time_to=time_to, + include_weak_cooccurrence=include_weak_cooccurrence, + include_code=include_code, + include_complete_rows=include_complete_rows, + ) + index_info = self._graph_index_info(rows[1]) if rows[1] else { + "generation": self.store.schema_version, + "state": "ready", + "active_job_id": None, + "updated_at": None, + "last_error": "", + } + if owns_transaction: + self.store.conn.commit() + return (*rows, index_info) + except BaseException: + if owns_transaction and self.store.conn.in_transaction: + self.store.conn.rollback() + raise + + def _graph_scene_rows_unlocked(self, *, workspace: str, repo: Optional[str] = None, + as_of: Optional[float] = None, + entity_types: Optional[list[str]] = None, + memory_types: Optional[list[str]] = None, + time_from: Optional[float] = None, + time_to: Optional[float] = None, + include_weak_cooccurrence: bool = True, + include_code: bool = False, + include_complete_rows: bool = False) -> tuple: + """Load the complete scoped graph for deterministic server-side ranking. + + This is intentionally read-only. Graph population is an explicit write/index + concern; no GET path calls the legacy lazy backfill helpers. + """ + ws = self._clean_ws(workspace) + wid = self._lookup_workspace(ws) + if wid is None: + return ws, "", [], [], [], [], [], [] + self._assert_graph_index_ready(wid) + clean_entity_types = ( + _clean_string_list( + entity_types, field="entity_types", max_items=64, + max_chars=MAX_NAME_CHARS, + ) + if entity_types is not None else [] + ) + clean_memory_types = sorted({ + _enum(value, MemoryType, "memory_type").value + for value in _clean_string_list( + memory_types, field="memory_types", max_items=4, + max_chars=MAX_NAME_CHARS, + ) + }) if memory_types is not None else [] + repo_id = None + if repo: + repo_name = _clean_name(repo, field="repo") + repo_id = self._lookup_repo(wid, repo_name) + if repo_id is None: + raise ValidationError(f"no repo named '{repo_name}' in workspace '{ws}'") + entity_sql = ( + "SELECT id, workspace_id, repo_id, name, etype, canonical_id, " + "normalized_name, canonical_method, canonical_confidence, created_at " + "FROM entities WHERE workspace_id=?" + ) + entity_params: list[Any] = [wid] + if repo_id: + entity_sql += " AND (repo_id=? OR repo_id IS NULL)" + entity_params.append(repo_id) + if clean_entity_types: + clean_types = sorted(set(clean_entity_types)) + if clean_types: + marks = ",".join("?" for _ in clean_types) + entity_sql += f" AND etype IN ({marks})" + entity_params.extend(clean_types) + entity_sql += " ORDER BY canonical_id, id LIMIT ?" + entity_params.append(MAX_GRAPH_ANALYSIS_ENTITIES + 1) + entity_rows = [dict(row) for row in self.store.conn.execute( + entity_sql, entity_params + ).fetchall()] + if len(entity_rows) > MAX_GRAPH_ANALYSIS_ENTITIES: + if include_complete_rows: + raise GraphSceneCapacityExceeded( + resource="entity rows", count=len(entity_rows), + limit=MAX_GRAPH_ANALYSIS_ENTITIES, + ) + raise ValidationError( + "graph analysis exceeds the entity candidate limit; filter by repository" + ) + + try: + t = float(as_of) if as_of is not None else __import__("time").time() + except (TypeError, ValueError, OverflowError): + raise ValidationError("as_of must be a finite timestamp") + if not math.isfinite(t): + raise ValidationError("as_of must be a finite timestamp") + try: + lower_time = float(time_from) if time_from is not None else None + upper_time = float(time_to) if time_to is not None else None + except (TypeError, ValueError, OverflowError): + raise ValidationError("time range values must be finite timestamps") + if ((lower_time is not None and not math.isfinite(lower_time)) + or (upper_time is not None and not math.isfinite(upper_time))): + raise ValidationError("time range values must be finite timestamps") + if lower_time is not None and upper_time is not None and lower_time > upper_time: + raise ValidationError("time_from must be less than or equal to time_to") + edge_sql = ( + "SELECT id, workspace_id, repo_id, src, dst, relation, layer, weight, " + "valid_from, valid_to, ingested_at, expired_at, provenance FROM edges " + "WHERE workspace_id=? AND (valid_from IS NULL OR valid_from<=?) " + "AND (valid_to IS NULL OR ? MAX_GRAPH_ANALYSIS_EDGES: + if include_complete_rows: + raise GraphSceneCapacityExceeded( + resource="raw relations", count=len(edge_rows), + limit=MAX_GRAPH_ANALYSIS_EDGES, + ) + raise ValidationError( + "graph analysis exceeds the relation candidate limit; filter by repository" + ) + + if include_code: + repo_sql = "SELECT id, name FROM repos WHERE workspace_id=?" + repo_params: list[Any] = [wid] + if repo_id: + repo_sql += " AND id=?" + repo_params.append(repo_id) + repo_rows = self.store.conn.execute( + repo_sql + " ORDER BY name, id", repo_params + ).fetchall() + for repo_row in repo_rows: + remaining_entities = MAX_GRAPH_ANALYSIS_ENTITIES - len(entity_rows) + symbol_rows = [dict(row) for row in self.store.conn.execute( + "SELECT id, kind, name, fqname, file FROM symbols " + "WHERE repo_id=? ORDER BY id LIMIT ?", + (repo_row["id"], remaining_entities + 1), + ).fetchall()] + if len(symbol_rows) > remaining_entities: + if include_complete_rows: + raise GraphSceneCapacityExceeded( + resource="entity rows", + count=MAX_GRAPH_ANALYSIS_ENTITIES + 1, + limit=MAX_GRAPH_ANALYSIS_ENTITIES, + ) + raise ValidationError( + "graph analysis exceeds the entity candidate limit; " + "filter the code overlay by repository" + ) + endpoint: dict[str, str] = {} + for symbol in symbol_rows: + node_id = f"code:{symbol['id']}" + label = symbol.get("fqname") or symbol.get("name") or symbol["id"] + entity_rows.append({ + "id": node_id, "workspace_id": wid, "repo_id": repo_row["id"], + "name": f"{repo_row['name']}:{label}", + "etype": f"code_{symbol.get('kind') or 'symbol'}", + "canonical_id": node_id, + "normalized_name": normalize_entity_name(label), + "canonical_method": "code_identity", "canonical_confidence": 1.0, + }) + for key in (symbol.get("id"), symbol.get("fqname"), symbol.get("name")): + if key: + endpoint.setdefault(str(key), node_id) + remaining_edges = MAX_GRAPH_ANALYSIS_EDGES - len(edge_rows) + code_edges = self.store.conn.execute( + "SELECT id, src, dst, relation, layer FROM code_edges " + "WHERE repo_id=? ORDER BY id LIMIT ?", + (repo_row["id"], remaining_edges + 1), + ).fetchall() + if len(code_edges) > remaining_edges: + if include_complete_rows: + raise GraphSceneCapacityExceeded( + resource="raw relations", + count=MAX_GRAPH_ANALYSIS_EDGES + 1, + limit=MAX_GRAPH_ANALYSIS_EDGES, + ) + raise ValidationError( + "graph analysis exceeds the relation candidate limit; " + "filter the code overlay by repository" + ) + for code_edge in code_edges: + source = endpoint.get(str(code_edge["src"] or "")) + target = endpoint.get(str(code_edge["dst"] or "")) + if source and target and source != target: + edge_rows.append({ + "id": f"code-edge:{code_edge['id']}", "workspace_id": wid, + "repo_id": repo_row["id"], "src": source, "dst": target, + "relation": code_edge["relation"] or "references", + "layer": code_edge["layer"] or "entity", "weight": 1.0, + "valid_from": None, "valid_to": None, "ingested_at": None, + "expired_at": None, + "provenance": json.dumps({"source": "code_index"}), + }) + if evidence_filter: + endpoints = { + str(edge.get(key) or "") for edge in edge_rows + for key in ("src", "dst") if edge.get(key) + } + canonical_by_member = { + str(entity.get("id") or ""): str( + entity.get("canonical_id") or entity.get("id") or "" + ) + for entity in entity_rows + } + endpoint_canonicals = { + canonical_by_member.get(endpoint, endpoint) for endpoint in endpoints + } + entity_rows = [ + entity for entity in entity_rows + if str(entity.get("canonical_id") or entity.get("id") or "") + in endpoint_canonicals + ] + edge_ids = [row["id"] for row in edge_rows if not str(row["id"]).startswith("code-edge:")] + # Bounded IN chunks avoid a second scan of the relation table while preserving + # the exact selected edge ids. Weak co-occurrence is filtered after canonical + # relation bundling, once its aggregate support is known. + support_rows = self.store.edge_supports_in_scope( + edge_ids, at=t, limit=MAX_GRAPH_ANALYSIS_SUPPORTS + 1 + ) + if len(support_rows) > MAX_GRAPH_ANALYSIS_SUPPORTS: + if include_complete_rows: + raise GraphSceneCapacityExceeded( + resource="evidence rows", count=len(support_rows), + limit=MAX_GRAPH_ANALYSIS_SUPPORTS, + ) + raise ValidationError( + "graph analysis exceeds the evidence candidate limit; filter by repository" + ) + # Attach only public analytical metadata from supporting memories. This both + # makes the memory/time facets evidence-backed and ensures requested evidence + # filters cannot be bypassed by another support row on the same relation. + support_memory_ids = sorted({ + str(row.get("memory_id") or "") for row in support_rows + if row.get("memory_id") + }) + support_memory_meta: dict[str, tuple[str, float]] = {} + for start in range(0, len(support_memory_ids), 500): + chunk = support_memory_ids[start:start + 500] + marks = ",".join("?" for _ in chunk) + memory_sql = ( + "SELECT id, mtype, COALESCE(valid_from, ingested_at, 0) AS support_time " + "FROM memories WHERE workspace_id=? AND id IN (" + marks + ") " + "AND (valid_from IS NULL OR valid_from<=?) " + "AND (valid_to IS NULL OR ?=?") + memory_params.append(lower_time) + if upper_time is not None: + memory_where.append("COALESCE(valid_from, ingested_at, 0)<=?") + memory_params.append(upper_time) + scoped_memory_sql = "SELECT id FROM memories WHERE " + " AND ".join(memory_where) + memory_rows = [dict(row) for row in self.store.conn.execute( + "SELECT id, repo_id, session_id, scope, mtype, title, " + "substr(content, 1, 160) AS content, substr(summary, 1, 160) AS summary, " + "importance, valid_from, ingested_at, pinned FROM memories WHERE " + + " AND ".join(memory_where) + " ORDER BY id LIMIT ?", + [*memory_params, MAX_GRAPH_COMPLETE_MEMORIES + 1], + ).fetchall()] + if len(memory_rows) > MAX_GRAPH_COMPLETE_MEMORIES: + raise GraphSceneCapacityExceeded( + resource="memory nodes", count=len(memory_rows), + limit=MAX_GRAPH_COMPLETE_MEMORIES, + ) + + memory_link_rows = [dict(row) for row in self.store.conn.execute( + "WITH selected_memory AS (" + scoped_memory_sql + ") " + "SELECT links.a, links.b, links.relation, links.layer, links.reason, " + "links.created_at FROM mem_links links " + "JOIN selected_memory source ON source.id=links.a " + "JOIN selected_memory target ON target.id=links.b " + "ORDER BY links.a, links.b, links.relation, links.layer, links.created_at " + "LIMIT ?", + [*memory_params, MAX_GRAPH_COMPLETE_MEMORY_LINKS + 1], + ).fetchall()] + if len(memory_link_rows) > MAX_GRAPH_COMPLETE_MEMORY_LINKS: + raise GraphSceneCapacityExceeded( + resource="memory connectors", count=len(memory_link_rows), + limit=MAX_GRAPH_COMPLETE_MEMORY_LINKS, + ) + + if include_code: + code_sql = ( + "WITH selected_memory AS (" + scoped_memory_sql + ") " + "SELECT links.id, links.repo_id, links.symbol_id, links.memory_id, " + "links.relation, links.confidence FROM code_memory_links links " + "JOIN selected_memory memory ON memory.id=links.memory_id " + "JOIN repos repo ON repo.id=links.repo_id WHERE repo.workspace_id=?" + ) + code_params: list[Any] = [*memory_params, wid] + if repo_id: + code_sql += " AND links.repo_id=?" + code_params.append(repo_id) + code_sql += " ORDER BY links.id LIMIT ?" + code_params.append(MAX_GRAPH_COMPLETE_CODE_MEMORY_LINKS + 1) + code_memory_link_rows = [dict(row) for row in self.store.conn.execute( + code_sql, code_params, + ).fetchall()] + if len(code_memory_link_rows) > MAX_GRAPH_COMPLETE_CODE_MEMORY_LINKS: + raise GraphSceneCapacityExceeded( + resource="code-memory connectors", + count=len(code_memory_link_rows), + limit=MAX_GRAPH_COMPLETE_CODE_MEMORY_LINKS, + ) + return ( + ws, wid, entity_rows, edge_rows, enriched_supports, + memory_rows, memory_link_rows, code_memory_link_rows, + ) + + def graph_scene(self, *, workspace: str, level: str = "overview", + center_id: Optional[str] = None, + system_id: Optional[str] = None, + seeds: Optional[list[str]] = None, + repo: Optional[str] = None, + layers: Optional[list[str]] = None, + relations: Optional[list[str]] = None, + entity_types: Optional[list[str]] = None, + memory_types: Optional[list[str]] = None, + as_of: Optional[float] = None, depth: int = 1, + time_from: Optional[float] = None, + time_to: Optional[float] = None, + min_support: int = 1, min_confidence: float = 0.0, + include_weak_cooccurrence: bool = False, + include_code: bool = False, + node_limit: Optional[int] = None, + edge_limit: Optional[int] = None) -> dict: + started = time.perf_counter() + clean_workspace = self._clean_ws(workspace) + clean_level = _clean_text( + level, field="level", max_chars=32 + ).lower() + if clean_level not in {"overview", "system", "neighborhood", "path", "complete"}: + raise ValidationError( + "level must be one of: overview, system, neighborhood, path, complete" + ) + clean_center_id = ( + _clean_text(center_id, field="center_id", max_chars=MAX_NAME_CHARS) + if center_id is not None else None + ) + clean_system_id = ( + _clean_text(system_id, field="system_id", max_chars=MAX_NAME_CHARS) + if system_id is not None else None + ) + clean_seeds = list(dict.fromkeys(_clean_string_list( + seeds, field="seeds", max_items=64, max_chars=MAX_NAME_CHARS, + ))) if seeds is not None else [] + clean_repo = _clean_name(repo, field="repo") if repo is not None else None + clean_relations = sorted(set(_clean_string_list( + relations, field="relations", max_items=64, max_chars=MAX_NAME_CHARS, + ))) if relations is not None else [] + clean_entity_types = sorted(set(_clean_string_list( + entity_types, field="entity_types", max_items=64, + max_chars=MAX_NAME_CHARS, + ))) if entity_types is not None else [] + clean_memory_types = sorted({ + _enum(value, MemoryType, "memory_type").value + for value in _clean_string_list( + memory_types, field="memory_types", max_items=4, + max_chars=MAX_NAME_CHARS, + ) + }) if memory_types is not None else [] + clean_layers = None + if layers is not None: + layer_values = _clean_string_list( + layers, field="layers", max_items=64, max_chars=MAX_NAME_CHARS, + ) + clean_layers = sorted({ + _enum(value, GraphLayer, "layer").value for value in layer_values + }) + + def bounded_int(value: Any, field: str, minimum: int, maximum: int) -> int: + try: + parsed = int(value) + except (TypeError, ValueError, OverflowError): + raise ValidationError(f"{field} must be an integer") + if parsed < minimum or parsed > maximum: + raise ValidationError(f"{field} must be between {minimum} and {maximum}") + return parsed + + clean_depth = bounded_int(depth, "depth", 0, 2) + clean_min_support = bounded_int(min_support, "min_support", 0, 1_000_000) + clean_node_limit = ( + bounded_int(node_limit, "node_limit", 1, 300) + if node_limit is not None else None + ) + clean_edge_limit = ( + bounded_int(edge_limit, "edge_limit", 0, 900) + if edge_limit is not None else None + ) + if clean_level == "complete" and ( + clean_node_limit is not None or clean_edge_limit is not None + ): + raise ValidationError( + "complete scenes do not accept node_limit or edge_limit; " + "use graph filters instead of silently truncating the chart" + ) + try: + clean_min_confidence = float(min_confidence) + except (TypeError, ValueError, OverflowError): + raise ValidationError("min_confidence must be a finite number") + if not math.isfinite(clean_min_confidence) or not 0.0 <= clean_min_confidence <= 1.0: + raise ValidationError("min_confidence must be between 0 and 1") + try: + clean_as_of = float(as_of) if as_of is not None else None + except (TypeError, ValueError, OverflowError): + raise ValidationError("as_of must be a finite timestamp") + if clean_as_of is not None and not math.isfinite(clean_as_of): + raise ValidationError("as_of must be a finite timestamp") + try: + clean_time_from = float(time_from) if time_from is not None else None + clean_time_to = float(time_to) if time_to is not None else None + except (TypeError, ValueError, OverflowError): + raise ValidationError("time range values must be finite timestamps") + if ((clean_time_from is not None and not math.isfinite(clean_time_from)) + or (clean_time_to is not None and not math.isfinite(clean_time_to))): + raise ValidationError("time range values must be finite timestamps") + if (clean_time_from is not None and clean_time_to is not None + and clean_time_from > clean_time_to): + raise ValidationError("time_from must be less than or equal to time_to") + + cache_workspace_id = self._lookup_workspace(clean_workspace) + if cache_workspace_id: + self._assert_graph_index_ready(cache_workspace_id) + revision = self._graph_scene_revision() + cache_key = ( + revision, clean_workspace, clean_level, clean_center_id or "", + clean_system_id or "", tuple(clean_seeds), clean_repo or "", + tuple(clean_layers or ()), tuple(clean_relations), tuple(clean_entity_types), + tuple(clean_memory_types), clean_as_of, clean_time_from, clean_time_to, + clean_depth, clean_min_support, + clean_min_confidence, bool(include_weak_cooccurrence), + bool(include_code), clean_node_limit, clean_edge_limit, + ) + cached = self._graph_scene_cache.get(cache_key) + if cached is not None and (clean_as_of is not None or time.time() < cached[0]): + self._graph_scene_cache.move_to_end(cache_key) + scene = copy.deepcopy(cached[1]) + scene["meta"]["cache_hit"] = True + scene["meta"]["query_ms"] = round( + (time.perf_counter() - started) * 1000.0, 3 + ) + return scene + if cached is not None: + del self._graph_scene_cache[cache_key] + query_at = clean_as_of if clean_as_of is not None else time.time() + (ws, _wid, entities, edges, supports, memories, memory_links, + code_memory_links, index_info) = self._graph_scene_rows( + workspace=clean_workspace, repo=clean_repo, as_of=query_at, + entity_types=clean_entity_types, memory_types=clean_memory_types, + time_from=clean_time_from, time_to=clean_time_to, + include_weak_cooccurrence=include_weak_cooccurrence, + include_code=include_code, + include_complete_rows=clean_level == "complete", + ) + selected_layers = set(clean_layers) if clean_layers is not None else None + selected_relations = set(clean_relations) or None + filters = { + "repo": clean_repo, + "layers": sorted(selected_layers) if selected_layers is not None else None, + "relations": sorted(selected_relations) if selected_relations else None, + "entity_types": clean_entity_types, + "memory_types": clean_memory_types, + "as_of": clean_as_of, + "time_from": clean_time_from, + "time_to": clean_time_to, + "min_support": clean_min_support, + "min_confidence": clean_min_confidence, + "include_weak_cooccurrence": bool(include_weak_cooccurrence), + "include_code": bool(include_code), + } + filters = {key: value for key, value in filters.items() + if value not in (None, [], False)} + scene = build_graph_scene( + ws, entities, edges, supports, level=clean_level, + memory_rows=memories, memory_link_rows=memory_links, + code_memory_link_rows=code_memory_links, + center_id=clean_center_id, system_id=clean_system_id, + seeds=clean_seeds, depth=clean_depth, + node_limit=clean_node_limit, edge_limit=clean_edge_limit, + include_weak_cooccurrence=include_weak_cooccurrence, + layers=selected_layers, relations=selected_relations, + min_support=clean_min_support, min_confidence=clean_min_confidence, + filters=filters, index_generation=int(index_info["generation"]), + ) + scene["meta"]["index_state"] = index_info["state"] + scene["meta"]["query_ms"] = round((time.perf_counter() - started) * 1000.0, 3) + scene["meta"]["cache_hit"] = False + if clean_level == "complete": + scene["meta"]["safety_limits"] = { + "entity_rows": MAX_GRAPH_ANALYSIS_ENTITIES, + "raw_relations": MAX_GRAPH_ANALYSIS_EDGES, + "evidence_rows": MAX_GRAPH_ANALYSIS_SUPPORTS, + "memory_nodes": MAX_GRAPH_COMPLETE_MEMORIES, + "memory_connectors": MAX_GRAPH_COMPLETE_MEMORY_LINKS, + "code_memory_connectors": MAX_GRAPH_COMPLETE_CODE_MEMORY_LINKS, + "payload_bytes": MAX_GRAPH_COMPLETE_PAYLOAD_BYTES, + } + payload_bytes = len(json.dumps( + scene, ensure_ascii=False, separators=(",", ":") + ).encode("utf-8")) + if payload_bytes > MAX_GRAPH_COMPLETE_PAYLOAD_BYTES: + raise GraphSceneCapacityExceeded( + resource="payload bytes", count=payload_bytes, + limit=MAX_GRAPH_COMPLETE_PAYLOAD_BYTES, + ) + scene["meta"]["payload_bytes_estimate"] = payload_bytes + valid_until = ( + math.inf if clean_as_of is not None or not _wid + else self._graph_scene_valid_until(_wid, query_at) + ) + # One complete scene can be many megabytes. Keep at most one in the shared + # LRU while retaining the normal 16-entry budget for compact analytical views. + if clean_level == "complete": + for key in [key for key in self._graph_scene_cache if key[2] == "complete"]: + self._graph_scene_cache.pop(key, None) + self._graph_scene_cache[cache_key] = (valid_until, copy.deepcopy(scene)) + self._graph_scene_cache.move_to_end(cache_key) + while len(self._graph_scene_cache) > 16: + self._graph_scene_cache.popitem(last=False) + return scene + + def graph_suggest(self, query: str, *, workspace: str, limit: int = 8, + repo: Optional[str] = None, + memory_types: Optional[list[str]] = None, + as_of: Optional[float] = None, + time_from: Optional[float] = None, + time_to: Optional[float] = None, + include_weak_cooccurrence: bool = False) -> dict: + clean_query = _clean_text( + query, field="query", max_chars=1_000, required=False + ) + ws = self._clean_ws(workspace) + wid = self._lookup_workspace(ws) + limit = max(1, min(25, int(limit))) + needle = normalize_entity_name(clean_query) + empty_groups = { + "systems": [], "entities": [], "memories": [], "repositories": [], + "relations": [], "code_symbols": [], + } + if not wid: + return {"workspace": ws, "query": clean_query, "groups": empty_groups} + self._assert_graph_index_ready(wid) + repo_id = None + if repo: + clean_repo = _clean_name(repo, field="repo") + repo_id = self._lookup_repo(wid, clean_repo) + if repo_id is None: + raise ValidationError(f"no repo named '{clean_repo}' in workspace '{ws}'") + try: + suggestion_at = float(as_of) if as_of is not None else time.time() + lower_time = float(time_from) if time_from is not None else None + upper_time = float(time_to) if time_to is not None else None + except (TypeError, ValueError, OverflowError): + raise ValidationError("graph suggestion times must be finite timestamps") + if (not math.isfinite(suggestion_at) + or (lower_time is not None and not math.isfinite(lower_time)) + or (upper_time is not None and not math.isfinite(upper_time))): + raise ValidationError("graph suggestion times must be finite timestamps") + if lower_time is not None and upper_time is not None and lower_time > upper_time: + raise ValidationError("time_from must be less than or equal to time_to") + clean_memory_types = sorted({ + _enum(value, MemoryType, "memory_type").value + for value in _clean_string_list( + memory_types, field="memory_types", max_items=4, + max_chars=MAX_NAME_CHARS, + ) + }) if memory_types is not None else [] + escaped = needle.replace("\\", "\\\\").replace("%", "\\%").replace("_", "\\_") + like = f"%{escaped}%" + prefix = f"{escaped}%" + + # Search identity rows directly instead of rebuilding Louvain/PageRank for each + # keystroke. A canonical entity id also resolves to its current deterministic + # community in ``build_graph_scene``, so the same stable result can represent an + # entity or a system without maintaining a second search index. + entity_sql = ( + "SELECT id, canonical_id, name, normalized_name, etype, repo_id " + "FROM entities WHERE workspace_id=? AND (" + "normalized_name LIKE ? ESCAPE '\\' OR canonical_id=? OR id=?)" + ) + entity_params: list[Any] = [wid, like, clean_query, clean_query] + if repo_id: + entity_sql += " AND (repo_id=? OR repo_id IS NULL)" + entity_params.append(repo_id) + entity_sql += ( + " ORDER BY CASE WHEN canonical_id=? OR id=? THEN -1 " + "WHEN normalized_name=? THEN 0 " + "WHEN normalized_name LIKE ? ESCAPE '\\' THEN 1 ELSE 2 END, " + "length(normalized_name), normalized_name, id LIMIT 500" + ) + entity_params.extend((clean_query, clean_query, needle, prefix)) + matched_rows = [dict(row) for row in self.store.conn.execute( + entity_sql, entity_params, + ).fetchall()] + matched_by_canonical: dict[str, list[dict]] = {} + for row in matched_rows: + canonical_id = str(row.get("canonical_id") or row["id"]) + matched_by_canonical.setdefault(canonical_id, []).append(row) + + def entity_rank(item: tuple[str, list[dict]]) -> tuple: + canonical_id, rows = item + exact_id = canonical_id == clean_query or any( + str(row["id"]) == clean_query for row in rows + ) + best = min(rows, key=lambda row: ( + 0 if row["normalized_name"] == needle else + 1 if str(row["normalized_name"]).startswith(needle) else 2, + len(str(row["normalized_name"])), str(row["normalized_name"]), row["id"], + )) + return ( + -1 if exact_id else + 0 if best["normalized_name"] == needle else + 1 if str(best["normalized_name"]).startswith(needle) else 2, + len(str(best["normalized_name"])), + str(best["normalized_name"]), canonical_id, + ) + + ranked_identity_items = sorted(matched_by_canonical.items(), key=entity_rank) + + def useful_identity(item: tuple[str, list[dict]]) -> bool: + """Keep search useful without making extractor fragments undiscoverable. + + Exact label queries remain available by stable canonical id. For broader + prefix/substring searches, however, sentence fragments such as ``If Python`` + and ``Python-based`` must not crowd out the actual ``Python`` entity. + """ + _canonical_id, rows = item + best = min(rows, key=lambda row: ( + 0 if row["normalized_name"] == needle else + 1 if str(row["normalized_name"]).startswith(needle) else 2, + len(str(row["normalized_name"])), str(row["normalized_name"]), row["id"], + )) + exact = ( + _canonical_id == clean_query + or any(str(row["id"]) == clean_query for row in rows) + or str(best["normalized_name"]) == needle + ) + return exact or not is_broad_search_fragment( + str(best.get("name") or ""), + str(best.get("etype") or "person_or_concept"), + ) + + selected_canonical_ids = [item[0] for item in ranked_identity_items + if useful_identity(item)][:limit] + member_rows: list[dict] = [] + if selected_canonical_ids: + marks = ",".join("?" for _ in selected_canonical_ids) + member_sql = ( + "SELECT id, canonical_id, name, normalized_name, etype, repo_id " + f"FROM entities WHERE workspace_id=? AND canonical_id IN ({marks})" + ) + member_params: list[Any] = [wid, *selected_canonical_ids] + if repo_id: + member_sql += " AND (repo_id=? OR repo_id IS NULL)" + member_params.append(repo_id) + member_sql += " ORDER BY canonical_id, normalized_name, id" + member_rows = [dict(row) for row in self.store.conn.execute( + member_sql, member_params, + ).fetchall()] + members_by_canonical: dict[str, list[dict]] = {} + member_to_canonical: dict[str, str] = {} + for row in member_rows: + canonical_id = str(row.get("canonical_id") or row["id"]) + members_by_canonical.setdefault(canonical_id, []).append(row) + member_to_canonical[str(row["id"])] = canonical_id + support_counts: Counter = Counter() + member_ids = sorted(member_to_canonical) + if member_ids: + seen_supports: dict[str, set[str]] = {} + for start in range(0, len(member_ids), 400): + chunk = member_ids[start:start + 400] + marks = ",".join("?" for _ in chunk) + support_sql = ( + "SELECT endpoint, memory_id FROM (" + "SELECT relation.src AS endpoint, support.memory_id FROM edges relation " + "JOIN edge_supports support ON support.edge_id=relation.id " + f"WHERE relation.workspace_id=? AND relation.src IN ({marks}) " + "AND relation.valid_to IS NULL AND relation.expired_at IS NULL " + "AND support.valid_to IS NULL AND support.expired_at IS NULL " + "UNION ALL " + "SELECT relation.dst AS endpoint, support.memory_id FROM edges relation " + "JOIN edge_supports support ON support.edge_id=relation.id " + f"WHERE relation.workspace_id=? AND relation.dst IN ({marks}) " + "AND relation.valid_to IS NULL AND relation.expired_at IS NULL " + "AND support.valid_to IS NULL AND support.expired_at IS NULL)" + ) + rows = self.store.conn.execute( + support_sql, (wid, *chunk, wid, *chunk) + ).fetchall() + for row in rows: + canonical_id = member_to_canonical.get(str(row["endpoint"]), "") + if canonical_id: + seen_supports.setdefault(canonical_id, set()).add( + str(row["memory_id"]) + ) + support_counts.update({key: len(value) for key, value in seen_supports.items()}) + entity_results = [] + for canonical_id in selected_canonical_ids: + rows = members_by_canonical.get(canonical_id) or matched_by_canonical[canonical_id] + best = min(rows, key=lambda row: ( + 0 if row["normalized_name"] == needle else + 1 if str(row["normalized_name"]).startswith(needle) else 2, + len(str(row["normalized_name"])), str(row["normalized_name"]), row["id"], + )) + aliases = sorted({str(row["name"]) for row in rows}, key=lambda value: ( + normalize_entity_name(value), value, + )) + types = Counter(str(row.get("etype") or "person_or_concept") for row in rows) + entity_results.append({ + "id": canonical_id, "label": best["name"], "kind": "entity", + "type": min(types, key=lambda value: (-types[value], value)), + "aliases": aliases, + "repo_ids": sorted({str(row["repo_id"]) for row in rows if row.get("repo_id")}), + "support_count": int(support_counts.get(canonical_id, 0)), + }) + system_results = [{ + "id": item["id"], "label": f"{item['label']} System", "kind": "system", + "anchor_id": item["id"], + "member_count": len(members_by_canonical.get(item["id"], [])) or 1, + "mass": float(item["support_count"]), + } for item in entity_results] + + memories = [] + repositories = [] + relations_out = [] + code_symbols = [] + if wid: + memory_sql = ( + "SELECT id, title, content, mtype, repo_id FROM memories " + "WHERE workspace_id=? AND (valid_from IS NULL OR valid_from<=?) " + "AND (valid_to IS NULL OR ? dict: + clean_canonical_id = _clean_text( + canonical_id, field="canonical_id", max_chars=MAX_NAME_CHARS + ) + (ws, wid, entities, edges, supports, _memories, _memory_links, + _code_memory_links, _index_info) = self._graph_scene_rows( + workspace=workspace, repo=repo, as_of=as_of, + memory_types=memory_types, time_from=time_from, time_to=time_to, + include_weak_cooccurrence=include_weak_cooccurrence, + ) + graph = build_canonical_graph( + entities, edges, supports, + include_weak_cooccurrence=include_weak_cooccurrence, min_support=0, + ) + resolved = graph["member_to_canonical"].get( + clean_canonical_id, clean_canonical_id + ) + node = graph["nodes"].get(resolved) + if node is None: + raise ValidationError( + f"no entity '{clean_canonical_id}' in workspace '{ws}'" + ) + repo_names = {row["id"]: row["name"] for row in self.store.conn.execute( + "SELECT id, name FROM repos WHERE workspace_id=?", (wid,) + ).fetchall()} if wid else {} + relations_out = [] + connected_edge_ids: set[str] = set() + memory_ids: set[str] = set() + for edge in graph["edges"]: + if resolved not in {edge["source"], edge["target"]}: + continue + direction = "outgoing" if edge["source"] == resolved else "incoming" + other_id = edge["target"] if direction == "outgoing" else edge["source"] + relations_out.append({ + **{key: value for key, value in edge.items() + if key not in {"support_memory_ids"} and not key.startswith("_")}, + "direction": direction, "other_id": other_id, + "other_label": graph["nodes"][other_id]["label"], + }) + connected_edge_ids.update(edge["_underlying_edge_ids_all"]) + memory_ids.update(edge["_support_ids_all"]) + support_map: dict[str, dict] = {} + for row in supports: + if row["edge_id"] not in connected_edge_ids: + continue + memory_id = str(row["memory_id"]) + current = support_map.get(memory_id) + if current is None or float(row.get("confidence") or 0.0) > float( + current.get("confidence") or 0.0): + support_map[memory_id] = row + relation_total = len(relations_out) + layer_order = {"causal": 0, "entity": 1, "temporal": 2, "semantic": 3} + relations_out = sorted(relations_out, key=lambda item: ( + item["relation"] == "co_occurs", + layer_order.get(item["layer"], 4), + item["direction"], + -item["strength"], + item["id"], + ))[:GRAPH_ENTITY_RELATION_LIMIT] + evidence = [] + if memory_ids: + ordered_ids = sorted(memory_ids, key=lambda memory_id: ( + -float(support_map.get(memory_id, {}).get("confidence") or 0.0), + memory_id, + ))[:GRAPH_ENTITY_EVIDENCE_LIMIT] + for start in range(0, len(ordered_ids), 500): + chunk = ordered_ids[start:start + 500] + marks = ",".join("?" for _ in chunk) + for memory in self.store.conn.execute( + "SELECT id, title, content, mtype, valid_from, valid_to, ingested_at, " + "expired_at, provenance FROM memories WHERE workspace_id=? " + "AND id IN (" + marks + ") " + "ORDER BY id", (wid, *chunk) + ).fetchall(): + support = support_map.get(memory["id"], {}) + try: + memory_provenance = json.loads(memory["provenance"] or "{}") + except (TypeError, ValueError, RecursionError): + memory_provenance = {} + if not isinstance(memory_provenance, dict): + memory_provenance = {} + evidence.append({ + "memory_id": memory["id"], "title": memory["title"] or "", + "excerpt": str(memory["content"] or "")[:500], + "memory_type": memory["mtype"], + "source_kind": support.get("source_kind", "legacy_unknown"), + "confidence": float(support.get("confidence", 0.5)), + "valid_from": memory["valid_from"], "valid_to": memory["valid_to"], + "ingested_at": memory["ingested_at"], "expired_at": memory["expired_at"], + "provenance": memory_provenance, + }) + evidence.sort(key=lambda item: ( + -float(item["confidence"]), + -float(item.get("valid_from") or item.get("ingested_at") or 0.0), + item["memory_id"], + )) + member_ids = node["member_ids"] + history_filter = ( + "workspace_id=? AND (valid_to IS NOT NULL OR expired_at IS NOT NULL) " + "AND (src IN (SELECT id FROM entities WHERE workspace_id=? AND canonical_id=?) " + "OR dst IN (SELECT id FROM entities WHERE workspace_id=? AND canonical_id=?))" + ) + history_params = (wid, wid, resolved, wid, resolved) + history_total = int(self.store.conn.execute( + f"SELECT COUNT(*) AS n FROM edges WHERE {history_filter}", history_params + ).fetchone()["n"]) + history = [dict(row) for row in self.store.conn.execute( + "SELECT id, src, dst, relation, layer, weight, valid_from, valid_to, " + "ingested_at, expired_at FROM edges WHERE " + history_filter + " " + "ORDER BY COALESCE(valid_to, expired_at, valid_from, ingested_at) DESC, id DESC " + "LIMIT ?", + (*history_params, GRAPH_ENTITY_HISTORY_LIMIT), + ).fetchall()] + for item in history: + item["event"] = "Relation invalidated" if item.get("valid_to") is not None else ( + "Relation expired" + ) + return { + "workspace": ws, "canonical_id": resolved, "label": node["label"], + "type": node["type"], "member_ids": member_ids, + "aliases": node.get("aliases", []), + "repositories": [{"id": repo_id, "name": repo_names.get(repo_id, repo_id)} + for repo_id in node["repo_ids"]], + "mass": {key: node[key] for key in ( + "mass_score", "gravity_mass", "visual_radius", "weighted_degree", + "pagerank", "support_count", "anchor_role", "core_affinity" + )}, + "relations": relations_out, + "evidence": evidence, "history": history, + "totals": { + "relations": relation_total, + "evidence": len(memory_ids), + "history": history_total, + }, + "truncation": { + "relations": relation_total > len(relations_out), + "evidence": len(memory_ids) > len(evidence), + "history": history_total > len(history), + }, + "as_of": as_of, + } + + def graph_path(self, source: str, target: str, *, workspace: str, + repo: Optional[str] = None, as_of: Optional[float] = None, + memory_types: Optional[list[str]] = None, + time_from: Optional[float] = None, + time_to: Optional[float] = None, + max_hops: int = 8, max_visits: int = 10_000, + include_weak_cooccurrence: bool = False) -> dict: + clean_source = _clean_text( + source, field="source", max_chars=MAX_NAME_CHARS + ) + clean_target = _clean_text( + target, field="target", max_chars=MAX_NAME_CHARS + ) + try: + clean_max_hops = int(max_hops) + clean_max_visits = int(max_visits) + except (TypeError, ValueError, OverflowError): + raise ValidationError("max_hops and max_visits must be integers") + if not 1 <= clean_max_hops <= 8: + raise ValidationError("max_hops must be between 1 and 8") + if not 1 <= clean_max_visits <= 50_000: + raise ValidationError("max_visits must be between 1 and 50000") + (ws, _wid, entities, edges, supports, _memories, _memory_links, + _code_memory_links, _index_info) = self._graph_scene_rows( + workspace=workspace, repo=repo, as_of=as_of, + memory_types=memory_types, time_from=time_from, time_to=time_to, + include_weak_cooccurrence=include_weak_cooccurrence, + ) + graph = build_canonical_graph( + entities, edges, supports, + include_weak_cooccurrence=include_weak_cooccurrence, min_support=0, + ) + result = strongest_path( + graph, clean_source, clean_target, max_hops=clean_max_hops, + max_visits=clean_max_visits, + ) + return { + "workspace": ws, "source": clean_source, "target": clean_target, **result, + } + def graph(self, *, workspace: str, limit: int = 2000, layers: Optional[list] = None, include_code: bool = False, repo: Optional[str] = None, backfill: bool = True) -> dict: @@ -2630,6 +4601,7 @@ def graph(self, *, workspace: str, limit: int = 2000, wid = self._lookup_workspace(ws) if wid is None: return empty_graph(ws) + self._assert_graph_index_ready(wid) limit = max(1, min(5000, int(limit))) conn = self.store.conn ents = conn.execute( diff --git a/engraphis/static/dashboard.css b/engraphis/static/dashboard.css new file mode 100644 index 0000000..f94a0c1 --- /dev/null +++ b/engraphis/static/dashboard.css @@ -0,0 +1,746 @@ +:root{color-scheme:dark;--color-canvas:#0e1014;--color-panel:#15181e;--color-raised:#1d2129;--color-inset:#252a34;--color-border:#303642;--color-border-strong:#46505f;--color-text:#e7e9ee;--color-text-muted:#a7adb8;--color-text-dim:#7e8795;--color-accent:#8c83e8;--color-accent-strong:#6257c4;--color-on-accent:#f8f7ff;--color-accent-bg:rgba(140,131,232,.12);--color-success:#58b882;--color-warning:#d7a84b;--color-danger:#df7478;--color-info:#6f9fd8;--color-teal:#5aafb3;--color-overlay:rgba(5,7,10,.72);--color-success-bg:rgba(88,184,130,.12);--color-warning-bg:rgba(215,168,75,.12);--color-danger-bg:rgba(223,116,120,.12);--color-info-bg:rgba(111,159,216,.12);--graph-edge:140,148,164;--entity-concept:#8c83e8;--entity-mention:#5aafb3;--entity-hashtag:#d7a84b;--entity-email:#6f9fd8;--entity-organization:#58b882;--entity-location:#df7478;--space-1:4px;--space-2:8px;--space-3:12px;--space-4:16px;--space-5:20px;--space-6:24px;--space-7:28px;--space-8:32px;--radius-sm:4px;--radius:6px;--radius-lg:10px;--shadow-dialog:0 16px 48px rgba(0,0,0,.34);--sidebar-w:220px;--font:"Aptos","Segoe UI Variable","Segoe UI",-apple-system,BlinkMacSystemFont,Helvetica,Arial,sans-serif;--mono:"Cascadia Code","SFMono-Regular",Consolas,"Liberation Mono",Menlo,monospace;--transition:.16s cubic-bezier(.25,.8,.25,1)} +*{margin:0;padding:0;box-sizing:border-box}html{font-size:16px}body{font-family:var(--font);background:var(--bg);color:var(--text);-webkit-font-smoothing:antialiased;overflow:hidden} +::-webkit-scrollbar{width:6px;height:6px}::-webkit-scrollbar-track{background:transparent}::-webkit-scrollbar-thumb{background:var(--surface3);border-radius:3px}::-webkit-scrollbar-thumb:hover{background:var(--border-light)} +a{color:var(--accent);text-decoration:none}button{font-family:inherit;cursor:pointer} +.app{display:flex;height:100vh}.sidebar{width:var(--sidebar-w);background:var(--surface);border-right:1px solid var(--border);display:flex;flex-direction:column;flex-shrink:0;z-index:50}.main{flex:1;display:flex;flex-direction:column;min-width:0}.content{flex:1;overflow-y:auto;padding:0} +.brand{padding:11px 14px 9px;border-bottom:1px solid var(--border);display:flex;align-items:center;gap:10px}.brand-mark{width:20px;height:20px;flex-shrink:0}.brand-name{font-size:16px;font-weight:600;letter-spacing:-.2px}.brand-sub{font-size:12px;color:var(--text-dim);letter-spacing:.5px;text-transform:uppercase} +.vault-switcher{padding:6px 10px;border-bottom:1px solid var(--border);cursor:pointer;display:flex;align-items:center;gap:8px;font-size:12px;color:var(--text-muted);transition:all var(--transition)}.vault-switcher:hover{background:var(--surface2);color:var(--text)}.vault-dot{width:8px;height:8px;border-radius:50%;flex-shrink:0}.vault-switcher-name{flex:1;overflow:hidden;text-overflow:ellipsis;white-space:nowrap} +.nav{flex:1;padding:5px 6px;overflow-y:auto}.nav-section{font-size:12px;font-weight:600;color:var(--text-dim);text-transform:uppercase;letter-spacing:.9px;padding:9px 8px 3px}.nav-section:first-child{padding-top:3px}.nav-item{display:flex;width:100%;align-items:center;gap:10px;padding:5px 8px;border:0;border-radius:var(--radius-sm);background:transparent;cursor:pointer;color:var(--text-muted);transition:all var(--transition);font:inherit;text-align:left;user-select:none}.nav-item:hover{background:var(--surface2);color:var(--text)}.nav-item.active{background:var(--accent-bg);color:var(--accent)}.nav-icon{width:15px;height:15px;flex-shrink:0;color:var(--text-dim)}.nav-item:hover .nav-icon{color:var(--text-muted)}.nav-badge{margin-left:auto;font-size:12px;color:var(--text-dim);font-variant-numeric:tabular-nums} +.sidebar-footer{padding:6px 12px;border-top:1px solid var(--border);font-size:12px;color:var(--text-dim);display:flex;align-items:center;gap:6px}.pulse-dot{width:7px;height:7px;border-radius:50%;background:var(--green)} +.topbar{height:42px;border-bottom:1px solid var(--border);display:flex;align-items:center;padding:0 20px;gap:12px;flex-shrink:0;background:var(--surface)}.topbar-title{font-size:24px;font-weight:600;margin:0;letter-spacing:-.2px;flex:0 0 auto;outline:none}.topbar-sub{font-size:12px;color:var(--text-dim);min-width:0;overflow:hidden;text-overflow:ellipsis;white-space:nowrap}.topbar-lock{font-size:12px;flex:0 0 auto}.topbar-lock:empty{display:none}.topbar-spacer{flex:1}kbd{font-family:var(--mono);font-size:12px;padding:1px 5px;background:var(--surface3);border-radius:3px;border:1px solid var(--border)} +.view{display:none;padding:var(--space-6) var(--space-7);max-width:1200px}.view.active{display:block}.view-head{margin-bottom:var(--space-5)} +.card{background:var(--surface);border:1px solid var(--border);border-radius:var(--radius);padding:16px;margin-bottom:12px}.card-head{font-size:12px;font-weight:600;color:var(--text-dim);text-transform:uppercase;letter-spacing:.8px;margin-bottom:2px;display:flex;align-items:center;gap:8px}.card-head .count{margin-left:auto;font-size:12px;color:var(--text-dim);font-weight:400;text-transform:none} +.stat-grid{display:grid;grid-template-columns:repeat(auto-fill,minmax(160px,1fr));gap:10px;margin-bottom:18px}.stat{background:var(--surface);border:1px solid var(--border);border-radius:var(--radius);padding:14px;position:relative;overflow:hidden;transition:border-color var(--transition)}.stat:hover{border-color:var(--border-light)}.stat-val{font-size:32px;font-weight:600;letter-spacing:-.5px;font-variant-numeric:tabular-nums}.stat-lbl{font-size:12px;color:var(--text-dim);text-transform:uppercase;letter-spacing:.6px;margin-top:2px}.stat-bar{position:absolute;bottom:0;left:0;height:2px;transition:width .6s ease} +.cols-2{display:grid;grid-template-columns:1fr 1fr;gap:12px}.cols-3{display:grid;grid-template-columns:1fr 1fr 1fr;gap:12px}@media(max-width:1000px){.cols-2,.cols-3{grid-template-columns:1fr}} +.tbl{width:100%;border-collapse:collapse;font-size:16px}.tbl th{text-align:left;padding:7px 10px;color:var(--text-dim);font-weight:500;font-size:12px;text-transform:uppercase;letter-spacing:.4px;border-bottom:1px solid var(--border)}.tbl td{padding:9px 10px;border-bottom:1px solid var(--surface2);font-variant-numeric:tabular-nums}.tbl tbody tr:last-child td{border-bottom:none}.tbl tr{cursor:pointer;transition:background var(--transition)}.tbl tr:hover{background:var(--surface2)}.truncate{max-width:240px;overflow:hidden;text-overflow:ellipsis;white-space:nowrap} +.pill{display:inline-flex;align-items:center;padding:2px 7px;border-radius:10px;font-size:12px;font-weight:500}.pill-accent{background:var(--accent-bg);color:var(--accent)}.pill-green{background:var(--color-success-bg);color:var(--green)}.pill-blue{background:var(--color-info-bg);color:var(--blue)}.pill-amber{background:var(--color-warning-bg);color:var(--amber)}.pill-red{background:var(--color-danger-bg);color:var(--red)}.pill-cyan{background:var(--color-info-bg);color:var(--cyan)}.pill-muted{background:var(--surface2);color:var(--text-muted)} +.field{margin-bottom:12px}.field-lbl{display:block;font-size:12px;color:var(--text-muted);margin-bottom:5px;font-weight:500}.input,.textarea,.select{width:100%;padding:8px 11px;background:var(--surface2);border:1px solid var(--border);border-radius:var(--radius-sm);color:var(--text);font-size:16px;font-family:inherit;transition:border-color var(--transition)}.input:focus,.textarea:focus,.select:focus{outline:none;border-color:var(--accent);box-shadow:0 0 0 3px var(--accent-bg)}.textarea{min-height:120px;resize:vertical;font-family:var(--mono);font-size:12px;line-height:1.6}.field-hint{font-size:12px;color:var(--text-dim);margin-top:3px}.select-sm{padding:4px 6px;font-size:12px} +.btn{display:inline-flex;align-items:center;gap:6px;padding:7px 14px;border-radius:var(--radius-sm);border:1px solid transparent;font-size:16px;font-weight:500;transition:background-color var(--transition),border-color var(--transition),color var(--transition);white-space:nowrap}.btn-primary{background:var(--accent-dim);color:var(--color-on-accent);border-color:var(--accent-dim)}.btn-primary:hover{background:var(--accent);border-color:var(--accent)}.btn-ghost{background:var(--surface2);color:var(--text-muted);border-color:var(--border)}.btn-ghost:hover{background:var(--surface3);color:var(--text)}.btn-danger{background:var(--color-danger-bg);color:var(--red);border-color:var(--red)}.btn-danger:hover{background:var(--color-danger-bg);border-color:var(--color-text)}.btn-sm{padding:5px 10px;font-size:12px}.btn:disabled{opacity:.4;cursor:not-allowed} +.search-row{display:flex;gap:8px;margin-bottom:14px}.search-row .input{flex:1} +.browser{display:flex;height:calc(100vh - 84px)}.browser-list{width:320px;border-right:1px solid var(--border);overflow-y:auto;flex-shrink:0}.browser-reader{flex:1;overflow-y:auto;padding:var(--space-6) var(--space-7)} +.mem-card{padding:10px 12px;border-bottom:1px solid var(--surface2);cursor:pointer;transition:background var(--transition)}.mem-card:hover{background:var(--surface2)}.mem-card.active{background:var(--surface2)}.mem-card-title{font-size:16px;font-weight:500;margin-bottom:2px;overflow:hidden;text-overflow:ellipsis;white-space:nowrap}.mem-card-meta{display:flex;align-items:center;gap:5px;font-size:12px;color:var(--text-dim)}.ret-bar{display:inline-block;width:36px;height:3px;background:var(--surface3);border-radius:1.5px;overflow:hidden;vertical-align:middle}.ret-fill{height:100%;border-radius:1.5px}.ret-h{background:var(--green)}.ret-m{background:var(--amber)}.ret-l{background:var(--red)} +#mem-cards .mem-drag-handle{flex:0 0 auto;margin-top:1px;color:var(--text-dim);cursor:grab;line-height:1;letter-spacing:-2px;user-select:none}#mem-cards .mem-card.dragging{opacity:.4}#mem-cards .mem-card.drag-over-top{box-shadow:inset 0 2px 0 0 var(--accent)}#mem-cards .mem-card.drag-over-bottom{box-shadow:inset 0 -2px 0 0 var(--accent)} +.md-reader{max-width:74ch}.md-reader h1{font-size:32px;font-weight:600;margin-bottom:6px;letter-spacing:-.3px}.md-reader h2{font-size:24px;font-weight:600;margin:20px 0 8px}.md-reader h3{font-size:16px;font-weight:600;margin:16px 0 6px}.md-reader p{margin-bottom:10px;line-height:1.7}.md-reader ul,.md-reader ol{margin:0 0 10px 20px;line-height:1.7}.md-reader li{margin-bottom:3px}.md-reader code{font-family:var(--mono);font-size:12px;background:var(--surface2);padding:2px 5px;border-radius:3px;color:var(--cyan)}.md-reader pre{background:var(--surface2);padding:12px;border-radius:var(--radius-sm);overflow-x:auto;margin-bottom:10px;border:1px solid var(--border)}.md-reader pre code{background:none;padding:0;color:var(--text)}.md-reader blockquote{border-left:2px solid var(--border-light);padding-left:14px;color:var(--text-muted);margin-bottom:10px}.md-reader table{width:100%;border-collapse:collapse;margin-bottom:10px;font-size:16px}.md-reader th,.md-reader td{padding:6px 8px;border:1px solid var(--border)}.md-reader th{background:var(--surface2);font-weight:600}.md-reader hr{border:none;border-top:1px solid var(--border);margin:16px 0}.md-meta{display:flex;gap:8px;flex-wrap:wrap;margin-bottom:16px;padding-bottom:12px;border-bottom:1px solid var(--border)} +.editor-wrap{display:flex;flex-direction:column;height:calc(100vh - 84px)}.editor-toolbar{display:flex;gap:6px;padding:8px 12px;border-bottom:1px solid var(--border);background:var(--surface);align-items:center}.editor-body{flex:1;display:flex;overflow:hidden}.editor-edit{flex:1;border:none;background:var(--bg);color:var(--text);font-family:var(--mono);font-size:16px;line-height:1.7;padding:var(--space-5) var(--space-6);resize:none;outline:none}.editor-preview{flex:1;overflow-y:auto;padding:var(--space-5) var(--space-6);border-left:1px solid var(--border)} +.dropzone{border:1px dashed var(--border-light);border-radius:var(--radius);padding:var(--space-8);text-align:center;cursor:pointer;transition:all var(--transition)}.dropzone:hover,.dropzone.drag{border-color:var(--accent);background:var(--accent-bg)}.dropzone-icon{width:28px;height:28px;margin:0 auto 8px;color:var(--text-dim)} +.recall-card{background:var(--surface);border:1px solid var(--border);border-radius:var(--radius);padding:14px;margin-bottom:8px;transition:border-color var(--transition)}.recall-card:hover{border-color:var(--border-light)}.recall-head{display:flex;justify-content:space-between;align-items:flex-start;margin-bottom:8px}.recall-title{font-size:16px;font-weight:500}.recall-scores{display:flex;gap:14px;font-size:12px;color:var(--text-dim)}.recall-body{font-size:16px;line-height:1.6;color:var(--text-muted);white-space:pre-wrap;max-height:180px;overflow-y:auto} +.chat-wrap{display:flex;flex-direction:column;height:calc(100vh - 84px)}.chat-msgs{flex:1;overflow-y:auto;padding:var(--space-4) var(--space-6)}.chat-msg{margin-bottom:16px;max-width:760px}.chat-role{font-size:12px;font-weight:600;text-transform:uppercase;letter-spacing:.5px;margin-bottom:3px}.chat-role.user{color:var(--blue)}.chat-role.assistant{color:var(--accent)}.chat-bubble{padding:12px 16px;border-radius:var(--radius);font-size:16px;line-height:1.7;white-space:pre-wrap}.chat-bubble.user{background:var(--surface2);border:1px solid var(--border)}.chat-bubble.assistant{background:var(--surface);border:1px solid var(--border)}.chat-input-row{display:flex;gap:8px;padding:10px var(--space-6) var(--space-4);border-top:1px solid var(--border);background:var(--surface)}.chat-input-row .textarea{min-height:48px;max-height:96px;flex:1} +.thought{background:var(--surface);border:1px solid var(--border);border-radius:var(--radius);padding:14px;margin-bottom:8px}.thought-meta{font-size:12px;color:var(--text-dim);margin-bottom:8px;display:flex;gap:6px;align-items:center}.thought-field{margin-bottom:6px}.thought-key{font-size:12px;font-weight:600;text-transform:uppercase;letter-spacing:.4px;color:var(--text-dim)}.thought-val{font-size:16px;line-height:1.6;margin-top:2px} +.graph-wrap{display:flex;gap:12px;height:calc(100vh - 120px)}.graph-canvas{flex:1;background:var(--surface);border:1px solid var(--border);border-radius:var(--radius);position:relative;overflow:hidden}.graph-sidebar{width:260px;flex-shrink:0;overflow-y:auto}.graph-stats{position:absolute;top:10px;right:10px;z-index:10;font-size:12px;color:var(--text-dim);background:var(--surface2);padding:5px 10px;border-radius:var(--radius-sm);border:1px solid var(--border)} +.force-graph-container{position:relative;width:100%;height:100%}.force-graph-container canvas{display:block;width:100%;height:100%;user-select:none;outline:none;-webkit-tap-highlight-color:transparent}.force-graph-container .clickable{cursor:pointer}.force-graph-container .grabbable{cursor:grab}.force-graph-container .grabbable:active{cursor:grabbing} +#view-graph{max-width:1500px}.graph-layout{display:grid;grid-template-columns:minmax(0,1fr) minmax(0,560px);gap:14px;align-items:stretch}.graph-main{min-width:0}.graph-network{position:relative;height:600px;background:var(--surface);border:1px solid var(--border);border-radius:var(--radius);overflow:hidden}.graph-controls-column{min-width:0;display:grid;grid-template-columns:repeat(2,minmax(0,1fr));grid-template-rows:1fr auto;gap:12px;align-items:stretch;align-self:stretch} +/* The rail matches the canvas height exactly. Rather than parking that slack in one card + as dead space, the two tall cards absorb it and hand it to their own children via + `space-between` — the sliders and the ranked list simply breathe more. Legend keeps its + natural height (`auto` row) so it stays a compact bar. */ +.graph-controls-column>.card:first-child{display:flex;flex-direction:column;justify-content:space-between} +.graph-side-stack>.card:first-child{flex:1;display:flex;flex-direction:column} +/* Rows absorb the slack by growing (capped), not by opening gaps: the whole row is the + click/hover target, so taller rows are a real gain. The cap keeps a sparse top-N from + stretching into absurdly tall bands — leftover then sits harmlessly at the card's foot. */ +#graph-top{flex:1;display:flex;flex-direction:column} +#graph-top>.gtop-row{flex:1 1 auto;max-height:34px} +.graph-legend-card{grid-column:1/-1;padding:8px 10px} +/* The other cards are ~250px wide, so a right-floated `.count` still reads as belonging to + their title. This bar is ~540px, which strands the number in open space — keep it beside + the label (and units on it) so it reads as "Legend · 3 types" rather than a loose digit. */ +.graph-legend-card .card-head .count{margin-left:0} +/* Right rail is two equal-height tracks: Controls, and a stack of Top connected + Stats. + `stretch` matches the tracks and `margin-top:auto` pins Stats to the bottom, so Stats + stays flush with the bottom of Controls no matter how the entity list grows. */ +.graph-side-stack{min-width:0;display:flex;flex-direction:column;gap:12px}.graph-side-stack>.card:last-child{margin-top:auto}.graph-controls-column .card{margin-bottom:0}.graph-legend-grid{display:flex;flex-wrap:wrap;column-gap:22px;row-gap:4px;align-content:start;min-height:0;overflow:auto}.graph-stats-grid{display:grid;grid-template-columns:repeat(2,minmax(0,1fr));column-gap:18px;row-gap:2px;font-size:12px}@media(max-width:1280px){.graph-layout{grid-template-columns:minmax(0,1fr) 268px}.graph-controls-column{display:flex;flex-direction:column}}@media(max-width:900px){.graph-layout{grid-template-columns:1fr}.graph-controls-column{display:grid;grid-template-columns:repeat(2,minmax(0,1fr))}.graph-network{height:520px}}@media(max-width:640px){.graph-controls-column{grid-template-columns:1fr}.graph-network{height:440px}} +/* Legend entries are self-contained chips: dot, name, then the count immediately after it. + The count used to be `margin-left:auto`, which pinned it to the far edge of its grid + column — with a short label like "Mention" that left ~100px of air between name and + number, so each count read as if it belonged to the *next* entry along the bar. */ +.gtype-row{flex:1 1 auto;min-width:0;display:flex;align-items:center;gap:6px;padding:2px 0;font-size:12px}.gtype-dot{width:9px;height:9px;border-radius:50%;flex-shrink:0}.gtype-name{text-transform:capitalize;color:var(--text-muted);overflow:hidden;text-overflow:ellipsis;white-space:nowrap}.gtype-count{color:var(--text-muted);font-variant-numeric:tabular-nums;font-size:12px;font-weight:500;background:var(--surface2);border-radius:999px;padding:0 7px;line-height:18px;flex-shrink:0} +.gtop-row{display:flex;align-items:center;gap:7px;padding:5px 4px;border-radius:6px;cursor:pointer;transition:background var(--transition)}.gtop-row:hover{background:var(--surface2)}.gtop-row:hover .gtop-name{color:var(--text)}.gtop-rank{width:13px;flex-shrink:0;font-size:12px;color:var(--text-dim);text-align:right;font-variant-numeric:tabular-nums}.gtop-dot{width:7px;height:7px;border-radius:50%;flex-shrink:0}.gtop-name{width:66px;font-size:12px;color:var(--text-muted);overflow:hidden;text-overflow:ellipsis;white-space:nowrap}.gtop-bar{flex:1;height:5px;background:var(--surface2);border-radius:3px;overflow:hidden}.gtop-bar-fill{display:block;height:100%;border-radius:3px;transition:width .5s cubic-bezier(.4,0,.2,1)}.gtop-n{width:20px;text-align:right;font-size:12px;color:var(--text-dim);font-variant-numeric:tabular-nums} +.tl-item{display:flex;gap:12px;padding:8px 0;border-bottom:1px solid var(--surface2)}.tl-time{font-size:12px;color:var(--text-dim);white-space:nowrap;min-width:120px;font-variant-numeric:tabular-nums}.tl-body{flex:1;font-size:16px;line-height:1.5} +.empty{text-align:center;padding:50px var(--space-5);color:var(--text-dim)}.empty-icon{width:36px;height:36px;margin:0 auto 12px;opacity:.4} +.spinner{width:18px;height:18px;border:2px solid var(--surface3);border-top-color:var(--accent);border-radius:50%;animation:spin .7s linear infinite}@keyframes spin{to{transform:rotate(360deg)}} +.toast{position:fixed;top:14px;right:14px;padding:9px 14px;border-radius:var(--radius-sm);font-size:16px;z-index:400;opacity:0;transform:translateY(-6px);transition:all .25s;pointer-events:none}.toast.show{opacity:1;transform:translateY(0)}.toast-ok{background:rgba(74,222,128,.1);border:1px solid rgba(74,222,128,.3);color:var(--green)}.toast-err{background:rgba(248,113,113,.1);border:1px solid rgba(248,113,113,.3);color:var(--red)} +.cfg-row{display:flex;justify-content:space-between;padding:6px 0;border-bottom:1px solid var(--surface2);font-size:16px}.cfg-key{color:var(--text-muted)}.cfg-val{font-family:var(--mono);font-size:12px} +.vault-card{background:var(--surface);border:1px solid var(--border);border-radius:var(--radius);padding:16px;cursor:pointer;transition:all var(--transition);position:relative}.vault-card:hover{border-color:var(--border-light)}.vault-card.active{border-color:var(--accent);box-shadow:0 0 0 1px var(--accent-dim)}.vault-card-name{font-size:16px;font-weight:600;margin-bottom:4px;display:flex;align-items:center;gap:8px}.vault-card-desc{font-size:12px;color:var(--text-muted);margin-bottom:8px}.vault-card-stats{display:flex;gap:12px;font-size:12px;color:var(--text-dim)}.vault-card-actions{position:absolute;top:12px;right:12px;display:flex;gap:4px;opacity:0;transition:opacity var(--transition)}.vault-card:hover .vault-card-actions{opacity:1} +.dup-card{background:var(--surface);border:1px solid var(--border);border-radius:var(--radius);padding:12px;margin-bottom:8px}.dup-pair{display:flex;gap:12px}.dup-side{flex:1}.dup-side-title{font-size:12px;font-weight:500;margin-bottom:3px}.dup-side-content{font-size:12px;color:var(--text-muted);line-height:1.5;max-height:60px;overflow:hidden} +.ctx-preview{background:var(--bg);border:1px solid var(--border);border-radius:var(--radius);padding:14px;font-family:var(--mono);font-size:12px;line-height:1.6;white-space:pre-wrap;max-height:400px;overflow-y:auto;color:var(--text-muted)} +.cmd-overlay{position:fixed;inset:0;background:rgba(0,0,0,.5);z-index:200;display:none;align-items:flex-start;justify-content:center;padding-top:100px}.cmd-overlay.show{display:flex}.cmd-box{width:460px;background:var(--surface);border:1px solid var(--border-light);border-radius:var(--radius-lg);box-shadow:0 12px 40px rgba(0,0,0,.5);overflow:hidden}.cmd-input{width:100%;padding:13px 16px;background:none;border:none;border-bottom:1px solid var(--border);color:var(--text);font-size:16px;font-family:inherit;outline:none}.cmd-list{max-height:300px;overflow-y:auto}.cmd-item{padding:9px 16px;cursor:pointer;display:flex;align-items:center;gap:10px;font-size:16px;transition:background var(--transition)}.cmd-item:hover,.cmd-item.sel{background:var(--surface2)} +.progress-bar{width:100%;height:4px;background:var(--surface3);border-radius:2px;overflow:hidden;margin-top:8px}.progress-fill{height:100%;background:var(--accent);border-radius:2px;transition:width .3s ease} +.import-result{padding:8px 12px;border-bottom:1px solid var(--surface2);font-size:12px;display:flex;align-items:center;gap:8px}.import-status-ok{color:var(--green)}.import-status-err{color:var(--red)} +body[data-theme=light]{color-scheme:light;--color-canvas:#f4f5f7;--color-panel:#fbfbfc;--color-raised:#eceff3;--color-inset:#e2e6eb;--color-border:#cfd4dc;--color-border-strong:#aeb6c2;--color-text:#20242b;--color-text-muted:#555e6b;--color-text-dim:#707a88;--color-accent:#5547b8;--color-accent-strong:#463a9c;--color-on-accent:#f8f7ff;--color-success:#26744c;--color-warning:#8a5b13;--color-danger:#a63f49;--color-info:#315f9b;--color-teal:#28737a;--color-overlay:rgba(30,34,40,.52);--color-success-bg:rgba(38,116,76,.1);--color-warning-bg:rgba(138,91,19,.1);--color-danger-bg:rgba(166,63,73,.1);--color-info-bg:rgba(49,95,155,.1);--graph-edge:102,111,126;--entity-concept:#5547b8;--entity-mention:#28737a;--entity-hashtag:#8a5b13;--entity-email:#315f9b;--entity-organization:#26744c;--entity-location:#a63f49;--color-accent-bg:rgba(85,71,184,.1)} +body[data-theme=midnight]{--color-canvas:#0b1220;--color-panel:#111b2d;--color-raised:#18253b;--color-inset:#21314b;--color-border:#2b3c58;--color-border-strong:#405674;--color-text:#e2e9f2;--color-text-muted:#a2b0c3;--color-text-dim:#7c8da5;--color-accent:#79a6ef;--color-accent-strong:#5687d7;--color-on-accent:#091321;--color-success:#62bf92;--color-warning:#d8ad59;--color-danger:#e47f84;--color-info:#79a6ef;--color-teal:#65b5bd;--color-overlay:rgba(3,9,18,.75);--color-success-bg:rgba(98,191,146,.12);--color-warning-bg:rgba(216,173,89,.12);--color-danger-bg:rgba(228,127,132,.12);--color-info-bg:rgba(121,166,239,.12);--graph-edge:116,137,170;--entity-concept:#9b8eea;--entity-mention:#65b5bd;--entity-hashtag:#d8ad59;--entity-email:#79a6ef;--entity-organization:#62bf92;--entity-location:#e47f84;--color-accent-bg:rgba(121,166,239,.12)} +body[data-theme=solarized]{--color-canvas:#002b36;--color-panel:#073642;--color-raised:#0b414e;--color-inset:#104d59;--color-border:#25515a;--color-border-strong:#3a6971;--color-text:#e1e3d5;--color-text-muted:#adb8ae;--color-text-dim:#8fa29f;--color-accent:#68b4e4;--color-accent-strong:#3289bf;--color-on-accent:#002b36;--color-success:#aabd3d;--color-warning:#d0a12e;--color-danger:#f58a82;--color-info:#68b4e4;--color-teal:#48b4aa;--color-overlay:rgba(0,25,31,.76);--color-success-bg:rgba(170,189,61,.12);--color-warning-bg:rgba(208,161,46,.12);--color-danger-bg:rgba(245,138,130,.12);--color-info-bg:rgba(104,180,228,.12);--graph-edge:113,143,142;--entity-concept:#9a8adf;--entity-mention:#48b4aa;--entity-hashtag:#d0a12e;--entity-email:#68b4e4;--entity-organization:#aabd3d;--entity-location:#f58a82;--color-accent-bg:rgba(104,180,228,.12)} +body[data-theme=sepia]{color-scheme:light;--color-canvas:#f1ead9;--color-panel:#f8f2e4;--color-raised:#eae0ca;--color-inset:#dfd1b5;--color-border:#cbbd9f;--color-border-strong:#aa9877;--color-text:#3d3326;--color-text-muted:#655845;--color-text-dim:#7e6f59;--color-accent:#925420;--color-accent-strong:#754116;--color-on-accent:#fff8e9;--color-success:#476725;--color-warning:#7d530f;--color-danger:#9d4137;--color-info:#3b638e;--color-teal:#397877;--color-overlay:rgba(57,45,31,.55);--color-success-bg:rgba(71,103,37,.1);--color-warning-bg:rgba(125,83,15,.1);--color-danger-bg:rgba(157,65,55,.1);--color-info-bg:rgba(59,99,142,.1);--graph-edge:133,116,87;--entity-concept:#735ca8;--entity-mention:#397877;--entity-hashtag:#7d530f;--entity-email:#3b638e;--entity-organization:#476725;--entity-location:#9d4137;--color-accent-bg:rgba(146,84,32,.1)} +:root{ + --space-9:40px; + --space-10:48px; + --text-micro:11px; + --text-xs:12px; + --text-sm:14px; + --text-md:16px; + --text-lg:20px; + --text-xl:28px; + --text-2xl:36px; + --shell-sidebar-w:196px; + --shell-header-h:76px; + --mobile-header-h:60px; + --control-h:36px; + --control-h-sm:28px; + --content-max:1320px; + --panel-min:280px; + --compact-column-min:180px; + --dialog-max:860px; + --command-max:460px; + --entity-panel-max:400px; + --mobile-sidebar-max:280px; + --graph-list-max:300px; + --graph-canvas-max:560px; + --graph-canvas-compact:480px; + --graph-canvas-min:300px; + --editor-shell-min:560px; + --editor-pane-min:220px; + --control-label-w:84px; + --rule:1px; + --rule-strong:2px; + --font-display:"Bahnschrift SemiCondensed","Bahnschrift","Aptos Narrow",var(--font); +} +body[data-theme=dark]{--color-accent-strong:#6257c4} +body[data-theme=light]{--color-text-dim:#626b77} +body[data-theme=solarized]{--color-accent-strong:#58a7d8} +body[data-theme=sepia]{--color-text-dim:#6f614d} + +input,select,textarea{min-width:0} +code{overflow-wrap:anywhere} +.import-status-ok::before{content:"✓ ";font-weight:700} +.import-status-err::before{content:"× ";font-weight:700} +body{ + --bg:var(--color-canvas); + --surface:var(--color-panel); + --surface2:var(--color-raised); + --surface3:var(--color-inset); + --border:var(--color-border); + --border-light:var(--color-border-strong); + --text:var(--color-text); + --text-muted:var(--color-text-muted); + --text-dim:var(--color-text-dim); + --accent:var(--color-accent); + --accent-dim:var(--color-accent-strong); + --accent-bg:var(--color-accent-bg); + --green:var(--color-success); + --amber:var(--color-warning); + --red:var(--color-danger); + --blue:var(--color-info); + --cyan:var(--color-teal); + background:var(--color-canvas); + font-size:var(--text-md); + letter-spacing:0; +} + +/* Shell: a compact operations rail and a route ledger, not a dashboard frame. */ +.app{height:100vh;height:100dvh} +.sidebar{width:var(--shell-sidebar-w);background:var(--color-panel);backdrop-filter:none} +.brand{min-height:0;padding:calc(var(--space-3) / 2) var(--space-4);gap:var(--space-2)} +.brand-mark{width:var(--space-4);height:var(--space-4)} +.brand-name{color:var(--color-text);font-family:var(--font-display);font-size:var(--text-md);font-weight:600;letter-spacing:.01em} +.brand-sub{margin-top:var(--space-1);font-family:var(--mono);font-size:var(--text-micro);letter-spacing:.08em} +.vault-switcher{width:100%;padding:calc(var(--space-3) / 2) var(--space-4);border-width:0 0 var(--rule);background:transparent;text-align:left} +.vault-switcher:hover{background:var(--color-raised)} +.vault-switcher-meta{display:flex;flex:1;min-width:0;flex-direction:column;gap:var(--space-1)} +.vault-switcher-label{color:var(--color-text-dim);font-family:var(--mono);font-size:var(--text-micro);letter-spacing:.08em;line-height:1;text-transform:uppercase} +.vault-switcher-name{color:var(--color-text);font-size:var(--text-sm);font-weight:600} +.vault-dot{border-radius:0} +.nav{padding:calc(var(--space-2) / 2) 0} +.nav-section{padding:calc(var(--space-4) / 2) var(--space-4) calc(var(--space-1) / 2);color:var(--color-text-dim);font-family:var(--mono);font-size:var(--text-micro);font-weight:500;letter-spacing:.08em} +.nav-section:first-child{padding-top:calc(var(--space-2) / 2)} +.nav-item{position:relative;gap:var(--space-2);margin:0;padding:calc(var(--space-2) / 2) var(--space-4);border-left:var(--rule-strong) solid transparent;border-radius:0;color:var(--color-text-muted);font-size:var(--text-sm);font-weight:500} +.nav-item:hover{background:var(--color-raised);color:var(--color-text)} +.nav-item.active{border-left-color:var(--color-accent);background:transparent;color:var(--color-text);font-weight:600} +.nav-icon{width:var(--text-sm);height:var(--text-sm)} +.nav-item.active .nav-icon{color:var(--color-accent)} +.nav-badge{font-family:var(--mono);font-size:var(--text-micro)} +.sidebar-footer{padding:calc(var(--space-2) / 2) var(--space-4);font-family:var(--mono);font-size:var(--text-micro)} +.pulse-dot{width:var(--space-2);height:var(--space-2)} + +.topbar{height:var(--shell-header-h);padding:var(--space-3) var(--space-8);gap:var(--space-3);background:var(--color-canvas)} +.topbar-route{display:flex;min-width:0;flex-direction:column;gap:var(--space-1)} +.topbar-kicker{display:flex;align-items:center;gap:var(--space-2);color:var(--color-text-dim);font-family:var(--mono);font-size:var(--text-micro);letter-spacing:.08em;line-height:1;text-transform:uppercase} +.topbar-kicker-separator{color:var(--color-border-strong)} +.topbar-heading{display:flex;min-width:0;align-items:baseline;gap:var(--space-2)} +.topbar-title{flex:0 1 auto;color:var(--color-text);font-family:var(--font-display);font-size:var(--text-xl);font-weight:600;letter-spacing:-.01em;line-height:1} +.topbar-sub{max-width:72ch;color:var(--color-text-muted);font-size:var(--text-xs)} +.topbar-lock{font-size:var(--text-micro)} +.content{background:var(--color-canvas)} +.view{width:100%;max-width:var(--content-max);padding:calc(var(--space-8) / 2) var(--space-9) var(--space-8)} +.view.active{animation:none} +.view-head{margin-bottom:var(--space-5)} + +/* Operational notices stay legible without taking over the route. */ +.system-banner{margin:0!important;padding:var(--space-2) var(--space-8)!important;border-width:0 0 var(--rule)!important;border-radius:0!important;color:var(--color-text)!important;font-size:var(--text-xs)!important} +#sem-banner.system-banner{border-color:var(--color-warning)!important;background:var(--color-warning-bg)!important} +#auth-banner.system-banner{border-color:var(--color-border)!important;background:var(--color-raised)!important} +.system-notice{display:grid;grid-template-columns:minmax(0,1fr) auto;align-items:start;gap:var(--space-3)} +.system-notice details{min-width:0} +.system-notice summary{display:flex;min-height:var(--control-h-sm);align-items:center;gap:var(--space-2);color:var(--color-text);cursor:pointer;list-style:none} +.system-notice summary::-webkit-details-marker{display:none} +.system-notice summary::before{content:"+";color:var(--color-warning);font-family:var(--mono);font-weight:700} +.system-notice details[open] summary::before{content:"−"} +.system-notice-brief{color:var(--color-text-muted)} +.system-notice-detail{max-width:100ch;padding:var(--space-2) 0 var(--space-1) var(--space-5);color:var(--color-text-muted);line-height:1.55} +.system-notice-detail code{padding:var(--space-1);background:var(--color-raised);color:var(--color-text);font-family:var(--mono)} +.system-notice-reason{margin-top:var(--space-2);color:var(--color-text-dim)} + +/* Shared primitives: rules, rows and sections replace nested generic cards. */ +.card,.stat,.recall-card,.vault-card,.thought,.dup-card{box-shadow:none;transition:border-color var(--transition),background-color var(--transition)} +.card{margin-bottom:var(--space-3);padding:var(--space-4) 0 var(--space-5);border:0;border-top:var(--rule) solid var(--color-border);border-radius:0;background:transparent} +.card-head{margin-bottom:var(--space-3);color:var(--color-text-dim);font-family:var(--mono);font-size:var(--text-micro);font-weight:600;letter-spacing:.08em;line-height:1.4} +.card-head .count{color:var(--color-text-muted);font-family:var(--mono);font-size:var(--text-micro)} +.cols-2{gap:var(--space-6)} +.cols-3{gap:var(--space-5)} +.stat{display:flex;min-width:0;flex-direction:column;padding:var(--space-4) var(--space-5);border:0;border-top:var(--rule-strong) solid var(--color-border);border-radius:0;background:transparent} +.stat:hover{border-color:var(--color-border-strong)} +.stat-val{color:var(--color-text);font-family:var(--font-display);font-size:var(--text-xl);font-weight:600;line-height:1.1} +.stat-lbl{order:-1;margin:0 0 var(--space-2);color:var(--color-text-dim);font-family:var(--mono);font-size:var(--text-micro);letter-spacing:.08em} +.recall-card{margin:0;padding:var(--space-4) var(--space-1);border-width:0 0 var(--rule);border-radius:0;background:transparent} +.recall-card:hover{background:var(--color-raised);border-color:var(--color-border-strong)} +.recall-title{font-family:var(--font-display);font-size:var(--text-lg)} +.recall-body{font-size:var(--text-sm)} +.thought,.dup-card,.vault-card{padding:var(--space-4) var(--space-3);border:0;border-top:var(--rule) solid var(--color-border);border-radius:0;background:transparent} +.thought:hover,.dup-card:hover,.vault-card:hover{border-color:var(--color-border-strong)} +.vault-card{cursor:default} +.vault-card.active{border-color:var(--color-accent);box-shadow:inset var(--rule-strong) 0 0 var(--color-accent)} +.vault-card-actions{position:static;display:flex;margin-top:var(--space-3);flex-wrap:wrap;gap:var(--space-1);opacity:1} +.workbench-command{display:flex;align-items:center;gap:var(--space-2);margin-bottom:var(--space-4)!important;padding-bottom:var(--space-4);border-bottom:var(--rule) solid var(--color-border)} +.workbench-command .input{flex:1} +.workbench-command .command-count{width:calc(var(--space-10) + var(--space-9))} +.workbench-command .input,.workbench-command .select{background:transparent;border-color:var(--color-border-strong)} + +.btn{min-height:var(--control-h);padding:var(--space-2) var(--space-3);border-radius:var(--radius-sm);font-size:var(--text-sm)} +.btn-sm{min-height:var(--control-h-sm);padding:var(--space-1) var(--space-2);font-size:var(--text-xs)} +.btn-primary{border-color:var(--color-accent-strong);background:var(--color-accent-strong);color:var(--color-on-accent);box-shadow:none} +.btn-primary:hover{border-color:var(--color-accent);background:var(--color-accent);color:var(--color-on-accent)} +.btn-ghost{background:transparent} +.btn-ghost:hover{background:var(--color-raised)} +.btn:active{transform:none} +.input,.select{min-height:var(--control-h);border-color:var(--color-border);background:var(--color-raised);font-size:var(--text-sm)} +.pill{gap:var(--space-1);padding:var(--space-1) var(--space-2);border-radius:var(--radius-sm);font-family:var(--mono);font-size:var(--text-micro);letter-spacing:.04em} +.pill-muted{padding-inline:0;border-radius:0;background:transparent;color:var(--color-text-muted)} +.pill-green,.toast-ok{background:var(--color-success-bg);color:var(--color-success)} +.pill-amber{background:var(--color-warning-bg);color:var(--color-warning)} +.pill-red,.toast-err{background:var(--color-danger-bg);color:var(--color-danger)} +.pill-blue{background:var(--color-info-bg);color:var(--color-info)} +.pill-green::before{content:"✓";font-weight:700} +.pill-red::before{content:"×";font-weight:700} + +.tbl{font-size:var(--text-sm)} +.tbl th{padding:var(--space-2) var(--space-3);color:var(--color-text-dim);font-family:var(--mono);font-size:var(--text-micro);letter-spacing:.06em} +.tbl td{padding:var(--space-2) var(--space-3)} +.tl-item{gap:var(--space-4);padding:var(--space-3) 0;border-color:var(--color-border)} +.tl-time{font-family:var(--mono);font-size:var(--text-xs)} +.tl-body{font-size:var(--text-sm)} +.tl-item.clickable,.mem-card,.gtop-row{cursor:pointer} +.tl-item.clickable:hover,.mem-card:hover,.gtop-row:hover{background:var(--color-raised)} +#mem-cards{border-top:var(--rule) solid var(--color-border)} +.mem-card{padding:var(--space-3) var(--space-2)} +.mem-card-title{font-family:var(--font-display);font-size:var(--text-md)} +.audit-row{display:flex;align-items:center;gap:var(--space-2);padding:var(--space-2) var(--space-3);border-bottom:var(--rule) solid var(--color-raised);font-size:var(--text-xs)} +.cfg-row,.lic-body .cfg-row{display:flex;justify-content:space-between;padding:var(--space-2) 0;border-bottom:var(--rule) solid var(--color-raised);font-size:var(--text-sm)} +.lic-feat{display:inline-flex;margin:var(--space-1) var(--space-2) var(--space-1) 0;gap:var(--space-1);color:var(--color-text-muted);font-size:var(--text-xs)} +.chain{margin-left:var(--space-2);padding-left:var(--space-4);border-left:var(--rule) solid var(--color-border)} +.chain-item{position:relative;margin-bottom:var(--space-3)} +.chain-item::before{content:"";position:absolute;top:var(--space-1);left:calc(-1 * (var(--space-4) + var(--space-1)));width:var(--space-2);height:var(--space-2);border-radius:50%;background:var(--color-accent)} +.chain-item.old::before{background:var(--color-text-dim)} +.temporal-meta{display:grid;grid-template-columns:repeat(auto-fit,minmax(var(--compact-column-min),1fr));gap:var(--space-2) var(--space-4);margin-top:var(--space-2);font-size:var(--text-xs)} +.temporal-meta>div{min-width:0} +.temporal-meta dt{color:var(--color-text-dim);font-weight:600} +.temporal-meta dd{color:var(--color-text-muted);font-family:var(--mono);overflow-wrap:anywhere} +.diff-body{font-size:var(--text-md);line-height:1.75;white-space:pre-wrap;word-break:break-word} +.diff-ins{border-radius:var(--radius-sm);background:var(--color-success-bg);color:var(--color-success)} +.diff-del{border-radius:var(--radius-sm);background:var(--color-danger-bg);color:var(--color-danger);text-decoration:line-through} +.trial-banner{margin-bottom:var(--space-3);padding:var(--space-2) var(--space-3);border:var(--rule) solid var(--color-accent-strong);border-radius:var(--radius-sm);background:var(--accent-bg);color:var(--color-text);font-size:var(--text-xs)} +.empty{padding:var(--space-6) 0;color:var(--color-text-dim);line-height:1.5;text-align:left} +.empty .btn{margin-top:var(--space-3)} + +/* Route compositions inherit the same ledger geometry. */ +#view-overview{padding-top:calc(var(--space-6) / 2)} +#view-overview .stat-grid{grid-template-columns:minmax(0,1.5fr) repeat(3,minmax(0,1fr));gap:0;margin-bottom:var(--space-8);border-top:var(--rule) solid var(--color-border);border-bottom:var(--rule) solid var(--color-border)} +#view-overview .stat{min-height:calc(var(--space-10) * 2);justify-content:center;border-top:0;border-left:var(--rule) solid var(--color-border)} +#view-overview .stat:first-child{border-left:0} +#view-overview .stat:first-child .stat-val{font-size:var(--text-2xl)} +.overview-ledger-grid{display:grid;grid-template-columns:minmax(0,1.55fr) minmax(var(--panel-min),1fr);gap:var(--space-8)} +#view-overview .card{margin-bottom:0} +#view-settings{padding:var(--space-2) var(--space-5) var(--space-4)} +.settings-grid{display:grid;grid-template-columns:repeat(2,minmax(0,1fr));gap:var(--space-3);align-items:start} +.settings-column{display:flex;min-width:0;flex-direction:column;gap:var(--space-3);container-type:inline-size} +.settings-column:first-child>.card:first-child{border-top-color:var(--color-accent)} +#view-settings .card{margin:0;padding:var(--space-2) 0 calc(var(--space-5) / 2)} +#view-settings .card-head{margin-bottom:calc(var(--space-3) / 2)} +#view-settings .cfg-row,#view-settings .lic-body .cfg-row{padding:calc(var(--space-2) / 2) 0} +#view-settings .empty{padding:calc(var(--space-6) / 2) 0} +#view-settings .cfg-row code{overflow-wrap:anywhere;word-break:break-word} +.settings-account-grid{display:grid;grid-template-columns:minmax(0,1.15fr) minmax(168px,.85fr);gap:var(--space-4);align-items:start} +.settings-account-panel{min-width:0} +.settings-engine-panel{padding-left:var(--space-4);border-left:var(--rule) solid var(--color-raised)} +.settings-account-panel .cfg-row{display:grid;grid-template-columns:max-content minmax(0,1fr);justify-content:start;column-gap:var(--space-3);align-items:center} +.settings-account-panel .cfg-row>:last-child{min-width:0;justify-self:start;text-align:left;overflow-wrap:anywhere} +@container (max-width:460px){ + .settings-account-grid{grid-template-columns:1fr;gap:var(--space-3)} + .settings-engine-panel{padding-top:var(--space-3);padding-left:0;border-top:var(--rule) solid var(--color-raised);border-left:0} +} +#theme-select{width:auto;padding:4px 8px} +#cfg-store{font-family:var(--mono);font-size:12px;color:var(--text-dim)} +#graph-explorer-title{font-size:16px} +#graph-explorer-search{width:min(320px,100%)} +.workspace-actions{justify-content:flex-end} +#ws-new-btn{display:none;flex-shrink:0} +.workspace-source-card{display:none;margin-bottom:14px} +.import-path-row{display:flex;align-items:center;gap:8px;margin-top:10px;flex-wrap:wrap} +.import-path-label{font-size:12px;color:var(--text-dim);white-space:nowrap} +.import-path-input{flex:1;min-width:180px} +.import-pattern-input{width:90px} +.import-derive-row{display:flex;align-items:center;gap:6px;font-size:12px;margin-top:8px} +.import-status{font-size:12px;color:var(--text-dim);margin-top:8px} +.code-import-grid{display:grid;grid-template-columns:1fr 1fr auto;gap:8px} +.code-import-grid-secondary{margin-top:8px} +#auth-overlay{z-index:200} +#auth-overlay .mm-box{max-width:420px} +#auth-body{padding-top:4px} +#folder-overlay .mm-box{max-width:560px} +#action-overlay{z-index:400} +#action-overlay .mm-box{max-width:520px} +.action-error{min-height:1.25rem;color:var(--red)} +.hosted-trial-actions{display:flex;gap:var(--space-2);flex-wrap:wrap;margin-top:var(--space-3)} +.optional-label{font-weight:400} +.folder-access-fieldset{border:0} +#folder-create-status{margin-top:var(--space-2)} +#ws-cards>.cols-2{grid-template-columns:repeat(auto-fit,minmax(var(--panel-min),1fr))} +.dropzone{padding:var(--space-6);border-radius:var(--radius-sm);text-align:left} +.dropzone-icon{margin:0 0 var(--space-2)} +.dropzone-actions{display:flex;justify-content:flex-start;gap:var(--space-2);margin-top:var(--space-3)} + +/* Purposeful containment remains for canvases, inspectors, dialogs and menus. */ +.graph-network{border-radius:var(--radius-sm);background:var(--color-panel)} +.graph-controls-column .card{margin-bottom:0;padding:var(--space-3);border:var(--rule) solid var(--color-border);border-radius:var(--radius-sm);background:var(--color-panel)} +.graph-side-stack{gap:var(--space-3)} +.graph-explorer{margin-top:var(--space-4);padding-top:var(--space-5);border-top:var(--rule) solid var(--color-border)} +.graph-explorer-head{display:flex;align-items:flex-start;justify-content:space-between;gap:var(--space-3);margin-bottom:var(--space-3);flex-wrap:wrap} +.graph-explorer-grid{display:grid;grid-template-columns:minmax(0,1fr) minmax(0,1.35fr);gap:var(--space-4)} +.graph-explorer-list{max-height:var(--graph-list-max);overflow:auto;border-top:var(--rule) solid var(--color-border)} +.graph-explorer-item{display:flex;width:100%;align-items:center;gap:var(--space-2);padding:var(--space-2);border:0;border-bottom:var(--rule) solid var(--color-border);background:transparent;color:var(--color-text);font-size:var(--text-xs);text-align:left} +.graph-explorer-item:hover{background:var(--color-raised)} +.graph-explorer-item.active{background:var(--accent-bg);box-shadow:inset var(--rule-strong) 0 0 var(--color-accent)} +.graph-explorer-item>span:not(.gtype-dot){min-width:0;overflow-wrap:anywhere} +.graph-explorer-meta{margin-left:auto;color:var(--color-text-dim);white-space:normal;text-align:right} +.graph-tools,.editor-toolbar,.mm-actions,.search-row{flex-wrap:wrap} +.editor-toolbar{background:var(--color-panel)} +.editor-save-state{color:var(--color-text-muted);font-size:var(--text-xs);white-space:nowrap} +.editor-save-state.dirty{color:var(--color-warning);font-weight:600} +.gslider{display:flex;align-items:center;gap:var(--space-2);margin:var(--space-2) 0;color:var(--color-text-muted);font-size:var(--text-xs)} +.gslider label{width:var(--control-label-w);flex-shrink:0} +.gslider input[type=range]{height:var(--space-1);flex:1;accent-color:var(--color-accent);cursor:pointer} +.graph-preset{display:grid;gap:var(--space-1);margin-bottom:var(--space-3);padding-bottom:var(--space-3);border-bottom:var(--rule) solid var(--color-border)} +.graph-preset-row{display:flex;align-items:center;gap:var(--space-2)} +.graph-preset-row .field-lbl{width:var(--control-label-w);flex-shrink:0;margin:0} +.graph-preset-row .select{min-width:0;flex:1} +.graph-preset-help{min-height:2.8em;color:var(--color-text-dim);font-size:var(--text-xs);line-height:1.4} +.graph-layout{grid-template-columns:minmax(0,1fr) minmax(300px,340px);gap:var(--space-4);align-items:stretch} +.graph-main{min-width:0} +.graph-network{height:min(680px,calc(100dvh - 190px));min-height:540px;isolation:isolate;border-color:var(--color-border-strong);background-color:var(--color-canvas);background-image:radial-gradient(circle at 45% 42%,var(--color-accent-bg) 0,transparent 58%),linear-gradient(rgba(var(--graph-edge),.055) 1px,transparent 1px),linear-gradient(90deg,rgba(var(--graph-edge),.055) 1px,transparent 1px);background-size:auto,32px 32px,32px 32px;box-shadow:inset 0 0 0 var(--rule) rgba(var(--graph-edge),.05)} +#graph-net{z-index:1} +#graph-empty{z-index:3} +.graph-hud{position:absolute;top:var(--space-3);left:var(--space-3);z-index:2;display:flex;max-width:calc(100% - var(--space-6));align-items:center;gap:var(--space-3);padding:var(--space-1);border:0;border-radius:0;background:transparent;color:var(--color-text-muted);pointer-events:none;text-shadow:0 1px 2px var(--color-canvas),0 0 8px var(--color-canvas)} +.graph-hud-copy{display:grid;gap:1px} +.graph-hud-kicker{font-family:var(--mono);font-size:var(--text-xs);letter-spacing:.08em;text-transform:uppercase;color:var(--color-accent)} +.graph-hud-count{font-size:var(--text-xs);font-weight:600;color:var(--color-text)} +.graph-layout-status{display:inline-flex;align-items:center;gap:var(--space-1);font-family:var(--mono);font-size:var(--text-xs);white-space:nowrap} +.graph-layout-status::before{width:6px;height:6px;border-radius:50%;background:var(--color-success);content:""} +.graph-layout-status.busy::before{background:var(--color-warning);box-shadow:0 0 0 3px var(--color-warning-bg)} +.graph-controls-column{display:flex;height:min(680px,calc(100dvh - 190px));min-height:540px;flex-direction:column;gap:var(--space-3);overflow-y:auto;padding-right:2px} +.graph-controls-column>.card:first-child{display:block} +.graph-control-card{display:grid!important;gap:var(--space-3)} +.graph-card-title-row,.graph-section-head,.graph-palette-row{display:flex;align-items:center;justify-content:space-between;gap:var(--space-2)} +.graph-card-title-row .card-head,.graph-section-head .field-lbl{margin:0} +.graph-performance-badge{font-family:var(--mono);font-size:var(--text-xs);color:var(--color-success)} +.graph-action-row{display:grid;grid-template-columns:repeat(3,minmax(0,1fr));gap:var(--space-1)} +.graph-control-section{display:grid;gap:var(--space-2);padding-top:var(--space-3);border-top:var(--rule) solid var(--color-border)} +.graph-section-label{font-family:var(--font-display);font-size:var(--text-xs);font-weight:600;color:var(--color-text)} +.graph-layer-grid{display:grid;grid-template-columns:1fr 1fr;gap:var(--space-1) var(--space-2);font-size:var(--text-xs)} +.graph-check{display:flex;align-items:center;gap:var(--space-2);color:var(--color-text-muted);cursor:pointer} +.graph-check input{accent-color:var(--color-accent)} +.graph-palette-row .field-lbl{margin:0;white-space:nowrap} +.graph-palette-row .select{min-width:0;flex:1} +.graph-color-grid{display:grid;grid-template-columns:1fr 1fr;gap:var(--space-1)} +.graph-color-item{display:flex;min-width:0;align-items:center;gap:var(--space-2);padding:var(--space-1) var(--space-2);border:0;border-bottom:var(--rule) solid var(--color-border);border-radius:0;background:transparent;color:var(--color-text-muted);font-size:var(--text-xs);cursor:pointer} +.graph-color-item:hover{border-color:var(--color-border-strong);color:var(--color-text)} +.graph-color-item span{min-width:0;overflow:hidden;text-overflow:ellipsis;white-space:nowrap;text-transform:capitalize} +.graph-color-input{width:26px;height:26px;flex:0 0 26px;padding:2px;border:var(--rule) solid var(--color-border-strong);border-radius:50%;background:transparent;cursor:pointer} +.graph-color-input::-webkit-color-swatch-wrapper{padding:0} +.graph-color-input::-webkit-color-swatch{border:0;border-radius:50%} +.graph-palette-help{color:var(--color-text-dim);font-size:var(--text-xs);line-height:1.4} +.graph-advanced{padding-top:var(--space-3);border-top:var(--rule) solid var(--color-border)} +.graph-advanced>summary{display:flex;align-items:center;justify-content:space-between;color:var(--color-text-muted);font-size:var(--text-xs);font-weight:600;cursor:pointer;list-style:none} +.graph-advanced>summary::after{content:"+";font-family:var(--mono);color:var(--color-accent)} +.graph-advanced[open]>summary::after{content:"−"} +.graph-advanced-body{padding-top:var(--space-2)} +.graph-side-stack{display:grid;gap:var(--space-3)} +.graph-side-stack>.card:last-child{margin-top:0} +.graph-legend-card{padding:var(--space-3)} +.graph-legend-grid{display:grid;grid-template-columns:1fr;gap:var(--space-1);overflow:visible} +.graph-explorer-more{width:100%;padding:var(--space-2);border:0;border-bottom:var(--rule) solid var(--color-border);background:var(--color-panel);color:var(--color-accent);font-size:var(--text-xs);cursor:pointer} +.graph-explorer-more:hover{background:var(--color-raised)} +.graph-explorer-summary{color:var(--color-text-dim);font-size:var(--text-xs)} +@media(max-width:1100px){.graph-controls-column{height:auto;min-height:0;overflow:visible}.graph-network{height:min(var(--graph-canvas-max),62vh)}} +@media(max-width:768px){.graph-hud{right:var(--space-3);align-items:flex-start;justify-content:space-between}.graph-network{min-height:var(--graph-canvas-min)}.graph-color-grid{grid-template-columns:1fr}} + +.folder-access-grid{display:grid;grid-template-columns:repeat(auto-fit,minmax(var(--compact-column-min),1fr));gap:var(--space-2)} +.folder-access-option{display:flex;align-items:flex-start;gap:var(--space-2);padding:var(--space-3);border:var(--rule) solid var(--color-border);border-radius:var(--radius-sm);color:var(--color-text-muted);cursor:pointer} +.folder-access-option:hover{border-color:var(--color-border-strong);background:var(--color-raised)} +.folder-access-option:has(input:checked){border-color:var(--color-accent);background:var(--accent-bg);color:var(--color-text)} +.folder-access-option input{margin-top:3px;accent-color:var(--color-accent)} +.folder-access-option strong{display:block;margin-bottom:var(--space-1);color:var(--color-text);font-size:var(--text-sm)} +.folder-access-option span span{display:block;font-size:var(--text-xs);line-height:1.4} + +.theme-wrap{position:relative} +.theme-menu{display:none;position:absolute;top:var(--control-h-sm);right:0;z-index:120;min-width:calc(var(--space-9) * 4);padding:var(--space-1);border:var(--rule) solid var(--color-border-strong);border-radius:var(--radius-sm);background:var(--color-panel);box-shadow:var(--shadow-dialog)} +.theme-opt{display:flex;width:100%;align-items:center;gap:var(--space-2);padding:var(--space-2);border:0;border-radius:var(--radius-sm);background:transparent;color:var(--color-text-muted);cursor:pointer;font-size:var(--text-xs);text-align:left;white-space:nowrap} +.theme-opt:hover,.theme-opt.sel{background:var(--color-raised);color:var(--color-text)} +.theme-sw{display:flex;width:var(--space-4);height:var(--space-4);flex-shrink:0;align-items:center;justify-content:center;border:var(--rule) solid var(--color-border-strong);border-radius:var(--radius-sm)} +.theme-sw i{width:var(--space-2);height:var(--space-2);border-radius:50%} +.theme-opt-name{flex:1} +.theme-chk{color:var(--color-accent);font-size:var(--text-xs)} + +.mm-overlay{position:fixed;inset:0;z-index:300;display:none;align-items:center;justify-content:center;padding:var(--space-4);overflow-y:auto;background:var(--color-overlay);backdrop-filter:none} +.mm-overlay.show{display:flex;animation:none} +.mm-overlay[aria-hidden=true]{display:none} +.mm-box{display:flex;width:min(var(--dialog-max),calc(100vw - var(--space-8)));max-height:calc(100dvh - var(--space-8));flex-direction:column;overflow:hidden;border:var(--rule) solid var(--color-border-strong);border-radius:var(--radius-lg);background:var(--color-panel);box-shadow:var(--shadow-dialog)} +.mm-head{display:flex;align-items:flex-start;justify-content:space-between;gap:var(--space-3);padding:var(--space-4) var(--space-5) var(--space-2)} +.mm-title{font-family:var(--font-display);font-size:var(--text-xl);font-weight:600;word-break:break-word} +.mm-meta{display:flex;gap:var(--space-2);padding:0 var(--space-5) var(--space-3);border-bottom:var(--rule) solid var(--color-border);flex-wrap:wrap} +.mm-body{flex:1;padding:var(--space-4) var(--space-5);overflow-y:auto;line-height:1.7} +.mm-actions{display:flex;gap:var(--space-2);padding:var(--space-3) var(--space-5);border-top:var(--rule) solid var(--color-border);background:var(--color-raised);flex-wrap:wrap} +.cmd-overlay{padding:var(--space-6) var(--space-3);background:var(--color-overlay)} +.cmd-box{width:min(var(--command-max),calc(100vw - var(--space-6)));max-height:calc(100dvh - var(--space-10));box-shadow:var(--shadow-dialog)} +.ep-panel{position:fixed;top:0;right:calc(-1 * (var(--entity-panel-max) + var(--space-5)));z-index:250;display:flex;width:min(var(--entity-panel-max),92vw);height:100dvh;flex-direction:column;border-left:var(--rule) solid var(--color-border);background:var(--color-panel);box-shadow:var(--shadow-dialog);transition:right var(--transition)} +.ep-panel.show{right:0} +.ep-head{padding:var(--space-4) var(--space-5) var(--space-3);border-bottom:var(--rule) solid var(--color-border)} +.ep-name{display:flex;align-items:center;gap:var(--space-2);font-family:var(--font-display);font-size:var(--text-xl);font-weight:600;word-break:break-word} +.ep-sub{margin-top:var(--space-1);color:var(--color-text-dim);font-size:var(--text-xs);letter-spacing:.06em;text-transform:uppercase} +.ep-body{flex:1;padding:var(--space-3);overflow-y:auto} +.ep-section{padding:var(--space-3) var(--space-2) var(--space-1);color:var(--color-text-dim);font-family:var(--mono);font-size:var(--text-micro);font-weight:600;letter-spacing:.08em;text-transform:uppercase} +.ep-mem{margin-bottom:var(--space-2);padding:var(--space-2);border:var(--rule) solid var(--color-border);border-radius:var(--radius-sm);background:var(--color-raised);cursor:pointer;transition:border-color var(--transition),background-color var(--transition)} +.ep-mem:hover{border-color:var(--color-border-strong)} +.ep-mem-title{margin-bottom:var(--space-1);font-size:var(--text-xs);font-weight:500} +.ep-mem-preview{max-height:var(--space-10);overflow:hidden;color:var(--color-text-muted);font-size:var(--text-xs);line-height:1.5} +.ep-edge{display:flex;align-items:center;gap:var(--space-2);padding:var(--space-2);border-radius:var(--radius-sm);cursor:pointer;font-size:var(--text-xs)} +.ep-edge:hover{background:var(--color-raised)} +.ep-edge .rel{color:var(--color-text-dim);font-size:var(--text-micro);letter-spacing:.05em;text-transform:uppercase} + +.vault-card-name,.recall-title,.tl-body{min-width:0;overflow-wrap:anywhere} +.graph-network[aria-busy=true],[aria-busy=true]{cursor:progress} +.status-region:empty{display:none} +.toast{top:var(--space-3);right:var(--space-3);max-width:min(var(--command-max),calc(100vw - var(--space-6)));box-shadow:var(--shadow-dialog)} +.toast-ok{border:var(--rule) solid var(--color-success)} +.toast-err{border:var(--rule) solid var(--color-danger)} +.mobile-nav-toggle{display:none;width:var(--control-h);height:var(--control-h);align-items:center;justify-content:center;padding:var(--space-2);border:0;background:transparent;color:var(--color-text)} +.sidebar-backdrop{display:none} +.btn[aria-disabled=true],.btn:disabled{opacity:.55;cursor:not-allowed} +.graph-entity-link{border:0;background:transparent;color:var(--color-accent);font:inherit;text-decoration:underline;text-underline-offset:var(--space-1)} +.graph-relation-row{justify-content:flex-start;flex-wrap:wrap} +.graph-relation-label{color:var(--color-text-dim)} +:is(a,button,summary,.nav-item,.mem-card,.recall-card,.vault-card,.gtop-row,[role=button],input,select,textarea,[tabindex]):focus-visible{outline:var(--rule-strong) solid var(--color-accent);outline-offset:var(--rule-strong);border-radius:var(--radius-sm)} +.topbar-title:focus-visible{outline:none;border-radius:0} +.sr-only{position:absolute!important;width:var(--rule)!important;height:var(--rule)!important;padding:0!important;margin:calc(-1 * var(--rule))!important;overflow:hidden!important;clip:rect(0,0,0,0)!important;white-space:nowrap!important;border:0!important} + +@media(max-width:1100px){ + .graph-layout{grid-template-columns:1fr} + .graph-controls-column{display:grid;grid-template-columns:repeat(2,minmax(0,1fr))} + .graph-network{height:min(var(--graph-canvas-max),58vh)} +} +@media(max-width:768px){ + .app,.main{width:100%;min-width:0} + .content{width:100%;overflow-x:hidden} + .sidebar{position:fixed;inset:0 auto 0 0;width:min(84vw,var(--mobile-sidebar-max));transform:translateX(-100%);transition:transform var(--transition);box-shadow:none} + .app.mobile-nav-open .sidebar{transform:translateX(0);box-shadow:var(--shadow-dialog)} + .mobile-nav-toggle{display:inline-flex;flex:0 0 auto} + .sidebar-backdrop{position:fixed;inset:0;z-index:40;border:0;background:var(--color-overlay)} + .app.mobile-nav-open .sidebar-backdrop{display:block} + .topbar{height:var(--mobile-header-h);padding:0 var(--space-3);gap:var(--space-2)} + .topbar-route{flex:1;min-width:0} + .topbar-kicker,.topbar-sub{display:none} + .topbar-heading{min-width:0} + .topbar-title{min-width:0;overflow:hidden;font-size:var(--text-lg);text-overflow:ellipsis;white-space:nowrap} + .topbar-spacer{display:none} + .system-banner{padding:var(--space-2) var(--space-3)!important} + .view{width:100%;max-width:none;padding:calc(var(--space-5) / 2) var(--space-4) var(--space-5)} + .view-head{flex-wrap:wrap} + #view-overview .stat-grid{grid-template-columns:repeat(2,minmax(0,1fr));margin-bottom:var(--space-6)} + #view-overview .stat{min-height:calc(var(--space-10) + var(--space-8));padding:var(--space-3)} + #view-overview .stat:nth-child(odd){border-left:0} + #view-overview .stat:nth-child(n+3){border-top:var(--rule) solid var(--color-border)} + #view-overview .stat:first-child .stat-val{font-size:var(--text-xl)} + .overview-ledger-grid{grid-template-columns:1fr;gap:var(--space-6)} + .workbench-command{align-items:stretch;flex-wrap:wrap} + #view-recall .workbench-command .input,#view-memories .workbench-command .input,#view-why .workbench-command .input,#view-timeline .workbench-command .input{flex:1 1 100%!important} + .editor-wrap{height:calc(100dvh - var(--mobile-header-h) - var(--space-8))!important;min-height:var(--editor-shell-min)} + .editor-toolbar{align-content:flex-start} + .editor-toolbar #ed-title{flex:1 1 calc(100% - var(--space-10) - var(--space-10))} + .editor-body{flex-direction:column;overflow:auto} + .editor-edit{width:100%;min-height:var(--editor-pane-min);flex:0 0 42%;padding:var(--space-4)} + .editor-preview{width:100%;min-height:var(--editor-pane-min);flex:1 0 auto;padding:var(--space-4);border-top:var(--rule) solid var(--color-border);border-left:0} + .graph-controls-column{grid-template-columns:1fr} + .graph-network{height:min(var(--graph-canvas-compact),56vh);min-height:var(--graph-canvas-min)} + .graph-explorer-grid{grid-template-columns:1fr} + .code-import-grid{grid-template-columns:1fr} + .dropzone{padding:var(--space-5) var(--space-3)} + .dropzone-actions{flex-wrap:wrap} + .dup-pair,.recall-head{flex-direction:column} + .mem-card-meta,.thought-meta,.recall-scores{flex-wrap:wrap} + .recall-scores{gap:var(--space-2)} + .mm-overlay{padding:var(--space-2);align-items:flex-start} + .mm-box{width:100%;max-height:calc(100dvh - var(--space-4))} + .mm-head,.mm-meta,.mm-body,.mm-actions{padding-left:var(--space-4);padding-right:var(--space-4)} + .toast{left:var(--space-3);right:var(--space-3);max-width:none} + .audit-row{align-items:flex-start;flex-wrap:wrap} + .tbl{display:block;max-width:100%;overflow-x:auto} + .truncate{max-width:45vw} + .cfg-row{gap:var(--space-2);flex-wrap:wrap} + .chat-input-row{padding-inline:var(--space-3);flex-wrap:wrap} + .view :is(input,select,textarea,pre,code){max-width:100%} + .card,.stat,.cols-2>*,.cols-3>*,.graph-layout>*,.graph-controls-column>*,.audit-row>*{min-width:0} + .vault-card-actions .btn,.editor-toolbar .btn,.mm-actions .btn{max-width:100%} +} +@media(max-width:768px){ + #view-settings{padding:calc(var(--space-5) / 4) var(--space-2) calc(var(--space-5) / 2)} + .settings-grid{grid-template-columns:1fr} +} +#tokens-body>div[style*="display:flex"],#sync-body>div[style*="display:flex"],#automation-body div[style*="display:flex"],#team-body div[style*="display:flex"]{flex-wrap:wrap} +@media(max-width:420px){ + .view{padding:calc(var(--space-4) / 2) var(--space-3) var(--space-4)} + .system-notice{grid-template-columns:minmax(0,1fr) auto;align-items:center} + .system-notice-brief{display:none} + .stat-val{font-size:var(--text-lg)} + .btn{white-space:normal} + .graph-network{min-height:var(--graph-canvas-min)} + .mm-title{font-size:var(--text-lg)} +} +@media(prefers-reduced-motion:reduce){ + *,*::before,*::after{scroll-behavior:auto!important;animation-duration:.01ms!important;animation-iteration-count:1!important;transition-duration:.01ms!important} +} + +/* Generated from former static style attributes. */ +[data-csp-style="s1"]{flex:1} +[data-csp-style="s2"]{cursor:pointer;font-size:12px;border:0} +[data-csp-style="s3"]{position:relative;display:inline-flex} +[data-csp-style="s4"]{background:none;border:none;color:var(--text-dim);font-size:16px;cursor:pointer;padding:2px 5px} +[data-csp-style="s5"]{background:var(--accent)} +[data-csp-style="s6"]{display:flex;align-items:center;gap:6px;flex:1} +[data-csp-style="s7"]{font-family:var(--mono);font-size:12px} +[data-csp-style="s8"]{display:none} +[data-csp-style="s9"]{font-size:12px} +[data-csp-style="s10"]{padding:14px} +[data-csp-style="s11"]{margin-bottom:12px} +[data-csp-style="s12"]{padding:8px} +[data-csp-style="s13"]{height:60vh;border:1px solid var(--border);border-radius:var(--radius);overflow:hidden} +[data-csp-style="s14"]{flex:1;min-width:120px} +[data-csp-style="s15"]{width:auto} +[data-csp-style="s16"]{font-size:12px;color:var(--text-dim);margin:10px 2px} +[data-csp-style="s17"]{position:absolute;inset:0} +[data-csp-style="s18"]{position:absolute;inset:0;display:flex;align-items:center;justify-content:center;text-align:center;padding:20px} +[data-csp-style="s19"]{margin:0} +[data-csp-style="s20"]{display:contents} +[data-csp-style="s21"]{background:var(--color-info,#6f9fd8)} +[data-csp-style="s22"]{background:var(--color-teal,#5aafb3)} +[data-csp-style="s23"]{background:var(--color-warning,#d7a84b)} +[data-csp-style="s24"]{background:var(--color-accent,#8c83e8)} +[data-csp-style="s25"]{margin-left:auto;display:flex;align-items:center;gap:6px} +[data-csp-style="s26"]{height:30px;width:148px} +[data-csp-style="s27"]{display:none;margin-left:auto} +[data-csp-style="s28"]{font-family:ui-monospace,SFMono-Regular,monospace;font-size:11px;color:var(--color-text);background:rgba(255,255,255,.07);padding:1px 6px;border-radius:4px;min-width:34px;text-align:right;flex-shrink:0} +[data-csp-style="s29"]{grid-template-columns:1fr 1fr 1fr} +[data-csp-style="s30"]{grid-template-columns:1fr 1fr 1fr 1fr} +[data-csp-style="s31"]{display:grid;grid-template-columns:1fr 1fr;gap:5px} +[data-csp-style="s32"]{justify-content:center;border-radius:7px} +[data-csp-style="s33"]{height:30px;width:100%;margin-top:8px} +[data-csp-style="s34"]{margin-left:auto} +[data-csp-style="s35"]{font-size:16px} +[data-csp-style="s36"]{width:min(320px,100%)} +[data-csp-style="s37"]{justify-content:flex-end} +[data-csp-style="s38"]{display:none;flex-shrink:0} +[data-csp-style="s39"]{margin-bottom:14px;display:none} +[data-csp-style="s40"]{display:flex;gap:8px;margin-top:10px;align-items:center;flex-wrap:wrap} +[data-csp-style="s41"]{font-size:12px;color:var(--text-dim);white-space:nowrap} +[data-csp-style="s42"]{flex:1;min-width:180px} +[data-csp-style="s43"]{width:90px} +[data-csp-style="s44"]{display:flex;align-items:center;gap:6px;font-size:12px;margin-top:8px} +[data-csp-style="s45"]{font-size:12px;color:var(--text-dim);margin-top:8px} +[data-csp-style="s46"]{display:grid;grid-template-columns:1fr 1fr auto;gap:8px} +[data-csp-style="s47"]{display:grid;grid-template-columns:1fr 1fr auto;gap:8px;margin-top:8px} +[data-csp-style="s48"]{display:flex;justify-content:space-between;padding:6px 0} +[data-csp-style="s49"]{width:auto;padding:4px 8px} +[data-csp-style="s50"]{font-family:var(--mono);font-size:12px;color:var(--text-dim)} +[data-csp-style="s51"]{margin-top:0} +[data-csp-style="s52"]{margin-top:12px} +[data-csp-style="s53"]{z-index:200} +[data-csp-style="s54"]{max-width:420px} +[data-csp-style="s55"]{padding-top:4px} +[data-csp-style="s56"]{max-width:560px} +[data-csp-style="s57"]{font-weight:400} +[data-csp-style="s58"]{border:0} +[data-csp-style="s59"]{margin-top:var(--space-2)} +[data-csp-style="s62"]{display:flex;align-items:center;gap:8px;padding:4px 0} +[data-csp-style="s63"]{width:80px;font-size:12px;text-transform:capitalize} +[data-csp-style="s64"]{flex:1;height:6px;background:var(--surface2);border-radius:3px;overflow:hidden} +[data-csp-style="s66"]{width:36px;text-align:right;font-size:12px;color:var(--text-dim)} +[data-csp-style="s67"]{padding:12px} +[data-csp-style="s68"]{padding:4px} +[data-csp-style="s69"]{font-size:12px;color:var(--text-muted);margin-bottom:10px} +[data-csp-style="s70"]{font-size:12px;color:var(--text-dim);margin-bottom:6px} +[data-csp-style="s72"]{font-size:12px;color:var(--text-dim)} +[data-csp-style="s73"]{display:grid;grid-template-columns:1fr 1fr;gap:12px 8px} +[data-csp-style="s74"]{padding:7px 20px;text-align:center} +[data-csp-style="s75"]{font-size:32px;margin-bottom:8px} +[data-csp-style="s76"]{font-weight:600;font-size:16px;margin-bottom:5px} +[data-csp-style="s77"]{color:var(--text-muted);font-size:12.5px;margin:0 auto 16px;max-width:360px} +[data-csp-style="s78"]{display:flex;gap:8px;justify-content:center;flex-wrap:wrap} +[data-csp-style="s79"]{display:flex;align-items:center;gap:8px;padding:2px 0} +[data-csp-style="s80"]{width:104px;font-size:12px;color:var(--text-muted);overflow:hidden;text-overflow:ellipsis;white-space:nowrap} +[data-csp-style="s81"]{flex:1;height:8px;background:var(--surface2);border-radius:4px;overflow:hidden} +[data-csp-style="s83"]{width:42px;text-align:right;font-size:12px;color:var(--text-dim);font-variant-numeric:tabular-nums} +[data-csp-style="s85"]{padding:10px} +[data-csp-style="s86"]{margin:24px auto} +[data-csp-style="s87"]{padding:20px} +[data-csp-style="s88"]{display:flex;align-items:center;gap:8px;font-size:16px;margin-bottom:12px;cursor:pointer} +[data-csp-style="s89"]{display:flex;gap:8px;margin:12px 0} +[data-csp-style="s90"]{font-size:12px;color:var(--text-muted)} +[data-csp-style="s91"]{margin-top:14px} +[data-csp-style="s92"]{font-family:var(--mono);background:var(--surface2);padding:2px 6px;border-radius:4px;display:inline-block;margin-top:4px} +[data-csp-style="s93"]{margin:10px 0} +[data-csp-style="s94"]{font-family:var(--mono);font-size:12px;white-space:pre-wrap;color:var(--text-dim);margin-top:6px;max-height:220px;overflow:auto} +[data-csp-style="s95"]{margin-top:18px} +[data-csp-style="s96"]{color:var(--text-dim);font-weight:400;font-size:12px} +[data-csp-style="s97"]{margin-top:8px} +[data-csp-style="s98"]{font-size:12px;color:var(--text-dim);margin-bottom:3px} +[data-csp-style="s99"]{margin-top:6px} +[data-csp-style="s100"]{flex:1;color:var(--text-muted);overflow:hidden;text-overflow:ellipsis;white-space:nowrap} +[data-csp-style="s101"]{color:var(--text-dim)} +[data-csp-style="s102"]{text-align:center;padding:8px} +[data-csp-style="s103"]{margin:0 auto} +[data-csp-style="s104"]{font-size:12px;color:var(--text-dim);margin-bottom:10px} +[data-csp-style="s105"]{padding:6px} +[data-csp-style="s106"]{display:flex;align-items:flex-start;gap:10px} +[data-csp-style="s107"]{flex:1;min-width:0} +[data-csp-style="s108"]{margin:20px auto} +[data-csp-style="s109"]{font-size:16px;white-space:pre-wrap} +[data-csp-style="s110"]{padding:6px 0} +[data-csp-style="s111"]{font-weight:500} +[data-csp-style="s112"]{font-size:16px;color:var(--text-muted)} +[data-csp-style="s113"]{padding:0} +[data-csp-style="s114"]{flex:1;overflow:hidden;text-overflow:ellipsis;white-space:nowrap} +[data-csp-style="s115"]{margin-bottom:10px} +[data-csp-style="s116"]{font-family:var(--mono);font-size:12px;white-space:pre-wrap;color:var(--text-muted)} +[data-csp-style="s117"]{cursor:pointer} +[data-csp-style="s118"]{color:var(--accent)} +[data-csp-style="s119"]{padding:8px;text-align:center} +[data-csp-style="s120"]{border-color:var(--red)} +[data-csp-style="s121"]{margin:10px 0 4px} +[data-csp-style="s122"]{display:flex;gap:6px;flex-wrap:wrap;margin-top:6px} +[data-csp-style="s123"]{display:flex;gap:6px;flex-wrap:wrap;margin-bottom:10px} +[data-csp-style="s124"]{display:flex;gap:6px} +[data-csp-style="s125"]{margin-top:10px} +[data-csp-style="s126"]{width:100%;margin-bottom:8px} +[data-csp-style="s127"]{margin-bottom:8px} +[data-csp-style="s128"]{width:100%;margin-bottom:10px} +[data-csp-style="s129"]{margin:16px auto} +[data-csp-style="s130"]{display:flex;gap:6px;flex-wrap:wrap;margin-top:8px} +[data-csp-style="s131"]{margin-bottom:0} +[data-csp-style="s132"]{display:flex;flex-wrap:wrap;gap:10px;align-items:flex-end;margin-bottom:12px} +[data-csp-style="s133"]{flex:1;min-width:150px;margin:0} +[data-csp-style="s134"]{min-width:0} +[data-csp-style="s135"]{width:190px;margin:0} +[data-csp-style="s136"]{margin:-4px 0 12px} +[data-csp-style="s137"]{display:flex;flex-wrap:wrap;gap:10px;align-items:flex-end} +[data-csp-style="s138"]{flex:1;min-width:160px;margin:0} +[data-csp-style="s139"]{flex:1;min-width:120px;margin:0} +[data-csp-style="s140"]{width:110px;margin:0} +[data-csp-style="s141"]{margin-top:12px;padding:0} +[data-csp-style="s142"]{padding:12px 14px} +[data-csp-style="s143"]{width:90px;font-size:12px} +[data-csp-style="s144"]{font-size:12px;padding:2px 8px} +[data-csp-style="s145"]{color:var(--text-dim);font-size:12px} +[data-csp-style="s146"]{padding:0;margin-top:0} +[data-csp-style="s147"]{padding:16px} +[data-csp-style="s148"]{display:flex;gap:6px;margin-bottom:10px} +[data-csp-style="s149"]{font-size:16px;color:var(--text-muted);margin-bottom:10px} +[data-csp-style="s150"]{display:flex;gap:6px;flex-wrap:wrap} +[data-csp-style="s151"]{font-weight:400;color:var(--text-dim)} +[data-csp-style="s152"]{width:100%} +[data-csp-style="s153"]{text-align:center;margin-top:10px} +[data-csp-style="s154"]{color:var(--accent);font-size:12px} +[data-csp-style="s155"]{text-align:center;margin-top:6px} +[data-csp-style="s156"]{text-align:center;margin-top:14px} +[data-csp-style="s157"]{display:flex;align-items:center;gap:10px} +[data-csp-style="s158"]{border-color:var(--amber)} +[data-csp-style="s159"]{font-family:var(--mono)} +[data-csp-style="s160"]{color:var(--amber)} +[data-csp-style="s161"]{display:flex;gap:8px;flex-wrap:wrap;align-items:center;margin:8px 0} +[data-csp-style="s162"]{padding:4px 0;font-size:12px;color:var(--text-muted)} +[data-csp-style="s163"]{font-size:12px;color:var(--text-muted);margin:8px 0 4px} +[data-csp-style="s164"]{position:relative} +[data-csp-style="s165"]{font-family:var(--mono);font-size:12px;min-height:90px;white-space:pre;width:100%} +[data-csp-style="s166"]{position:absolute;top:4px;right:4px} +[data-csp-style="s167"]{display:flex;gap:8px;margin:10px 0} +[data-csp-style="s168"]{font-size:12px;align-self:center} +[data-csp-style="s169"]{padding:6px 0;font-size:12px;color:var(--text-muted)} +[data-csp-style="s170"]{margin:8px 0} +[data-csp-style="s171"]{font-size:12px;color:var(--text-muted);margin:6px 0 4px} +[data-csp-style="s172"]{background:var(--surface2);padding:10px;margin:8px 0} +[data-csp-style="s173"]{font-size:12px;color:var(--text-muted);margin-bottom:4px} +[data-csp-style="s174"]{font-family:var(--mono);font-size:12px;word-break:break-all} +[data-csp-style="s175"]{font-size:12px;color:var(--text-muted);margin-top:8px} +[data-csp-style="s176"]{display:flex;gap:6px;margin:4px 0 8px} +[data-csp-style="s177"]{display:flex;gap:10px;align-items:center;margin-top:6px} +[data-csp-style="s178"]{margin-top:12px;border-top:1px solid var(--border);padding-top:10px} +[data-csp-style="s179"]{font-size:12px;text-transform:uppercase;letter-spacing:.04em;color:var(--text-dim);margin-bottom:6px} +[data-csp-style="s180"]{display:flex;align-items:center;gap:8px;font-size:16px;flex-wrap:wrap} +[data-csp-style="s181"]{width:60px;padding:2px 6px} +[data-csp-style="s182"]{margin-top:4px} +[data-csp-style="s184"]{grid-column:1/-1} +[data-csp-style="s185"]{display:flex;height:12px;border-radius:4px;overflow:hidden;margin-bottom:5px} +[data-csp-style="s187"]{display:flex;justify-content:space-between;font-size:11px;color:var(--color-text-dim)} +[data-csp-style="s189"]{font-size:12px;color:var(--text-dim);margin-bottom:8px} +[data-csp-style="s190"]{padding:8px 0;border-bottom:1px solid var(--surface2);cursor:pointer} +[data-csp-style="s191"]{font-weight:500;font-size:16px} +[data-csp-style="s194"]{display:flex;justify-content:space-between;padding:3px 0} +[data-csp-style="s196"]{font-weight:600;color:var(--text);margin-bottom:5px} +.is-hidden{display:none!important}.is-flex{display:flex!important}.is-block{display:block!important}.is-inline-flex{display:inline-flex!important}.theme-menu.is-open{display:block}.health-ok{background:var(--green)!important}.health-error{background:var(--red)!important}.cursor-pointer{cursor:pointer!important}.cursor-grab{cursor:grab!important}.tone-red{color:var(--red)!important}.tone-green{color:var(--green)!important}.tone-muted{color:var(--text-muted)!important}.is-disabled-visual{opacity:.7} +.graph-network.graph-style-galaxy{background:radial-gradient(58% 50% at 24% 22%,rgba(126,64,208,.30),transparent 66%),radial-gradient(52% 58% at 82% 78%,rgba(220,72,164,.20),transparent 68%),radial-gradient(46% 52% at 62% 42%,rgba(58,120,224,.16),transparent 70%),#06040f}.graph-network.graph-style-solar{background:radial-gradient(40% 48% at 50% 50%,rgba(255,184,92,.16),transparent 60%),radial-gradient(90% 90% at 50% 50%,rgba(18,32,64,.55),transparent 82%),#05070d}.graph-network.graph-style-cyber{background:linear-gradient(rgba(34,224,255,.055) 1px,transparent 1px) 0 0/30px 30px,linear-gradient(90deg,rgba(34,224,255,.055) 1px,transparent 1px) 0 0/30px 30px,radial-gradient(72% 60% at 50% 0%,rgba(255,62,165,.12),transparent 72%),#050810} +[data-graph-node-type="person_or_concept"]{background:var(--entity-concept);color:var(--entity-concept)}[data-graph-node-type="mention"]{background:var(--entity-mention);color:var(--entity-mention)}[data-graph-node-type="hashtag"]{background:var(--entity-hashtag);color:var(--entity-hashtag)}[data-graph-node-type="email"]{background:var(--entity-email);color:var(--entity-email)}[data-graph-node-type="organization"]{background:var(--entity-organization);color:var(--entity-organization)}[data-graph-node-type="location"]{background:var(--entity-location);color:var(--entity-location)} +.graph-tone-blue{color:var(--blue)}.graph-tone-cyan{color:var(--cyan)}.graph-tone-green{color:var(--green)}.graph-tone-dim{color:var(--text-dim)} +.tone-amber{color:var(--amber)!important}.overview-stat{font-size:24px;font-weight:600}.graph-stat-row{display:flex;justify-content:space-between;padding:3px 0} +.theme-sw-dark{background:#15181e}.theme-sw-dark i{background:#8c83e8}.theme-sw-light{background:#fbfbfc}.theme-sw-light i{background:#5547b8}.theme-sw-midnight{background:#111b2d}.theme-sw-midnight i{background:#79a6ef}.theme-sw-solarized{background:#073642}.theme-sw-solarized i{background:#58a7d8}.theme-sw-sepia{background:#f8f2e4}.theme-sw-sepia i{background:#925420} +.overview-type-bar,.analytics-bar,.graph-degree{appearance:none;border:0;background:var(--surface2);overflow:hidden}.overview-type-bar{flex:1;height:6px;border-radius:3px}.analytics-bar{flex:1;height:8px;border-radius:4px}.graph-degree{flex:1;height:5px;border-radius:3px}.overview-type-bar::-webkit-progress-bar,.analytics-bar::-webkit-progress-bar,.graph-degree::-webkit-progress-bar{background:var(--surface2)}.overview-type-bar::-webkit-progress-value,.analytics-bar::-webkit-progress-value,.graph-degree::-webkit-progress-value{background:var(--accent)}.overview-type-bar::-moz-progress-bar,.analytics-bar::-moz-progress-bar,.graph-degree::-moz-progress-bar{background:var(--accent)} +.analytics-bar.analytics-bar-green::-webkit-progress-value{background:var(--green)}.analytics-bar.analytics-bar-green::-moz-progress-bar{background:var(--green)}.analytics-bar.analytics-bar-blue::-webkit-progress-value{background:var(--blue)}.analytics-bar.analytics-bar-blue::-moz-progress-bar{background:var(--blue)}.analytics-bar.analytics-bar-cyan::-webkit-progress-value{background:var(--cyan)}.analytics-bar.analytics-bar-cyan::-moz-progress-bar{background:var(--cyan)}.analytics-bar.analytics-bar-dim::-webkit-progress-value{background:var(--accent-dim)}.analytics-bar.analytics-bar-dim::-moz-progress-bar{background:var(--accent-dim)} +progress.graph-degree[data-graph-node-type="person_or_concept"]::-webkit-progress-value{background:var(--entity-concept)}progress.graph-degree[data-graph-node-type="person_or_concept"]::-moz-progress-bar{background:var(--entity-concept)}progress.graph-degree[data-graph-node-type="mention"]::-webkit-progress-value{background:var(--entity-mention)}progress.graph-degree[data-graph-node-type="mention"]::-moz-progress-bar{background:var(--entity-mention)}progress.graph-degree[data-graph-node-type="hashtag"]::-webkit-progress-value{background:var(--entity-hashtag)}progress.graph-degree[data-graph-node-type="hashtag"]::-moz-progress-bar{background:var(--entity-hashtag)}progress.graph-degree[data-graph-node-type="email"]::-webkit-progress-value{background:var(--entity-email)}progress.graph-degree[data-graph-node-type="email"]::-moz-progress-bar{background:var(--entity-email)}progress.graph-degree[data-graph-node-type="organization"]::-webkit-progress-value{background:var(--entity-organization)}progress.graph-degree[data-graph-node-type="organization"]::-moz-progress-bar{background:var(--entity-organization)}progress.graph-degree[data-graph-node-type="location"]::-webkit-progress-value{background:var(--entity-location)}progress.graph-degree[data-graph-node-type="location"]::-moz-progress-bar{background:var(--entity-location)} +.graph-cluster-0{background:#22e0ff}.graph-cluster-1{background:#ff3ea5}.graph-cluster-2{background:#b6ff3c}.graph-cluster-3{background:#ffe14d}.graph-cluster-4{background:#8b7bff}.graph-cluster-5{background:#ff5c7a}.graph-cluster-6{background:#3affd0}.graph-cluster-7{background:#ff7be0}.graph-cluster-8{background:#7affea}.graph-cluster-9{background:#c0ff4a}.graph-heat-0{background:#3f7bff}.graph-heat-1{background:#6a5cff}.graph-heat-2{background:#a24bff}.graph-heat-3{background:#e0479f}.graph-heat-4{background:#ff6b6b}.graph-heat-5{background:#ffc23d}[class^="graph-heat-"]{flex:1} +/* Merged from the dashboard's second +
@@ -495,248 +14,283 @@ - +
- +
Memory operationsOperate…

Overview

- +
- - +
+
Memory types
-
Analytics
Loading…
+
Analytics
Loading…
-
-

Score ranks relevance to this query. Retention estimates how strongly the memory persists over time.

-
Enter a query to recall memories.
+
+

Score ranks relevance to this query. Retention estimates how strongly the memory persists over time.

+
Enter a query to recall memories.
-
+

Open a focused memory with Enter. Reorder it with Alt + Up or Alt + Down.

-
Loading memories…
+
Loading memories…
-
+
- - - - - + + + + + Saved - +
- +
-
+
-
Loading relevant memories…
+
Loading relevant memories…
-
-
Ask a question to see the current answer and its history.
+
+
Ask a question to see the current answer and its history.
-
-
Enter a topic to trace its history.
+
+
Enter a topic to trace its history.
-
-
Loading governance history…
+
+
Loading governance history…
-
+
-
Loading graph…
+
Loading graph…
-
-
Explore
Adaptive rendering
-
-
-
-
-
-
Tighter packing for disconnected components, with quieter relationships and clearer nodes.
+
+
+Show +
+ + + + +
+ +
+ + + + +
+
+
+ +
Layout
+
+ + + + +
+
Forces
+
42
+
20
+
26
+
Appearance
+
3
+
12
+
0.7
+
30
+
Color by
+
+ + + +
+
Style
+
+ + + + +
+
Display
+
+ + +
+ +
Legend
+
+
Stats
+
+
Top connected
+
+
+
+ -
-
Data and focus
-
- - - - -
- - - - - -
-
-
Node colors
-
-
Colors appear after graph data loads.
-
Choose a palette or tune each entity type. Preferences stay in this browser.
-
-
-Appearance and layout physics -
-
-
-
-
-
-
-
-
-
-
-
-
Top connected
-
Stats
-
-
Legend
-

Semantic graph explorer

Keyboard-accessible entities and relationships synchronized with the visual graph.

+

Semantic graph explorer

Keyboard-accessible entities and relationships synchronized with the visual graph.

Entities

Relations

-
-
-
Start with a dry run to preview changes. A dry run never modifies memories.
+
+
Start with a dry run to preview changes. A dry run never modifies memories.
-
Loading maintenance policy…
+
Loading maintenance policy…
-
-
")] + assert settings.count('
') == 2 + for heading in ( + "Cloud sync", + "Connect an LLM", + "License & Plan", + "Appearance & Engine", + "Connect an agent", + ): + assert heading in settings + assert settings.count('class="settings-account-panel ') == 2 + account = settings[settings.index('class="card settings-account-card"'): + settings.index('
Cloud sync')] + assert account.index("License & Plan") < account.index("Appearance & Engine") + + css = css_response.text + assert ".settings-grid{display:grid;grid-template-columns:repeat(2,minmax(0,1fr))" in css + assert "#view-settings .card{margin:0;padding:var(--space-2) 0" in css + assert ".settings-account-grid{display:grid;grid-template-columns:minmax(0,1.15fr)" in css + assert ".settings-account-panel .cfg-row{display:grid;grid-template-columns:max-content" in css + assert "@container (max-width:460px)" in css + assert ".settings-grid{grid-template-columns:1fr}" in css + + +def test_dashboard_csp_listener_registry_covers_all_declarative_handlers(): + root = Path(__file__).resolve().parents[1] + html = (root / "engraphis" / "static" / "index.html").read_text(encoding="utf-8") + script = (root / "engraphis" / "static" / "dashboard.js").read_text(encoding="utf-8") + refs = set(re.findall(r'data-on[a-z]+=["\'](h\d+)["\']', html + "\n" + script)) + definitions = set(re.findall(r'^\s*(h\d+):function\(event\)\{', script, re.MULTILINE)) + assert refs + assert refs <= definitions + assert not re.search(r"\.(?:style|cssText)\b", script) + assert "cssText" not in script + assert "[onclick" not in script + def test_unconfigured_remote_api_is_refused_but_loopback_is_allowed(monkeypatch, tmp_path): remote = ("203.0.113.8", 50000) with _client(monkeypatch, tmp_path, client=remote) as c: @@ -277,6 +372,31 @@ def test_remote_api_token_opens_the_single_user_api(monkeypatch, tmp_path): ).status_code == 200 +def test_deployment_token_cannot_bypass_the_api_after_bootstrap(monkeypatch, tmp_path): + """The hosted ownership proof is not an unrestricted service-account credential.""" + deployment = "d" * 32 + with _client( + monkeypatch, tmp_path, team=True, key=_team_key(), + client=("203.0.113.8", 50000), api_token="service-api-secret", + ) as c: + setup = c.post("/api/auth/setup", json={ + "email": "w@x.co", "name": "W", "password": "supersecret1", + }, headers={"Authorization": "Bearer " + deployment}) + assert setup.status_code == 200, setup.text + c.cookies.clear() + + denied = c.get( + "/api/workspaces", headers={"Authorization": "Bearer " + deployment}) + assert denied.status_code == 401 + assert c.get( + "/api/memories?workspace=demo", + headers={"Authorization": "Bearer " + deployment}, + ).status_code == 401 + allowed = c.get( + "/api/workspaces", headers={"Authorization": "Bearer service-api-secret"}) + assert allowed.status_code == 200 + + def test_forwarding_header_never_turns_a_proxied_request_into_loopback(monkeypatch, tmp_path): with _client(monkeypatch, tmp_path) as c: response = c.get("/api/bootstrap", headers={"X-Forwarded-For": "127.0.0.1"}) @@ -379,6 +499,60 @@ def test_automation_policy_round_trips_dream_knobs(monkeypatch, tmp_path): assert p["infer"] is True # the inference pass is Pro-gated via automation +def test_live_llm_extractor_swap_defers_close_and_reports_persistence_failure( + monkeypatch, tmp_path): + """An ingest holding the old extractor must not lose its provider mid-request.""" + import gc + import os + + from engraphis import config + from engraphis.routes import v2_api + + class FakeLLM: + def __init__(self): + self.closed = False + + def close(self): + self.closed = True + + class FakeExtractor: + def __init__(self, llm): + self.llm = llm + + monkeypatch.setattr( + config, "persist_project_env", + lambda values: (_ for _ in ()).throw(OSError("read-only application directory")), + ) + with _client(monkeypatch, tmp_path) as _: + engine = v2_api.service().engine + previous = engine.extractor + previous_settings = (settings.extractor, settings.llm_auto_extract) + previous_env = { + name: os.environ.get(name) if name in os.environ else None + for name in ("ENGRAPHIS_EXTRACTOR", "ENGRAPHIS_LLM_AUTO_EXTRACT") + } + fake_llm = FakeLLM() + in_flight = FakeExtractor(fake_llm) + engine.extractor = in_flight + try: + result = v2_api._set_llm_extractor(False) + assert result["extractor_enabled"] is False + assert result["persisted"] is False + assert engine.extractor is not None # no check-then-None ingest race + assert fake_llm.closed is False + del in_flight + gc.collect() + assert fake_llm.closed is True + finally: + engine.extractor = previous + settings.extractor, settings.llm_auto_extract = previous_settings + for name, value in previous_env.items(): + if value is None: + os.environ.pop(name, None) + else: + os.environ[name] = value + + def test_consolidate_inference_pass_is_pro_gated(monkeypatch, tmp_path): # The inference pass (infer=True) is a paid `automation` capability — the dream # pass 4 — so it must 402 on the free tier; the base sweep (infer=False) stays free. @@ -535,6 +709,7 @@ def test_viewer_role_denied_on_governance_and_admin_routes(monkeypatch, tmp_path assert viewer.get("/api/export?workspace=demo").status_code == 403 assert viewer.post("/api/license/activate", json={"key": _team_key()}).status_code == 403 + assert viewer.get("/api/license/trials/claim-id").status_code == 403 # a viewer can't create a folder either — creating a workspace is a member+ action assert viewer.post("/api/workspaces/create", json={"workspace": "viewer-folder"}).status_code == 403 @@ -566,6 +741,14 @@ def test_viewer_role_denied_on_governance_and_admin_routes(monkeypatch, tmp_path assert member.post("/api/resources/postgres", json={ "workspace": "demo", "dsn": "postgresql://example.invalid/db", }).status_code == 403 + assert member.post("/api/llm/test").status_code == 403 + assert member.post("/api/llm/extractor", json={"enabled": False}).status_code == 403 + assert member.post("/api/license/trials", json={ + "email": "member@x.co", "plan": "team", "deployment_token": "x" * 32, + }).status_code == 403 + assert member.post("/api/sync/token", json={ + "token": "engr_ut_" + "x" * 32, "read_only": True, + }).status_code == 403 assert admin.post("/api/workspaces/create", json={"workspace": "admin-folder"}).status_code == 200 # Default folders are personal, so neither is exposed to a viewer. @@ -806,34 +989,25 @@ def test_advertised_api_root_is_real(monkeypatch, tmp_path): def test_trial_start_and_rejection(monkeypatch, tmp_path): - """Trial starts. Re-calling during active trial is a no-op (returns current status). - - Since 0.8.4 the Pro trial is a REAL server-issued key (``licensing.start_trial`` -> - ``cloud_license.request_trial_key``), not a local grant — mock the relay client call - so this stays on the offline gate, same as the Team-trial test below. The re-call - must NOT hit the relay a second time (there is only one ``request_trial_key`` stub - below, good for exactly one call) — ``start_trial`` recognizes the already-active - trial key locally and short-circuits before ever reaching the relay client. Since - 2026-07-14 an email is required in the request body too (the mock below still - returns a key synchronously — ``pending=False`` — simulating a relay that short- - circuits, so the "activates immediately" shape of this test stays valid).""" + """The deprecated route delegates to an idempotent deployment-bound claim.""" from engraphis import cloud_license - monkeypatch.setenv("ENGRAPHIS_LICENSE_PUBKEY", ed25519_public_key(_SECRET).hex()) - trial_key = compose_key( - {"v": 1, "plan": "pro", "email": "trial@engraphis.local", "seats": 1, - "issued": int(time.time()), "expires": int(time.time() + 3 * 86400), - "trial": 1}, _SECRET) - monkeypatch.setattr( - cloud_license, "request_trial_key", - lambda base, mid, plan="pro", email="": (trial_key, "", False)) + calls = [] + + def create(*args, **kwargs): + calls.append((args, kwargs)) + return {"claim_id": "claim-1", "status": "pending", "pending": True} + + monkeypatch.setattr(cloud_license, "create_trial_claim", create) with _client(monkeypatch, tmp_path) as c: r = c.post("/api/license/trial", json={"email": "trial@engraphis.local"}) assert r.status_code == 200 - lic = c.get("/api/license").json() - assert lic["is_trial"] is True + assert r.json()["claim_id"] == "claim-1" + assert r.json()["deprecated"] is True + assert "key" not in r.json() r2 = c.post("/api/license/trial", json={"email": "trial@engraphis.local"}) - assert r2.status_code == 200 # no-op: already on trial - assert r2.json()["is_trial"] is True + assert r2.status_code == 200 and r2.json()["claim_id"] == "claim-1" + assert c.get("/api/license").json()["plan"] == "free" + assert len(calls) == 2 def test_trial_start_requires_email(monkeypatch, tmp_path): @@ -842,8 +1016,8 @@ def test_trial_start_requires_email(monkeypatch, tmp_path): from engraphis import cloud_license called = [] monkeypatch.setattr( - cloud_license, "request_trial_key", - lambda *a, **k: called.append(1) or (None, "should not be called", False)) + cloud_license, "create_trial_claim", + lambda *a, **k: called.append(1) or {"claim_id": "bad"}) with _client(monkeypatch, tmp_path) as c: r = c.post("/api/license/trial", json={}) assert r.status_code == 400 @@ -856,9 +1030,8 @@ def test_trial_start_route_surfaces_pending_status(monkeypatch, tmp_path): key is minted only once the emailed magic link is opened, not from this call.""" from engraphis import cloud_license monkeypatch.setattr( - cloud_license, "request_trial_key", - lambda base, mid, plan="pro", email="": - (None, "check your email to confirm and activate the trial", True)) + cloud_license, "create_trial_claim", + lambda *a, **k: {"claim_id": "claim-2", "status": "pending", "pending": True}) with _client(monkeypatch, tmp_path) as c: r = c.post("/api/license/trial", json={"email": "w@example.com"}) assert r.status_code == 200 @@ -867,25 +1040,102 @@ def test_trial_start_route_surfaces_pending_status(monkeypatch, tmp_path): assert c.get("/api/license").json()["plan"] == "free" -def test_team_trial_route_activates_relay_issued_key(monkeypatch, tmp_path): - """POST /api/license/team-trial delegates to licensing.start_team_trial(), which - needs the vendor relay (unlike the local-only Pro trial) — mock the relay client - call so this stays on the offline gate.""" +def test_team_trial_route_starts_deployment_bound_claim(monkeypatch, tmp_path): from engraphis import cloud_license monkeypatch.setenv("ENGRAPHIS_STATE_DIR", str(tmp_path / "state")) # isolate machine_id - # the relay-minted key is signed with the test keypair, so verification against - # the real key needs the same pubkey override _client(key=...) would normally set - monkeypatch.setenv("ENGRAPHIS_LICENSE_PUBKEY", ed25519_public_key(_SECRET).hex()) - trial_key = compose_key( - {"v": 1, "plan": "team", "email": "trial@engraphis.local", "seats": 1, - "issued": int(time.time()), "expires": int(time.time() + 3 * 86400)}, _SECRET) monkeypatch.setattr( - cloud_license, "request_team_trial_key", - lambda base, mid, email="": (trial_key, "", False)) + cloud_license, "create_trial_claim", + lambda *a, **k: {"claim_id": "team-claim", "status": "pending", "pending": True}) with _client(monkeypatch, tmp_path) as c: r = c.post("/api/license/team-trial", json={"email": "trial@engraphis.local"}) - assert r.status_code == 200 and r.json()["plan"] == "team" - assert c.get("/api/license").json()["plan"] == "team" + assert r.status_code == 200 and r.json()["claim_id"] == "team-claim" + assert r.json()["replacement"] == "/api/license/trials" + assert "key" not in r.json() + + +def test_local_trial_needs_no_deployment_token(monkeypatch, tmp_path): + """A loopback caller is already trusted by the auth gate, so a self-hosted local + instance can start a trial with neither ENGRAPHIS_DEPLOYMENT_TOKEN nor + ENGRAPHIS_DASHBOARD_URL configured: the token is derived from the machine id and the + dashboard URL defaults to the request origin. This is the local-PC trial fix.""" + from engraphis import cloud_license + monkeypatch.setenv("ENGRAPHIS_STATE_DIR", str(tmp_path / "state")) # isolate machine_id + seen = {} + + def create(base, deployment_token, mid, email, plan, *, dashboard_url="", **kw): + seen.update(token=deployment_token, dashboard_url=dashboard_url, plan=plan) + return {"claim_id": "local-claim", "status": "pending", "pending": True} + + monkeypatch.setattr(cloud_license, "create_trial_claim", create) + with _client(monkeypatch, tmp_path) as c: + monkeypatch.delenv("ENGRAPHIS_DEPLOYMENT_TOKEN", raising=False) + monkeypatch.delenv("ENGRAPHIS_DASHBOARD_URL", raising=False) + r = c.post("/api/license/trials", json={"email": "w@example.com", "plan": "pro"}) + assert r.status_code == 200, r.text + assert r.json()["claim_id"] == "local-claim" + assert "key" not in r.json() + assert seen["token"].startswith("local-") and len(seen["token"]) > 24 + assert seen["dashboard_url"].startswith("http://") + assert seen["plan"] == "pro" + + +def test_local_trial_poll_needs_no_deployment_token(monkeypatch, tmp_path): + """Polling a claim from loopback derives the same machine-bound token, so the + create -> confirm -> poll round-trip needs nothing configured locally.""" + from engraphis import cloud_license + monkeypatch.setenv("ENGRAPHIS_STATE_DIR", str(tmp_path / "state")) + seen = {} + + def claim(base, claim_id, token, mid, **kw): + seen.update(claim_id=claim_id, token=token) + return {"claim_id": claim_id, "confirmed": False, "status": "pending"} + + monkeypatch.setattr(cloud_license, "claim_trial", claim) + with _client(monkeypatch, tmp_path) as c: + monkeypatch.delenv("ENGRAPHIS_DEPLOYMENT_TOKEN", raising=False) + r = c.get("/api/license/trials/local-claim") + assert r.status_code == 200, r.text + assert seen["claim_id"] == "local-claim" + assert seen["token"].startswith("local-") + + +def test_remote_trial_still_requires_deployment_token(monkeypatch, tmp_path): + """The loopback exemption must not leak to the network. In the zero-user bootstrap + window /api/license/trials is reachable remotely, so a proxied/internet caller must + still present the configured ownership token — a missing or wrong token is refused + before any control-plane call.""" + from engraphis import cloud_license + called = [] + monkeypatch.setattr( + cloud_license, "create_trial_claim", + lambda *a, **k: called.append(1) or {"claim_id": "should-not-happen"}) + with _client(monkeypatch, tmp_path, team=True, client=("203.0.113.7", 40000)) as c: + missing = c.post("/api/license/trials", json={"email": "a@x.co", "plan": "pro"}) + assert missing.status_code == 401 + wrong = c.post( + "/api/license/trials", + json={"email": "a@x.co", "plan": "pro", "deployment_token": "z" * 32}) + assert wrong.status_code == 401 + ok = c.post( + "/api/license/trials", + json={"email": "a@x.co", "plan": "pro", "deployment_token": "d" * 32}) + assert ok.status_code == 200, ok.text + assert called == [1] + + +def test_forwarded_trial_is_not_treated_as_local(monkeypatch, tmp_path): + """The crux of the loopback exemption's safety: a proxied request (loopback socket peer + but an X-Forwarded-* header present) must NOT be treated as local, or a same-host proxy + could smuggle an internet caller past the ownership check. It still needs the token.""" + from engraphis import cloud_license + called = [] + monkeypatch.setattr(cloud_license, "create_trial_claim", + lambda *a, **k: called.append(1) or {"claim_id": "x"}) + with _client(monkeypatch, tmp_path, team=True) as c: # default client peer is loopback + r = c.post("/api/license/trials", json={"email": "a@x.co", "plan": "pro"}, + headers={"X-Forwarded-For": "203.0.113.9"}) + assert r.status_code == 401 + assert called == [] def test_team_license_routes_are_public_only_during_zero_user_bootstrap(monkeypatch, tmp_path): @@ -893,12 +1143,9 @@ def test_team_license_routes_are_public_only_during_zero_user_bootstrap(monkeypa from engraphis import cloud_license monkeypatch.setenv("ENGRAPHIS_STATE_DIR", str(tmp_path / "state")) monkeypatch.setenv("ENGRAPHIS_LICENSE_PUBKEY", ed25519_public_key(_SECRET).hex()) - trial_key = compose_key( - {"v": 1, "plan": "team", "email": "trial@engraphis.local", "seats": 5, - "issued": int(time.time()), "expires": int(time.time() + 3 * 86400)}, _SECRET) monkeypatch.setattr( - cloud_license, "request_team_trial_key", - lambda base, mid, email="": (trial_key, "", False)) + cloud_license, "create_trial_claim", + lambda *a, **k: {"claim_id": "team-public", "status": "pending", "pending": True}) with _client(monkeypatch, tmp_path, team=True) as admin: # The zero-user bootstrap can inspect status and reach both trial routes logged out. @@ -907,7 +1154,9 @@ def test_team_license_routes_are_public_only_during_zero_user_bootstrap(monkeypa pro_trial = admin.post("/api/license/trial", json={}) assert pro_trial.status_code == 400 and "email" in pro_trial.text.lower() team_trial = admin.post("/api/license/team-trial", json={"email": "w@x.co"}) - assert team_trial.status_code == 200 and team_trial.json()["plan"] == "team" + assert team_trial.status_code == 200 and team_trial.json()["pending"] is True + activated = admin.post("/api/license/activate", json={"key": _team_key()}) + assert activated.status_code == 200 and activated.json()["plan"] == "team" assert admin.post("/api/auth/setup", json={ "email": "w@x.co", "name": "W", "password": "supersecret1", @@ -961,9 +1210,10 @@ def test_license_activate_still_requires_admin_session(monkeypatch, tmp_path): def test_team_trial_route_surfaces_relay_denial_as_400(monkeypatch, tmp_path): from engraphis import cloud_license monkeypatch.setenv("ENGRAPHIS_STATE_DIR", str(tmp_path / "state")) - monkeypatch.setattr( - cloud_license, "request_team_trial_key", - lambda base, mid, email="": (None, "the free Team trial has already been used", False)) + def denied(*args, **kwargs): + raise RuntimeError("the free Team trial has already been used") + + monkeypatch.setattr(cloud_license, "create_trial_claim", denied) with _client(monkeypatch, tmp_path) as c: r = c.post("/api/license/team-trial", json={"email": "w@x.co"}) assert r.status_code == 400 diff --git a/tests/test_extractor.py b/tests/test_extractor.py index 9d8aada..a249d3f 100644 --- a/tests/test_extractor.py +++ b/tests/test_extractor.py @@ -48,6 +48,7 @@ def test_llm_extractor_parses_facts_with_hints(): assert len(facts) == 2 assert facts[0].mtype == MemoryType.SEMANTIC and facts[0].importance == 0.8 assert facts[0].keywords == ["paseto", "auth"] + assert facts[0].metadata["llm_extraction"]["mode"] == "llm" assert facts[1].mtype == MemoryType.EPISODIC @@ -132,5 +133,8 @@ def test_engine_ingest_preserves_structured_extractor_metadata(): metadata={"source": "test"}) rec = eng.store.get_memory(out["facts"][0]["id"]) assert rec.metadata["source"] == "test" + assert rec.metadata["llm_extraction"]["mode"] == "llm_structured" + assert rec.metadata["llm_extraction"]["fact_count"] == 1 + assert len(rec.metadata["llm_extraction"]["source_sha256"]) == 64 assert rec.metadata["entities"] == ["Engraphis", "SQLite"] assert rec.metadata["relations"][0]["target"] == "SQLite" diff --git a/tests/test_graph_explorer_v2.py b/tests/test_graph_explorer_v2.py new file mode 100644 index 0000000..bf5c7f9 --- /dev/null +++ b/tests/test_graph_explorer_v2.py @@ -0,0 +1,1671 @@ +"""Focused schema/API coverage for the analytical Galaxy Graph vertical slice.""" +# ruff: noqa: E402 -- optional-stack guard must run before importing FastAPI routes +import json +import sqlite3 +import threading +import time +import types + +import pytest + +pytest.importorskip("fastapi", reason="graph HTTP coverage requires the optional server stack") + +from fastapi import FastAPI +from fastapi.testclient import TestClient + +from engraphis.backends import graph_extractor as graph_extractor_module +from engraphis.backends.graph_extractor import GraphExtraction, get_graph_extractor +from engraphis.core import graph_scene as graph_scene_module +from engraphis.core.graph_scene import build_graph_scene +from engraphis.core.interfaces import Edge, MemoryRecord, MemoryType, Node, Scope +from engraphis.core.store import Store +from engraphis.routes import v2_api +from engraphis import service as service_module +from engraphis.service import GraphIndexRebuilding, MemoryService, ValidationError + + +def test_v4_migration_backfills_canonical_entities_and_edge_supports(tmp_path): + db = tmp_path / "v3.db" + conn = sqlite3.connect(db) + conn.executescript(""" + CREATE TABLE schema_migrations(version INTEGER PRIMARY KEY, applied_at REAL); + INSERT INTO schema_migrations VALUES (3, 0); + CREATE TABLE entities ( + id TEXT PRIMARY KEY, workspace_id TEXT, repo_id TEXT, name TEXT, etype TEXT, + canonical_id TEXT, created_at REAL, + UNIQUE(workspace_id, repo_id, name, etype) + ); + CREATE TABLE edges ( + id TEXT PRIMARY KEY, workspace_id TEXT, repo_id TEXT, src TEXT NOT NULL, + dst TEXT NOT NULL, relation TEXT NOT NULL, layer TEXT DEFAULT 'semantic', + weight REAL DEFAULT 1.0, valid_from REAL, valid_to REAL, ingested_at REAL, + expired_at REAL, provenance TEXT DEFAULT '{}' + ); + INSERT INTO entities VALUES ('ent_a', 'ws_a', 'repo_a', 'Redis', 'concept', NULL, 1); + INSERT INTO entities VALUES ('ent_b', 'ws_a', 'repo_b', ' redis ', 'concept', NULL, 2); + INSERT INTO edges VALUES ( + 'edg_a', 'ws_a', NULL, 'ent_a', 'ent_b', 'uses', 'entity', 1, + 1, NULL, 1, NULL, + '{"source":"structured_extractor","memory_id":"mem_a","memory_ids":["mem_a","mem_b"]}' + ); + """) + conn.commit() + conn.close() + + store = Store(str(db)) + rows = [dict(row) for row in store.conn.execute( + "SELECT id, normalized_name, canonical_id, canonical_confidence " + "FROM entities ORDER BY id" + ).fetchall()] + supports = store.edge_supports_in_scope(["edg_a"], at=2) + + assert store.schema_version == 4 + assert {row["normalized_name"] for row in rows} == {"redis"} + assert len({row["canonical_id"] for row in rows}) == 1 + assert all(row["canonical_confidence"] == 1.0 for row in rows) + assert [(row["memory_id"], row["source_kind"], row["confidence"]) + for row in supports] == [ + ("mem_a", "structured", 0.8), + ("mem_b", "structured", 0.8), + ] + indexes = {row["name"] for row in store.conn.execute( + "SELECT name FROM sqlite_master WHERE type='index'" + ).fetchall()} + assert {"idx_entity_canonical", "idx_entity_normalized", + "idx_edge_support_edge", "idx_edge_support_memory"} <= indexes + store.close() + + +def test_v4_migration_converges_duplicate_live_relations_without_losing_support(tmp_path): + db = tmp_path / "duplicate-v3.db" + conn = sqlite3.connect(db) + conn.executescript(""" + CREATE TABLE schema_migrations(version INTEGER PRIMARY KEY, applied_at REAL); + INSERT INTO schema_migrations VALUES (3, 0); + CREATE TABLE edges ( + id TEXT PRIMARY KEY, workspace_id TEXT, repo_id TEXT, src TEXT NOT NULL, + dst TEXT NOT NULL, relation TEXT NOT NULL, layer TEXT DEFAULT 'semantic', + weight REAL DEFAULT 1.0, valid_from REAL, valid_to REAL, ingested_at REAL, + expired_at REAL, provenance TEXT DEFAULT '{}' + ); + INSERT INTO edges VALUES ( + 'edg_a', 'ws_a', NULL, 'ent_a', 'ent_b', 'uses', 'entity', 1, + 1, NULL, 1, NULL, '{"source":"manual","memory_id":"mem_a"}' + ); + INSERT INTO edges VALUES ( + 'edg_b', 'ws_a', NULL, 'ent_a', 'ent_b', 'uses', 'entity', 2, + 2, NULL, 2, NULL, '{"source":"structured","memory_id":"mem_b"}' + ); + """) + conn.commit() + conn.close() + + store = Store(str(db)) + live = store.conn.execute( + "SELECT id, weight, provenance FROM edges WHERE valid_to IS NULL" + ).fetchall() + retired = store.conn.execute( + "SELECT id, valid_to FROM edges WHERE valid_to IS NOT NULL" + ).fetchall() + supports = store.edge_supports_in_scope(["edg_a"]) + + assert len(live) == 1 and live[0]["id"] == "edg_a" and live[0]["weight"] == 2 + assert len(retired) == 1 and retired[0]["id"] == "edg_b" + assert {row["memory_id"] for row in supports} == {"mem_a", "mem_b"} + provenance = json.loads(live[0]["provenance"]) + assert set(provenance["memory_ids"]) == {"mem_a", "mem_b"} + assert provenance["canonical_deduplicated_from"] == ["edg_b"] + indexes = {row["name"] for row in store.conn.execute( + "SELECT name FROM sqlite_master WHERE type='index'" + ).fetchall()} + assert {"idx_edge_workspace_live_unique", "idx_edge_repo_live_unique"} <= indexes + store.close() + + +def test_v4_migration_normalizes_reversed_undirected_endpoints_before_dedup(tmp_path): + db = tmp_path / "reversed-undirected-v3.db" + conn = sqlite3.connect(db) + conn.executescript(""" + CREATE TABLE schema_migrations(version INTEGER PRIMARY KEY, applied_at REAL); + INSERT INTO schema_migrations VALUES (3, 0); + CREATE TABLE edges ( + id TEXT PRIMARY KEY, workspace_id TEXT, repo_id TEXT, src TEXT NOT NULL, + dst TEXT NOT NULL, relation TEXT NOT NULL, layer TEXT DEFAULT 'semantic', + weight REAL DEFAULT 1.0, valid_from REAL, valid_to REAL, ingested_at REAL, + expired_at REAL, provenance TEXT DEFAULT '{}' + ); + INSERT INTO edges VALUES ( + 'edg_old', 'ws_a', NULL, 'z', 'a', 'co_occurs', 'semantic', 0.2, + 1, NULL, 1, NULL, '{"source":"regex","memory_id":"mem_a"}' + ); + INSERT INTO edges VALUES ( + 'edg_new', 'ws_a', NULL, 'a', 'z', 'co_occurs', 'semantic', 0.4, + 2, NULL, 2, NULL, '{"source":"regex","memory_id":"mem_b"}' + ); + INSERT INTO edges VALUES ( + 'edg_single', 'ws_a', NULL, 'y', 'x', 'related', 'semantic', 1, + 3, NULL, 3, NULL, '{}' + ); + """) + conn.commit() + conn.close() + + store = Store(str(db)) + cooccurs = store.conn.execute( + "SELECT id, src, dst, weight FROM edges " + "WHERE relation='co_occurs' AND valid_to IS NULL" + ).fetchall() + singleton = store.conn.execute( + "SELECT src, dst FROM edges WHERE id='edg_single'" + ).fetchone() + + assert [(row["id"], row["src"], row["dst"], row["weight"]) + for row in cooccurs] == [("edg_old", "a", "z", 0.4)] + assert (singleton["src"], singleton["dst"]) == ("x", "y") + assert {row["memory_id"] for row in store.edge_supports_in_scope(["edg_old"])} == { + "mem_a", "mem_b", + } + store.close() + + +def test_edge_writer_merges_equivalent_live_relation_supports(): + store = Store(":memory:") + workspace_id = store.get_or_create_workspace("acme") + first = store.upsert_edge(Edge( + id="edg_first", src="ent_a", dst="ent_b", relation="uses", + workspace_id=workspace_id, + provenance={"source": "manual", "memory_id": "mem_a"}, + )) + second = store.upsert_edge(Edge( + id="edg_second", src="ent_a", dst="ent_b", relation="uses", + workspace_id=workspace_id, + provenance={"source": "structured", "memory_id": "mem_b"}, + )) + + assert first == second == "edg_first" + assert store.conn.execute( + "SELECT COUNT(*) AS n FROM edges WHERE workspace_id=? AND valid_to IS NULL", + (workspace_id,), + ).fetchone()["n"] == 1 + supports = store.edge_supports_in_scope([first]) + assert {row["memory_id"] for row in supports} == {"mem_a", "mem_b"} + store.close() + + +def test_v4_reopen_keeps_graph_generation_stable_when_backfill_is_already_complete(tmp_path): + database = tmp_path / "stable-generation.db" + store = Store(str(database)) + workspace_id = store.get_or_create_workspace("acme") + store.upsert_entity(Node( + id="", name="Engraphis", ntype="concept", workspace_id=workspace_id, + )) + edge_id = store.upsert_edge(Edge( + id="edg_historical", src="ent_a", dst="ent_b", relation="uses", + workspace_id=workspace_id, + provenance={"source": "structured", "memory_id": "mem_historical"}, + )) + store.invalidate_edge(edge_id, at=42.0) + supports_before = store.conn.execute( + "SELECT COUNT(*) AS n FROM edge_supports WHERE edge_id=?", (edge_id,), + ).fetchone()["n"] + before = store.conn.execute( + "SELECT generation FROM graph_index_state WHERE workspace_id=?", + (workspace_id,), + ).fetchone()["generation"] + store.close() + + reopened = Store(str(database)) + after = reopened.conn.execute( + "SELECT generation FROM graph_index_state WHERE workspace_id=?", + (workspace_id,), + ).fetchone()["generation"] + supports_after = reopened.conn.execute( + "SELECT COUNT(*) AS n FROM edge_supports WHERE edge_id=?", (edge_id,), + ).fetchone()["n"] + + assert after == before + assert supports_after == supports_before == 1 + reopened.close() + + +def test_partial_live_edge_indexes_allow_history_but_reject_active_duplicates(): + store = Store(":memory:") + workspace_id = store.get_or_create_workspace("acme") + repo_id = store.get_or_create_repo(workspace_id, "web") + sql = ( + "INSERT INTO edges(id, workspace_id, repo_id, src, dst, relation, layer, " + "valid_from, valid_to, expired_at) VALUES (?,?,?,?,?,?,?,?,?,?)" + ) + store.conn.execute(sql, ( + "workspace_live", workspace_id, None, "a", "b", "uses", "entity", + 1.0, None, None, + )) + with pytest.raises(sqlite3.IntegrityError): + store.conn.execute(sql, ( + "workspace_duplicate", workspace_id, None, "a", "b", "uses", "entity", + 2.0, None, None, + )) + store.conn.execute(sql, ( + "workspace_history", workspace_id, None, "a", "b", "uses", "entity", + 0.0, 0.5, None, + )) + store.conn.execute(sql, ( + "workspace_expired", workspace_id, None, "a", "b", "uses", "entity", + 0.0, None, 0.5, + )) + store.conn.execute(sql, ( + "repo_live", workspace_id, repo_id, "a", "b", "uses", "entity", + 1.0, None, None, + )) + with pytest.raises(sqlite3.IntegrityError): + store.conn.execute(sql, ( + "repo_duplicate", workspace_id, repo_id, "a", "b", "uses", "entity", + 2.0, None, None, + )) + store.conn.execute(sql, ( + "repo_history", workspace_id, repo_id, "a", "b", "uses", "entity", + 0.0, 0.5, None, + )) + store.conn.commit() + + assert store.conn.execute( + "SELECT COUNT(*) AS n FROM edges WHERE src='a' AND dst='b'" + ).fetchone()["n"] == 5 + store.close() + + +def test_repeating_same_edge_writer_is_a_storage_noop(): + store = Store(":memory:") + workspace_id = store.get_or_create_workspace("acme") + edge = Edge( + id="edg_repeat", src="ent_a", dst="ent_b", relation="uses", + workspace_id=workspace_id, + provenance={"source": "structured", "memory_id": "mem_a"}, + ) + first = store.upsert_edge(edge) + stored_before = dict(store.conn.execute( + "SELECT valid_from, ingested_at, provenance FROM edges WHERE id=?", (first,) + ).fetchone()) + generation_before = store.conn.execute( + "SELECT generation FROM graph_index_state WHERE workspace_id=?", (workspace_id,) + ).fetchone()["generation"] + + second = store.upsert_edge(edge) + + stored_after = dict(store.conn.execute( + "SELECT valid_from, ingested_at, provenance FROM edges WHERE id=?", (second,) + ).fetchone()) + support_rows = store.conn.execute( + "SELECT valid_to, expired_at FROM edge_supports WHERE edge_id=?", (first,) + ).fetchall() + generation_after = store.conn.execute( + "SELECT generation FROM graph_index_state WHERE workspace_id=?", (workspace_id,) + ).fetchone()["generation"] + assert first == second == "edg_repeat" + assert stored_after == stored_before + assert len(support_rows) == 1 + assert support_rows[0]["valid_to"] is None and support_rows[0]["expired_at"] is None + assert generation_after == generation_before + store.close() + + +def test_repeated_support_writer_keeps_one_live_row_and_upgrades_confidence(): + store = Store(":memory:") + workspace_id = store.get_or_create_workspace("acme") + edge_id = store.upsert_edge(Edge( + id="edg_support", src="ent_a", dst="ent_b", relation="uses", + workspace_id=workspace_id, + )) + store.add_edge_support(edge_id, { + "source": "structured", "memory_id": "mem_a", "confidence": 0.6, + }) + store.add_edge_support(edge_id, { + "source": "structured", "memory_id": "mem_a", "confidence": 0.95, + }) + + supports = store.conn.execute( + "SELECT memory_id, source_kind, confidence FROM edge_supports " + "WHERE edge_id=? AND valid_to IS NULL AND expired_at IS NULL", + (edge_id,), + ).fetchall() + assert [(row["memory_id"], row["source_kind"], row["confidence"]) + for row in supports] == [("mem_a", "structured", 0.95)] + store.close() + + +def test_undirected_equivalent_writers_converge_after_endpoint_normalization(): + store = Store(":memory:") + workspace_id = store.get_or_create_workspace("acme") + first = store.upsert_edge(Edge( + id="edg_forward", src="ent_a", dst="ent_b", relation="co_occurs", + workspace_id=workspace_id, + provenance={"source": "regex", "memory_id": "mem_a"}, + )) + second = store.upsert_edge(Edge( + id="edg_reverse", src="ent_b", dst="ent_a", relation="co_occurs", + workspace_id=workspace_id, + provenance={"source": "regex", "memory_id": "mem_b"}, + )) + + row = store.conn.execute( + "SELECT id, src, dst FROM edges WHERE workspace_id=? AND valid_to IS NULL", + (workspace_id,), + ).fetchone() + assert first == second == row["id"] == "edg_forward" + assert (row["src"], row["dst"]) == ("ent_a", "ent_b") + assert {item["memory_id"] for item in store.edge_supports_in_scope([first])} == { + "mem_a", "mem_b", + } + store.close() + + +def test_scene_is_canonical_deterministic_and_strength_shortens_links(): + entities = [ + {"id": "a1", "canonical_id": "a1", "name": "Alpha", "etype": "concept", + "repo_id": "r1"}, + {"id": "a2", "canonical_id": "a1", "name": "alpha!", "etype": "concept", + "repo_id": "r2"}, + {"id": "b", "canonical_id": "b", "name": "Beta", "etype": "concept", + "repo_id": "r1"}, + {"id": "c", "canonical_id": "c", "name": "Gamma", "etype": "concept", + "repo_id": "r1"}, + ] + edges = [ + {"id": "strong", "src": "a1", "dst": "b", "relation": "uses", + "layer": "entity", "weight": 4.0, "provenance": "{}"}, + {"id": "weak", "src": "b", "dst": "c", "relation": "related_to", + "layer": "semantic", "weight": 0.05, "provenance": "{}"}, + {"id": "noise", "src": "a2", "dst": "c", "relation": "co_occurs", + "layer": "semantic", "weight": 0.2, "provenance": "{}"}, + ] + supports = [ + {"edge_id": "strong", "memory_id": "m1", "source_kind": "manual", + "confidence": 0.99, "provenance": "{}"}, + {"edge_id": "strong", "memory_id": "m2", "source_kind": "manual", + "confidence": 0.99, "provenance": "{}"}, + {"edge_id": "weak", "memory_id": "m3", "source_kind": "legacy_unknown", + "confidence": 0.5, "provenance": "{}"}, + {"edge_id": "noise", "memory_id": "m4", "source_kind": "co_occurrence", + "confidence": 0.25, "provenance": "{}"}, + ] + + first = build_graph_scene("w", entities, edges, supports) + second = build_graph_scene("w", entities, edges, supports) + + assert first == second + assert first["meta"]["total_nodes"] == 3 # a1/a2 collapse to one canonical entity + assert "noise" not in {edge["id"] for edge in first["edges"]} + by_id = {edge["id"]: edge for edge in first["edges"]} + assert by_id["strong"]["strength"] > by_id["weak"]["strength"] + assert by_id["strong"]["rest_length"] < by_id["weak"]["rest_length"] + assert first["nodes"][0]["anchor_role"] == "global" + assert first["nodes"][0]["x"] == first["nodes"][0]["y"] == 0.0 + + +def test_complete_scene_keeps_every_memory_and_raw_connector_deterministically(): + entities = [ + {"id": "a1", "canonical_id": "a", "name": "Alpha", "etype": "concept"}, + {"id": "a2", "canonical_id": "a", "name": "alpha", "etype": "concept"}, + {"id": "b", "canonical_id": "b", "name": "Beta", "etype": "concept"}, + ] + edges = [ + {"id": "raw_one", "src": "a1", "dst": "b", "relation": "uses", + "layer": "entity", "weight": 1.0, "provenance": "{}"}, + {"id": "raw_two", "src": "a2", "dst": "b", "relation": "uses", + "layer": "entity", "weight": 0.8, "provenance": "{}"}, + ] + supports = [ + {"id": 1, "edge_id": "raw_one", "memory_id": "m1", + "source_kind": "structured", "confidence": 0.8, "provenance": "{}"}, + {"id": 2, "edge_id": "raw_two", "memory_id": "m2", + "source_kind": "manual", "confidence": 1.0, "provenance": "{}"}, + ] + memories = [ + {"id": "m1", "title": "Alpha uses Beta", "mtype": "semantic", + "scope": "workspace", "importance": 0.8}, + {"id": "m2", "title": "Second source", "mtype": "episodic", + "scope": "workspace", "importance": 0.5}, + {"id": "m3", "title": "Unattached memory", "mtype": "procedural", + "scope": "workspace", "importance": 0.3}, + ] + memory_links = [{ + "a": "m1", "b": "m3", "relation": "related", "layer": "semantic", + "reason": "manual", "created_at": 10, + }] + + kwargs = { + "level": "complete", "memory_rows": memories, + "memory_link_rows": memory_links, "include_weak_cooccurrence": True, + } + first = build_graph_scene("w", entities, edges, supports, **kwargs) + second = build_graph_scene("w", entities, edges, supports, **kwargs) + + assert first == second + assert first["meta"]["complete_scene"] is True + assert first["meta"]["truncated"] is False + assert first["meta"]["entity_nodes"] == 2 + assert first["meta"]["memory_nodes"] == 3 + assert first["meta"]["raw_relations"] == 2 + assert first["meta"]["evidence_connectors"] == 4 + assert first["meta"]["memory_connectors"] == 1 + assert first["meta"]["shown_nodes"] == first["meta"]["total_nodes"] == 5 + assert first["meta"]["shown_edges"] == first["meta"]["total_edges"] == 7 + assert {node["node_kind"] for node in first["nodes"]} == {"entity", "memory"} + by_kind = {} + for edge in first["edges"]: + by_kind.setdefault(edge["connector_kind"], set()).add(edge["id"]) + assert by_kind["entity_relation"] == {"raw_one", "raw_two"} + assert len(by_kind["evidence"]) == 4 + assert len(by_kind["memory_link"]) == 1 + nodes = {node["id"]: node for node in first["nodes"]} + for community in first["communities"]: + community_id = community["id"] + expected_internal = sum( + edge["strength"] for edge in first["edges"] + if nodes[edge["source"]]["community_id"] == community_id + and nodes[edge["target"]]["community_id"] == community_id + ) + expected_external = sum( + edge["strength"] for edge in first["edges"] + if ((nodes[edge["source"]]["community_id"] == community_id) + != (nodes[edge["target"]]["community_id"] == community_id)) + ) + assert community["internal_strength"] == pytest.approx(expected_internal) + assert community["external_strength"] == pytest.approx(expected_external) + + +def test_complete_scene_keeps_every_enabled_code_memory_connector(): + entities = [{ + "id": "code:symbol", "canonical_id": "code:symbol", + "name": "repo:module.fn", "etype": "code_function", "repo_id": "repo", + }] + memories = [{ + "id": "memory", "title": "Function behavior", "mtype": "procedural", + "scope": "repo", "repo_id": "repo", "importance": 0.5, + }] + links = [ + {"id": "code-link-b", "symbol_id": "symbol", "memory_id": "memory", + "relation": "mentions", "confidence": 0.7}, + {"id": "code-link-a", "symbol_id": "symbol", "memory_id": "memory", + "relation": "documents", "confidence": 1.0}, + ] + + scene = build_graph_scene( + "w", entities, [], [], level="complete", memory_rows=memories, + code_memory_link_rows=links, + ) + + code_edges = [edge for edge in scene["edges"] + if edge["connector_kind"] == "code_memory"] + assert [edge["id"] for edge in code_edges] == ["code-link-a", "code-link-b"] + assert {(edge["source"], edge["target"]) for edge in code_edges} == { + ("memory", "code:symbol") + } + assert scene["meta"]["code_memory_connectors"] == 2 + + +def test_community_bridges_keep_aggregate_evidence_for_physics(): + nodes = { + "a": {"community_id": "ca"}, + "b": {"community_id": "cb"}, + "c": {"community_id": "cc"}, + } + + def edge(edge_id, source, target, strength, support_ids, support_count, + bundled_edge_count, relation="uses"): + return { + "id": edge_id, + "source": source, + "target": target, + "layer": "entity", + "relation": relation, + "strength": strength, + "_support_ids_all": set(support_ids), + "support_count": support_count, + "bundled_edge_count": bundled_edge_count, + } + + edges = [ + # Both public display strengths saturate at one. Physics must still see that + # the a-c bridge has materially more aggregate evidence than a-b. + edge("ab_known", "a", "b", 0.6, {"m1"}, 1, 1), + edge("ab_legacy", "a", "b", 0.6, set(), 2, 2), + edge("ac_one", "a", "c", 0.9, {"m2", "m3", "m4"}, 3, 3), + edge("ac_two", "a", "c", 0.9, {"m5", "m6", "m7"}, 3, 3), + edge("ac_three", "a", "c", 0.9, {"m8", "m9"}, 2, 2), + ] + graph = {"nodes": nodes, "edges": edges} + + first = graph_scene_module._bridges(graph, {"ca", "cb", "cc"}, 80) + second = graph_scene_module._bridges( + {"nodes": nodes, "edges": list(reversed(edges))}, {"ca", "cb", "cc"}, 80 + ) + + assert first == second + by_pair = { + (bridge["source_community"], bridge["target_community"]): bridge + for bridge in first + } + weaker = by_pair[("ca", "cb")] + stronger = by_pair[("ca", "cc")] + assert weaker["strength"] == stronger["strength"] == 1.0 + assert weaker["aggregate_strength"] == pytest.approx(1.2) + assert stronger["aggregate_strength"] == pytest.approx(2.7) + assert stronger["physics_strength"] > weaker["physics_strength"] + assert 0.0 <= weaker["physics_strength"] <= 1.0 + assert 0.0 <= stronger["physics_strength"] <= 1.0 + # Known support IDs and anonymous legacy support are both represented. + assert weaker["support_count"] == 3 + assert stronger["support_count"] == 8 + assert weaker["edge_count"] == 3 + assert stronger["edge_count"] == 8 + + +def test_canonical_bundle_filters_use_aggregate_support_and_confidence(): + entities = [ + {"id": "a1", "canonical_id": "a", "name": "Alpha", "etype": "concept"}, + {"id": "a2", "canonical_id": "a", "name": "alpha", "etype": "concept"}, + {"id": "b1", "canonical_id": "b", "name": "Beta", "etype": "concept"}, + {"id": "b2", "canonical_id": "b", "name": "beta", "etype": "concept"}, + ] + edges = [ + {"id": "one", "src": "a1", "dst": "b1", "relation": "co_occurs", + "layer": "semantic", "weight": 1.0, "provenance": "{}"}, + {"id": "two", "src": "a2", "dst": "b2", "relation": "co_occurs", + "layer": "semantic", "weight": 1.0, "provenance": "{}"}, + ] + supports = [ + {"edge_id": "one", "memory_id": "m1", "confidence": 0.25, + "provenance": "{}"}, + {"edge_id": "two", "memory_id": "m2", "confidence": 0.25, + "provenance": "{}"}, + ] + + graph = graph_scene_module.build_canonical_graph( + entities, edges, supports, include_weak_cooccurrence=False, + min_support=2, min_confidence=0.4, + ) + + assert len(graph["edges"]) == 1 + edge = graph["edges"][0] + assert edge["support_count"] == 2 + assert edge["confidence"] == pytest.approx(0.4375) + assert edge["bundled_edge_count"] == 2 + + +def test_edge_support_count_includes_identified_and_anonymous_evidence(): + entities = [ + {"id": "a", "name": "Alpha", "etype": "concept"}, + {"id": "b", "name": "Beta", "etype": "concept"}, + ] + edges = [{ + "id": "edge", "src": "a", "dst": "b", "relation": "uses", + "layer": "entity", "weight": 1.0, "provenance": "{}", + }] + supports = [ + {"edge_id": "edge", "memory_id": "known", "confidence": 0.8, + "provenance": "{}"}, + {"edge_id": "edge", "memory_id": "", "confidence": 0.5, + "provenance": "{}"}, + ] + + edge = graph_scene_module.build_canonical_graph(entities, edges, supports)["edges"][0] + + assert edge["support_count"] == 2 + assert edge["support_memory_ids"] == ["known"] + + +def test_overview_ranks_communities_by_the_mass_sent_to_physics(monkeypatch): + nodes = {} + community_members = {} + community_anchors = {} + + def add_community(community_id, gravity_masses, *, global_anchor=False): + member_ids = [] + for index, gravity_mass in enumerate(gravity_masses): + node_id = f"{community_id}_{index}" + member_ids.append(node_id) + nodes[node_id] = { + "id": node_id, + "canonical_id": node_id, + "label": node_id, + "type": "concept", + "member_ids": [node_id], + "member_count": 1, + "repo_ids": [], + "weighted_degree": 0.0, + "pagerank": 0.0, + "support_count": 0, + "entity_quality": 1.0, + "mass_score": 0.5, + "gravity_mass": gravity_mass, + "visual_radius": 5.0, + "component_id": f"component_{community_id}", + "community_id": community_id, + "anchor_role": "global" if global_anchor and index == 0 else ( + "community" if index == 0 else "none" + ), + "core_affinity": 0.5, + "scene_rank": 0.5, + } + community_members[community_id] = member_ids + community_anchors[community_id] = member_ids[0] + + add_community("c0", [8.0], global_anchor=True) + add_community("ca", [8.0]) + add_community("cb", [3.0, 3.0]) + for index in range(22): + add_community(f"f{index:02d}", [3.2, 3.2]) + + fake_graph = { + "nodes": nodes, + "edges": [], + "member_to_canonical": {node_id: node_id for node_id in nodes}, + "community_members": community_members, + "community_anchors": community_anchors, + "global_anchor": "c0_0", + } + monkeypatch.setattr( + graph_scene_module, "build_canonical_graph", lambda *_args, **_kwargs: fake_graph + ) + + scene = graph_scene_module.build_graph_scene("w", [], [], []) + chosen = {community["id"] for community in scene["communities"]} + + # ca has the larger raw sum (8 > 6), but cb has the larger system mass + # (sqrt(3) + sqrt(3) > sqrt(8)) and is therefore the community physics ranks. + assert "cb" in chosen + assert "ca" not in chosen + + +def test_overview_excludes_obvious_regex_extraction_noise_even_when_connected(): + entities = [ + {"id": "product", "name": "Engraphis", "etype": "person_or_concept"}, + {"id": "graph", "name": "Knowledge Graph", "etype": "person_or_concept"}, + {"id": "all", "name": "All", "etype": "person_or_concept"}, + {"id": "true", "name": "True", "etype": "person_or_concept"}, + {"id": "check", "name": "Check", "etype": "person_or_concept"}, + {"id": "if_python", "name": "If Python System", "etype": "person_or_concept"}, + {"id": "python_side", "name": "Python-side", "etype": "person_or_concept"}, + {"id": "generated", "name": "Generated Response", "etype": "person_or_concept"}, + {"id": "supported", "name": "Supported Python-version", + "etype": "person_or_concept"}, + ] + edges = [ + {"id": "good", "src": "product", "dst": "graph", "relation": "uses", + "layer": "entity", "weight": 1.0, "provenance": "{}"}, + {"id": "noise-a", "src": "all", "dst": "product", "relation": "uses", + "layer": "entity", "weight": 4.0, "provenance": "{}"}, + {"id": "noise-b", "src": "true", "dst": "product", "relation": "uses", + "layer": "entity", "weight": 4.0, "provenance": "{}"}, + {"id": "noise-c", "src": "check", "dst": "product", "relation": "uses", + "layer": "entity", "weight": 4.0, "provenance": "{}"}, + {"id": "noise-d", "src": "if_python", "dst": "product", "relation": "uses", + "layer": "entity", "weight": 4.0, "provenance": "{}"}, + {"id": "noise-e", "src": "python_side", "dst": "product", "relation": "uses", + "layer": "entity", "weight": 4.0, "provenance": "{}"}, + {"id": "noise-f", "src": "generated", "dst": "product", "relation": "uses", + "layer": "entity", "weight": 4.0, "provenance": "{}"}, + {"id": "noise-g", "src": "supported", "dst": "product", "relation": "uses", + "layer": "entity", "weight": 4.0, "provenance": "{}"}, + ] + + scene = build_graph_scene("w", entities, edges, [], level="overview") + system = build_graph_scene("w", entities, edges, [], level="system", system_id="product") + explicit = build_graph_scene( + "w", entities, edges, [], level="neighborhood", center_id="if_python", depth=0 + ) + + assert {node["label"] for node in scene["nodes"]} == {"Engraphis", "Knowledge Graph"} + assert all(node["entity_quality"] == 1.0 for node in scene["nodes"]) + assert {node["label"] for node in system["nodes"]} <= {"Engraphis", "Knowledge Graph"} + assert "Engraphis" in {node["label"] for node in system["nodes"]} + assert all(node["entity_quality"] == 1.0 for node in system["nodes"]) + assert [node["id"] for node in explicit["nodes"]] == ["if_python"] + assert explicit["nodes"][0]["entity_quality"] == 0.0 + + +def test_zero_evidence_ties_do_not_turn_isolates_into_maximum_mass_nodes(): + entities = [ + {"id": "a", "name": "Alpha", "etype": "concept"}, + {"id": "b", "name": "Beta", "etype": "concept"}, + {"id": "c", "name": "Gamma", "etype": "concept"}, + ] + connected = [{ + "id": "ab", "src": "a", "dst": "b", "relation": "uses", + "layer": "entity", "weight": 1.0, "provenance": "{}", + }] + + isolated_graph = graph_scene_module.build_canonical_graph(entities, [], []) + mixed_graph = graph_scene_module.build_canonical_graph(entities, connected, []) + + assert max(node["mass_score"] for node in isolated_graph["nodes"].values()) < 0.5 + assert mixed_graph["nodes"]["c"]["mass_score"] < mixed_graph["nodes"]["a"]["mass_score"] + assert mixed_graph["nodes"]["c"]["mass_score"] < mixed_graph["nodes"]["b"]["mass_score"] + + +def test_scene_bounds_public_support_ids_and_deduplicates_confidence(): + entities = [ + {"id": "a", "canonical_id": "a", "name": "Alpha", "etype": "concept"}, + {"id": "b", "canonical_id": "b", "name": "Beta", "etype": "concept"}, + ] + edges = [{"id": "edge", "src": "a", "dst": "b", "relation": "uses", + "layer": "entity", "weight": 1.0, "provenance": "{}"}] + supports = [ + {"edge_id": "edge", "memory_id": "same", "source_kind": "manual", + "confidence": 0.5, "provenance": "{}"}, + {"edge_id": "edge", "memory_id": "same", "source_kind": "structured", + "confidence": 0.5, "provenance": "{}"}, + *[ + {"edge_id": "edge", "memory_id": f"mem_{index:03d}", + "source_kind": "manual", "confidence": 0.5, "provenance": "{}"} + for index in range(205) + ], + ] + + scene = build_graph_scene("w", entities, edges, supports) + edge = scene["edges"][0] + + assert edge["support_count"] == 206 + assert len(edge["support_memory_ids"]) == 200 + assert edge["support_ids_truncated"] is True + # Repeated normalized rows for one memory are one source, not two independent votes. + baseline = build_graph_scene("w", entities, edges, supports[:2])["edges"][0] + assert baseline["confidence"] == pytest.approx(0.5) + + +def _seed_service() -> tuple[MemoryService, str, str, str]: + service = MemoryService.create(":memory:", graph_extractor="none") + workspace_id = service.store.get_or_create_workspace("acme") + memory_a = service.store.add_memory(MemoryRecord( + id="", content="Alpha uses Beta.", workspace_id=workspace_id, + scope=Scope.WORKSPACE, + )) + memory_b = service.store.add_memory(MemoryRecord( + id="", content="Beta causes Gamma.", workspace_id=workspace_id, + scope=Scope.WORKSPACE, + )) + alpha = service.store.upsert_entity(Node( + id="", name="Alpha", ntype="concept", workspace_id=workspace_id, + )) + beta = service.store.upsert_entity(Node( + id="", name="Beta", ntype="concept", workspace_id=workspace_id, + )) + gamma = service.store.upsert_entity(Node( + id="", name="Gamma", ntype="concept", workspace_id=workspace_id, + )) + service.store.upsert_edge(Edge( + id="edge_ab", src=alpha, dst=beta, relation="uses", workspace_id=workspace_id, + provenance={"source": "structured_extractor", "memory_id": memory_a}, + )) + service.store.upsert_edge(Edge( + id="edge_bg", src=beta, dst=gamma, relation="causes", workspace_id=workspace_id, + provenance={"source": "manual", "memory_id": memory_b}, + )) + return service, alpha, beta, gamma + + +def test_graph_explorer_endpoints_and_legacy_graph_gets_are_read_only(): + service, alpha, _beta, gamma = _seed_service() + app = FastAPI() + app.include_router(v2_api.router) + v2_api.set_service(service) + client = TestClient(app) + + before = { + table: service.store.conn.execute( + f"SELECT COUNT(*) AS n FROM {table}" + ).fetchone()["n"] + for table in ("entities", "edges", "edge_supports") + } + scene_response = client.get("/api/graph/scene", params={"workspace": "acme"}) + assert scene_response.status_code == 200 + scene = scene_response.json() + assert set(scene) == { + "meta", "nodes", "edges", "communities", "community_bridges", "facets" + } + assert scene["meta"]["shown_nodes"] == 3 + + suggestions = client.get( + "/api/graph/suggest", params={"workspace": "acme", "query": "alp"} + ).json() + assert suggestions["groups"]["entities"][0]["label"] == "Alpha" + + detail = client.get( + f"/api/graph/entities/{alpha}", params={"workspace": "acme"} + ) + assert detail.status_code == 200 + assert detail.json()["canonical_id"] == alpha + assert detail.json()["evidence"] + + path = client.get("/api/graph/path", params={ + "workspace": "acme", "source": alpha, "target": gamma, + }).json() + assert path["found"] is True + assert path["edge_ids"] == ["edge_ab", "edge_bg"] + + # A pre-existing memory with extraction subsequently enabled must not be lazily + # materialized by either the compatibility GET or the new scene GET. + service.remember("Delta works at Example Corp.", workspace="acme", scope="workspace") + service.engine.graph_extractor = get_graph_extractor("regex") + before_lazy = service.store.conn.execute( + "SELECT COUNT(*) AS n FROM entities" + ).fetchone()["n"] + assert client.get("/api/graph", params={"workspace": "acme"}).status_code == 200 + assert client.get("/api/graph/scene", params={"workspace": "acme"}).status_code == 200 + after_lazy = service.store.conn.execute( + "SELECT COUNT(*) AS n FROM entities" + ).fetchone()["n"] + assert after_lazy == before_lazy + + after = { + table: service.store.conn.execute( + f"SELECT COUNT(*) AS n FROM {table}" + ).fetchone()["n"] + for table in ("entities", "edges", "edge_supports") + } + assert after == before + + +def test_complete_scene_api_returns_all_scoped_memories_and_connector_kinds(): + service, _alpha, _beta, _gamma = _seed_service() + workspace_id = service.store.conn.execute( + "SELECT id FROM workspaces WHERE name='acme'" + ).fetchone()["id"] + existing = [row["id"] for row in service.store.conn.execute( + "SELECT id FROM memories WHERE workspace_id=? ORDER BY id", (workspace_id,) + ).fetchall()] + third = service.store.add_memory(MemoryRecord( + id="", content="A standalone procedural memory.", + mtype=MemoryType.PROCEDURAL, workspace_id=workspace_id, + scope=Scope.WORKSPACE, + )) + service.store.add_link(existing[0], third, relation="related", reason="manual") + app = FastAPI() + app.include_router(v2_api.router) + v2_api.set_service(service) + client = TestClient(app) + + response = client.get("/api/graph/scene", params={ + "workspace": "acme", "level": "complete", + }) + + assert response.status_code == 200 + scene = response.json() + meta = scene["meta"] + assert meta["level"] == "complete" + assert meta["complete_scene"] is True + assert meta["safety_state"] == "full" + assert meta["degraded"] is False + assert meta["truncated"] is False + assert meta["memory_nodes"] == 3 + assert meta["entity_nodes"] == 3 + assert meta["raw_relations"] == 2 + assert meta["evidence_connectors"] == 4 + assert meta["memory_connectors"] == 1 + assert meta["payload_bytes_estimate"] > 0 + assert meta["shown_nodes"] == meta["total_nodes"] + assert meta["shown_edges"] == meta["total_edges"] + assert {node["id"] for node in scene["nodes"] if node["node_kind"] == "memory"} \ + == {*existing, third} + assert {edge["id"] for edge in scene["edges"] + if edge["connector_kind"] == "entity_relation"} == {"edge_ab", "edge_bg"} + assert all(bridge["edge_ids_truncated"] is False + for bridge in scene["community_bridges"]) + + limited = client.get("/api/graph/scene", params={ + "workspace": "acme", "level": "complete", "node_limit": 3, + }) + assert limited.status_code == 400 + assert "do not accept node_limit" in limited.json()["detail"]["error"] + + +def test_complete_scene_capacity_error_is_explicit_and_never_samples(monkeypatch): + service = MemoryService.create(":memory:", graph_extractor="none") + workspace_id = service.store.get_or_create_workspace("acme") + for content in ("one", "two"): + service.store.add_memory(MemoryRecord( + id="", content=content, workspace_id=workspace_id, + scope=Scope.WORKSPACE, + )) + monkeypatch.setattr(service_module, "MAX_GRAPH_COMPLETE_MEMORIES", 1) + app = FastAPI() + app.include_router(v2_api.router) + v2_api.set_service(service) + response = TestClient(app).get("/api/graph/scene", params={ + "workspace": "acme", "level": "complete", + }) + + assert response.status_code == 413 + detail = response.json()["detail"] + assert detail["safety_state"] == "capacity_exceeded" + assert detail["degraded"] is True + assert detail["truncated"] is False + assert detail["resource"] == "memory nodes" + assert detail["count"] == 2 + assert detail["limit"] == 1 + + +def test_graph_scene_filters_supporting_memory_type_and_time_window(): + service = MemoryService.create(":memory:", graph_extractor="none") + workspace_id = service.store.get_or_create_workspace("acme") + semantic = service.store.add_memory(MemoryRecord( + id="", content="Alpha uses Beta.", mtype=MemoryType.SEMANTIC, + workspace_id=workspace_id, scope=Scope.WORKSPACE, valid_from=100, + ingested_at=100, + )) + procedural = service.store.add_memory(MemoryRecord( + id="", content="Beta deploys Gamma.", mtype=MemoryType.PROCEDURAL, + workspace_id=workspace_id, scope=Scope.WORKSPACE, valid_from=200, + ingested_at=200, + )) + alpha = service.store.upsert_entity(Node( + id="", name="Alpha", ntype="concept", workspace_id=workspace_id, + )) + beta = service.store.upsert_entity(Node( + id="", name="Beta", ntype="concept", workspace_id=workspace_id, + )) + gamma = service.store.upsert_entity(Node( + id="", name="Gamma", ntype="concept", workspace_id=workspace_id, + )) + service.store.upsert_edge(Edge( + id="edge_semantic", src=alpha, dst=beta, relation="uses", + workspace_id=workspace_id, valid_from=100, ingested_at=100, + provenance={"source": "structured", "memory_id": semantic}, + )) + service.store.upsert_edge(Edge( + id="edge_procedural", src=beta, dst=gamma, relation="deploys", + workspace_id=workspace_id, valid_from=200, ingested_at=200, + provenance={"source": "manual", "memory_id": procedural}, + )) + app = FastAPI() + app.include_router(v2_api.router) + v2_api.set_service(service) + client = TestClient(app) + + response = client.get("/api/graph/scene", params={ + "workspace": "acme", "memory_types": "procedural", + "time_from": 150, "time_to": 250, + }) + + assert response.status_code == 200 + scene = response.json() + assert {edge["id"] for edge in scene["edges"]} == {"edge_procedural"} + assert {node["label"] for node in scene["nodes"]} == {"Beta", "Gamma"} + assert scene["meta"]["filters"]["memory_types"] == ["procedural"] + assert scene["meta"]["filters"]["time_from"] == 150 + assert scene["facets"]["memory_types"] == [ + {"value": "procedural", "count": 1} + ] + context = { + "workspace": "acme", "memory_types": "procedural", + "time_from": 150, "time_to": 250, + "include_weak_cooccurrence": False, + } + suggestions = client.get( + "/api/graph/suggest", params={**context, "q": "Alpha"} + ).json() + # Identity search remains complete-index even while evidence-backed memory + # suggestions honor the active memory/time scope. + assert [item["id"] for item in suggestions["groups"]["entities"]] == [alpha] + assert suggestions["groups"]["memories"] == [] + detail = client.get(f"/api/graph/entities/{beta}", params=context).json() + assert {edge["id"] for edge in detail["relations"]} == {"edge_procedural"} + path = client.get("/api/graph/path", params={ + **context, "source": alpha, "target": gamma, + }).json() + assert path["found"] is False + assert client.get("/api/graph/scene", params={ + "workspace": "acme", "time_from": 250, "time_to": 150, + }).status_code == 400 + + +def test_graph_suggest_does_not_let_extractor_fragments_crowd_out_exact_identity(): + assert graph_scene_module.is_obvious_entity_noise( + "Full Python", "person_or_concept", + ) is False + assert graph_scene_module.is_broad_search_fragment( + "Full Python", "person_or_concept", + ) is True + service = MemoryService.create(":memory:", graph_extractor="none") + workspace_id = service.store.get_or_create_workspace("acme") + python_id = service.store.upsert_entity(Node( + id="", name="Python", ntype="person_or_concept", workspace_id=workspace_id, + )) + fragment_id = service.store.upsert_entity(Node( + id="", name="If Python", ntype="person_or_concept", workspace_id=workspace_id, + )) + broad_fragments = [ + "Python-based", "No Python", "Add Python", "Added Python", "Full Python", + "Three Python", "Orphan-Python", "Ignored Python", "Ignores Python", + "Compiled Python", "Codex-descended Python", + ] + for name in broad_fragments: + service.store.upsert_entity(Node( + id="", name=name, ntype="person_or_concept", workspace_id=workspace_id, + )) + + broad = service.graph_suggest("Python", workspace="acme") + assert [item["id"] for item in broad["groups"]["entities"]] == [python_id] + assert [item["id"] for item in broad["groups"]["systems"]] == [python_id] + + exact_fragment = service.graph_suggest("If Python", workspace="acme") + assert [item["id"] for item in exact_fragment["groups"]["entities"]] == [fragment_id] + exact_id = service.graph_suggest(fragment_id, workspace="acme") + assert [item["id"] for item in exact_id["groups"]["entities"]] == [fragment_id] + + +def test_scene_hash_versions_physics_and_index_generation(): + entities = [ + {"id": "a", "name": "Alpha", "etype": "concept"}, + {"id": "b", "name": "Beta", "etype": "concept"}, + {"id": "c", "name": "Gamma", "etype": "concept"}, + ] + baseline_edges = [ + {"id": "ab", "src": "a", "dst": "b", "relation": "uses", + "layer": "entity", "weight": 0.1, "provenance": "{}"}, + {"id": "bc", "src": "b", "dst": "c", "relation": "uses", + "layer": "entity", "weight": 1.0, "provenance": "{}"}, + ] + stronger_edges = [dict(edge) for edge in baseline_edges] + stronger_edges[0]["weight"] = 4.0 + + baseline = build_graph_scene("w", entities, baseline_edges, [], index_generation=4) + stronger = build_graph_scene("w", entities, stronger_edges, [], index_generation=4) + next_generation = build_graph_scene( + "w", entities, baseline_edges, [], index_generation=5 + ) + + assert baseline["meta"]["scene_hash"] != stronger["meta"]["scene_hash"] + assert baseline["meta"]["scene_hash"] != next_generation["meta"]["scene_hash"] + assert baseline["meta"]["algorithm_version"] == "galaxy-v2" + + +def test_graph_scene_cache_is_warm_and_invalidates_on_store_write(): + service, _alpha, _beta, _gamma = _seed_service() + + first = service.graph_scene(workspace="acme") + second = service.graph_scene(workspace="acme") + + assert first["meta"]["cache_hit"] is False + assert second["meta"]["cache_hit"] is True + assert second["meta"]["scene_hash"] == first["meta"]["scene_hash"] + + workspace_id = service.store.get_or_create_workspace("acme") + service.store.upsert_entity(Node( + id="", name="Delta", ntype="concept", workspace_id=workspace_id, + )) + refreshed = service.graph_scene(workspace="acme") + + assert refreshed["meta"]["cache_hit"] is False + assert refreshed["meta"]["total_nodes"] == first["meta"]["total_nodes"] + 1 + assert refreshed["meta"]["index_generation"] > first["meta"]["index_generation"] + + +def test_explicit_graph_index_dry_run_is_persisted_counted_and_audited(): + service, _alpha, _beta, _gamma = _seed_service() + before = { + table: service.store.conn.execute( + f"SELECT COUNT(*) AS n FROM {table}" + ).fetchone()["n"] + for table in ("entities", "edges") + } + + started = service.start_graph_index_job(workspace="acme", dry_run=True) + deadline = time.time() + 5 + job = started + while job["state"] in {"queued", "running"} and time.time() < deadline: + time.sleep(0.01) + job = service.graph_index_job(started["id"], workspace="acme") + + assert job["state"] == "completed" + assert job["progress"] == 1.0 + assert job["counts"]["memories_scanned"] == 2 + assert job["counts"]["entity_mentions"] >= 3 + assert job["counts"]["entities_added"] == 0 + assert { + table: service.store.conn.execute( + f"SELECT COUNT(*) AS n FROM {table}" + ).fetchone()["n"] + for table in ("entities", "edges") + } == before + receipt = service.store.conn.execute( + "SELECT operation, status FROM operation_receipts " + "WHERE operation='graph_index' ORDER BY rowid DESC LIMIT 1" + ).fetchone() + assert dict(receipt) == {"operation": "graph_index", "status": "ok"} + + +def test_mutating_graph_index_is_bounded_atomic_and_returns_ready(): + service = MemoryService.create(":memory:", graph_extractor="none") + service.remember( + "Alice Johnson works at Acme Corporation.", + workspace="acme", + scope="workspace", + ) + + started = service.start_graph_index_job(workspace="acme", dry_run=False) + deadline = time.time() + 5 + job = started + while job["state"] in {"queued", "running"} and time.time() < deadline: + time.sleep(0.01) + job = service.graph_index_job(started["id"], workspace="acme") + + assert job["state"] == "completed" + assert job["counts"]["memories_scanned"] == 1 + assert service.graph_index_status(workspace="acme")["index"]["state"] == "ready" + assert service.store.conn.in_transaction is False + assert service.store.conn.execute( + "SELECT COUNT(*) AS n FROM entities" + ).fetchone()["n"] == 2 + + +def test_zero_item_graph_job_cancelled_before_claim_stays_cancelled(): + service = MemoryService.create(":memory:", graph_extractor="none") + workspace_id = service.store.get_or_create_workspace("acme") + now = time.time() + service.store.conn.execute( + "INSERT INTO jobs(id, workspace_id, kind, state, dry_run, total_items, " + "processed_items, counts, errors, request, cancel_requested, runner_id, " + "heartbeat_at, created_at) VALUES(?,?,?,?,?,?,?,?,?,?,?,?,?,?)", + ( + "job_cancelled", workspace_id, "graph_index", "queued", 1, 0, 0, + "{}", "[]", "{}", 1, service._graph_runner_id, now, now, + ), + ) + service.store.conn.commit() + + service._run_graph_index_job("job_cancelled") + job = service.graph_index_job("job_cancelled", workspace="acme") + + assert job["state"] == "cancelled" + assert job["cancel_requested"] is True + + +def test_stale_graph_worker_lease_recovers_rebuilding_state(): + service, _alpha, _beta, _gamma = _seed_service() + workspace_id = service.store.get_or_create_workspace("acme") + stale = time.time() - service_module.GRAPH_INDEX_LEASE_SECONDS - 1 + service.store.conn.execute( + "INSERT INTO jobs(id, workspace_id, kind, state, dry_run, total_items, " + "processed_items, counts, errors, request, cancel_requested, runner_id, " + "heartbeat_at, created_at) VALUES(?,?,?,?,?,?,?,?,?,?,?,?,?,?)", + ( + "job_stale", workspace_id, "graph_index", "running", 0, 2, 1, + '{"memories_scanned":1,"error_count":0}', "[]", "{}", 0, + "dev_gone", stale, stale, + ), + ) + service.store.conn.execute( + "UPDATE graph_index_state SET state='rebuilding', active_job_id='job_stale' " + "WHERE workspace_id=?", (workspace_id,), + ) + service.store.conn.commit() + + scene = service.graph_scene(workspace="acme") + job = service.graph_index_job("job_stale", workspace="acme") + + assert scene["meta"]["index_state"] == "ready" + assert job["state"] == "failed" + assert job["errors"][-1]["code"] == "worker_lease_expired" + assert service.graph_index_status(workspace="acme")["index"]["active_job_id"] is None + + +def test_cross_service_graph_job_start_reuses_one_database_job(tmp_path, monkeypatch): + database = tmp_path / "shared.db" + first = MemoryService.create(str(database), graph_extractor="none") + first.remember("Alpha uses Beta.", workspace="acme", scope="workspace") + second = MemoryService.create(str(database), graph_extractor="none") + release = threading.Event() + entered = threading.Event() + + def blocked_worker(_service, _job_id): + entered.set() + release.wait(5) + + monkeypatch.setattr(MemoryService, "_run_graph_index_job", blocked_worker) + barrier = threading.Barrier(3) + results = [] + + def launch(service): + barrier.wait() + results.append(service.start_graph_index_job( + workspace="acme", dry_run=True + )) + + callers = [threading.Thread(target=launch, args=(service,)) + for service in (first, second)] + for caller in callers: + caller.start() + barrier.wait() + for caller in callers: + caller.join(5) + + try: + assert entered.wait(1) + assert len(results) == 2 + assert len({result["id"] for result in results}) == 1 + assert sum(bool(result["reused"]) for result in results) == 1 + finally: + release.set() + for service in (first, second): + for worker in service._graph_job_threads.values(): + worker.join(5) + service.store.close() + + +def test_cross_service_graph_status_is_one_database_snapshot(tmp_path, monkeypatch): + database = tmp_path / "status-race.db" + reader = MemoryService.create(str(database), graph_extractor="none") + reader.remember("Alpha uses Beta.", workspace="acme", scope="workspace") + writer = MemoryService.create(str(database), graph_extractor="none") + index_read = threading.Event() + continue_status = threading.Event() + release_worker = threading.Event() + original_info = reader._graph_index_info + + def paused_info(workspace_id): + info = original_info(workspace_id) + index_read.set() + continue_status.wait(5) + return info + + def blocked_worker(_service, _job_id): + release_worker.wait(5) + + monkeypatch.setattr(reader, "_graph_index_info", paused_info) + monkeypatch.setattr(MemoryService, "_run_graph_index_job", blocked_worker) + result = {} + status_thread = threading.Thread( + target=lambda: result.setdefault( + "status", reader.graph_index_status(workspace="acme") + ) + ) + status_thread.start() + assert index_read.wait(2) + started = writer.start_graph_index_job(workspace="acme", dry_run=True) + continue_status.set() + status_thread.join(5) + + try: + assert result["status"]["index"]["state"] == "ready" + assert result["status"]["job"] is None + assert started["state"] in {"queued", "running"} + finally: + release_worker.set() + for worker in writer._graph_job_threads.values(): + worker.join(5) + reader.store.close() + writer.store.close() + + +def test_graph_job_memory_candidate_limit_fails_before_persisting(monkeypatch): + service, _alpha, _beta, _gamma = _seed_service() + monkeypatch.setattr(service_module, "MAX_GRAPH_INDEX_MEMORIES", 1) + + with pytest.raises(ValidationError, match="memory candidate limit"): + service.start_graph_index_job(workspace="acme", dry_run=True) + + assert service.store.conn.in_transaction is False + assert service.store.conn.execute( + "SELECT COUNT(*) AS n FROM jobs" + ).fetchone()["n"] == 0 + + +def test_active_graph_job_blocks_workspace_lifecycle_and_terminal_rows_are_deleted(): + service, _alpha, _beta, _gamma = _seed_service() + workspace_id = service.store.get_or_create_workspace("acme") + now = time.time() + service.store.conn.execute( + "INSERT INTO jobs(id, workspace_id, kind, state, dry_run, total_items, " + "processed_items, counts, errors, request, cancel_requested, runner_id, " + "heartbeat_at, created_at) VALUES(?,?,?,?,?,?,?,?,?,?,?,?,?,?)", + ( + "job_active", workspace_id, "graph_index", "queued", 1, 2, 0, + "{}", "[]", "{}", 0, "dev_live", now, now, + ), + ) + service.store.conn.commit() + + with pytest.raises(ValidationError, match="still active"): + service.delete_workspace("acme") + with pytest.raises(ValidationError, match="still active"): + service.copy_workspace("acme", "acme-copy") + + service.store.conn.execute( + "UPDATE jobs SET state='completed', finished_at=? WHERE id='job_active'", (now,) + ) + service.store.conn.commit() + assert service.delete_workspace("acme")["deleted"] is True + assert service.store.conn.execute( + "SELECT COUNT(*) AS n FROM jobs WHERE workspace_id=?", (workspace_id,) + ).fetchone()["n"] == 0 + assert service.store.conn.execute( + "SELECT COUNT(*) AS n FROM graph_index_state WHERE workspace_id=?", (workspace_id,) + ).fetchone()["n"] == 0 + + +def test_edge_support_delete_advances_graph_generation(): + service, _alpha, _beta, _gamma = _seed_service() + workspace_id = service.store.get_or_create_workspace("acme") + before = service._graph_index_info(workspace_id)["generation"] + + service.store.conn.execute("DELETE FROM edge_supports WHERE edge_id='edge_ab'") + service.store.conn.commit() + + assert service._graph_index_info(workspace_id)["generation"] > before + + +def test_explicit_graph_index_write_populates_evidence_and_advances_generation(): + service = MemoryService.create(":memory:", graph_extractor="none") + workspace_id = service.store.get_or_create_workspace("acme") + memory_id = service.store.add_memory(MemoryRecord( + id="", content="Alice works at Acme Corp.", workspace_id=workspace_id, + scope=Scope.WORKSPACE, + )) + initial = service.graph_index_status(workspace="acme")["index"]["generation"] + + started = service.start_graph_index_job(workspace="acme", dry_run=False) + deadline = time.time() + 5 + job = started + while job["state"] in {"queued", "running"} and time.time() < deadline: + time.sleep(0.01) + job = service.graph_index_job(started["id"], workspace="acme") + + status = service.graph_index_status(workspace="acme") + supports = service.store.conn.execute( + "SELECT memory_id, source_kind, confidence FROM edge_supports " + "WHERE memory_id=? ORDER BY confidence DESC", + (memory_id,), + ).fetchall() + assert job["state"] == "completed" + assert job["counts"]["entities_added"] >= 2 + assert job["counts"]["relations_added"] >= 1 + assert status["index"]["state"] == "ready" + assert status["index"]["active_job_id"] is None + assert status["index"]["generation"] > initial + assert supports + assert all(row["memory_id"] == memory_id for row in supports) + + +def test_graph_index_job_honors_persisted_cancellation(monkeypatch): + from engraphis.backends import graph_extractor as graph_extractor_module + from engraphis.backends.graph_extractor import GraphExtraction + + service = MemoryService.create(":memory:", graph_extractor="none") + workspace_id = service.store.get_or_create_workspace("acme") + for content in ("Alice knows Bob.", "Carol knows Dana."): + service.store.add_memory(MemoryRecord( + id="", content=content, workspace_id=workspace_id, scope=Scope.WORKSPACE, + )) + entered = threading.Event() + release = threading.Event() + + class SlowExtractor: + def extract(self, content, *, title=""): + entered.set() + release.wait(timeout=2) + return GraphExtraction() + + monkeypatch.setattr( + graph_extractor_module, "get_graph_extractor", lambda _kind: SlowExtractor() + ) + started = service.start_graph_index_job(workspace="acme", dry_run=True) + assert entered.wait(timeout=2) + cancelled = service.cancel_graph_index_job(started["id"], workspace="acme") + assert cancelled["cancel_requested"] is True + release.set() + deadline = time.time() + 5 + job = cancelled + while job["state"] in {"queued", "running"} and time.time() < deadline: + time.sleep(0.01) + job = service.graph_index_job(started["id"], workspace="acme") + + assert job["state"] == "cancelled" + assert job["processed_items"] == 1 + + +def test_graph_reads_return_explicit_rebuilding_conflict(): + service, _alpha, _beta, _gamma = _seed_service() + workspace_id = service.store.get_or_create_workspace("acme") + service.store.conn.execute( + "UPDATE graph_index_state SET state='rebuilding', active_job_id='job_test' " + "WHERE workspace_id=?", + (workspace_id,), + ) + service.store.conn.commit() + app = FastAPI() + app.include_router(v2_api.router) + v2_api.set_service(service) + client = TestClient(app) + + response = client.get("/api/graph/scene", params={"workspace": "acme"}) + + assert response.status_code == 409 + assert response.json()["detail"] == { + "error": "graph index rebuilding (job job_test)", + "index_state": "rebuilding", + "job_id": "job_test", + } + + +def test_cross_service_scene_never_returns_partial_rebuild(tmp_path, monkeypatch): + database = tmp_path / "scene-race.db" + writer = MemoryService.create(str(database), graph_extractor="none") + writer.remember("Alpha works at Acme.", workspace="acme", scope="workspace") + writer.remember("Beta works at Bravo.", workspace="acme", scope="workspace") + ordered = writer.store.conn.execute( + "SELECT content FROM memories ORDER BY id" + ).fetchall() + blocking_content = ordered[1]["content"] + reader = MemoryService.create(str(database), graph_extractor="none") + second_started = threading.Event() + release_second = threading.Event() + ready_check_passed = threading.Event() + + class BlockingExtractor: + def extract(self, content, *, title=""): + if content == blocking_content: + second_started.set() + release_second.wait(5) + prefix = "Alpha" if "Alpha" in content else "Beta" + company = "Acme" if "Acme" in content else "Bravo" + return GraphExtraction( + entities=[(prefix, "concept"), (company, "company")], + relations=[(prefix, "works at", company)], + ) + + monkeypatch.setattr( + graph_extractor_module, "get_graph_extractor", lambda _kind: BlockingExtractor() + ) + original_revision = reader._graph_scene_revision + + def pause_after_ready_check(): + ready_check_passed.set() + assert second_started.wait(5) + return original_revision() + + monkeypatch.setattr(reader, "_graph_scene_revision", pause_after_ready_check) + result = {} + + def read_scene(): + try: + result["scene"] = reader.graph_scene(workspace="acme") + except Exception as exc: # captured for the parent test thread + result["error"] = exc + + read_thread = threading.Thread(target=read_scene) + read_thread.start() + assert ready_check_passed.wait(2) + job = writer.start_graph_index_job(workspace="acme", dry_run=False) + assert second_started.wait(5) + read_thread.join(5) + + try: + assert "scene" not in result + assert isinstance(result.get("error"), GraphIndexRebuilding) + finally: + release_second.set() + deadline = time.time() + 5 + while job["state"] in {"queued", "running"} and time.time() < deadline: + time.sleep(0.01) + job = writer.graph_index_job(job["id"], workspace="acme") + read_thread.join(5) + reader.store.close() + writer.store.close() + + +def test_current_graph_scene_cache_expires_at_next_temporal_boundary(monkeypatch): + service, _alpha, _beta, _gamma = _seed_service() + now = time.time() + service.store.conn.execute( + "UPDATE edges SET valid_to=? WHERE id='edge_bg'", (now + 1.0,) + ) + service.store.conn.commit() + clock = {"now": now} + monkeypatch.setattr(service_module, "time", types.SimpleNamespace( + time=lambda: clock["now"], perf_counter=time.perf_counter, + )) + + first = service.graph_scene(workspace="acme") + warm = service.graph_scene(workspace="acme") + clock["now"] = now + 2.0 + expired = service.graph_scene(workspace="acme") + + assert first["meta"]["total_edges"] == 2 + assert warm["meta"]["cache_hit"] is True + assert expired["meta"]["cache_hit"] is False + assert expired["meta"]["total_edges"] == 1 + + +@pytest.mark.parametrize(("kwargs", "message"), [ + ({"level": "unknown"}, "level must be one of"), + ({"seeds": ["seed"] * 65}, "too many seeds"), + ({"min_confidence": float("nan")}, "min_confidence"), + ({"node_limit": 301}, "node_limit"), + ({"edge_limit": -1}, "edge_limit"), +]) +def test_graph_scene_direct_service_inputs_are_bounded(kwargs, message): + service, _alpha, _beta, _gamma = _seed_service() + + with pytest.raises(ValidationError, match=message): + service.graph_scene(workspace="acme", **kwargs) + + +def test_graph_lookup_direct_service_inputs_are_bounded(): + service, _alpha, _beta, _gamma = _seed_service() + + with pytest.raises(ValidationError, match="query exceeds"): + service.graph_suggest("q" * 1_001, workspace="acme") + with pytest.raises(ValidationError, match="canonical_id exceeds"): + service.graph_entity("e" * 201, workspace="acme") + with pytest.raises(ValidationError, match="max_visits"): + service.graph_path("a", "b", workspace="acme", max_visits=50_001) + + +def test_entity_evidence_rechecks_workspace_on_forged_memory_pointer(): + service, alpha, _beta, _gamma = _seed_service() + private_workspace = service.store.get_or_create_workspace("private") + secret = service.store.add_memory(MemoryRecord( + id="", content="cross-workspace secret", workspace_id=private_workspace, + scope=Scope.WORKSPACE, + )) + acme_workspace = service.store.get_or_create_workspace("acme") + decoy = service.store.upsert_entity(Node( + id="", name="Decoy", ntype="concept", workspace_id=acme_workspace, + )) + # Edge provenance is untrusted/syncable data. Even if it names a valid foreign + # memory id, the second-hop evidence lookup must remain inside the requested scope. + service.store.upsert_edge(Edge( + id="edge_forged", src=alpha, dst=decoy, relation="mentions", + workspace_id=acme_workspace, + provenance={"source": "manual", "memory_id": secret}, + )) + + detail = service.graph_entity(alpha, workspace="acme") + + assert secret not in {item["memory_id"] for item in detail["evidence"]} + assert all("cross-workspace secret" not in item["excerpt"] + for item in detail["evidence"]) + + +def test_entity_inspector_bounds_history_and_reports_complete_counts(monkeypatch): + service, alpha, _beta, gamma = _seed_service() + workspace_id = service.store.get_or_create_workspace("acme") + service.store.upsert_edge(Edge( + id="edge_old_one", src=alpha, dst=gamma, relation="preceded", + workspace_id=workspace_id, + )) + service.store.upsert_edge(Edge( + id="edge_old_two", src=gamma, dst=alpha, relation="replaced", + workspace_id=workspace_id, + )) + closed_at = time.time() + service.store.invalidate_edge("edge_old_one", at=closed_at) + service.store.invalidate_edge("edge_old_two", at=closed_at + 0.001) + monkeypatch.setattr(service_module, "GRAPH_ENTITY_HISTORY_LIMIT", 1) + + detail = service.graph_entity(alpha, workspace="acme") + + assert detail["totals"]["history"] == 2 + assert detail["truncation"]["history"] is True + assert len(detail["history"]) == 1 + assert detail["history"][0]["event"] == "Relation invalidated" + + +def test_graph_analysis_candidate_limit_fails_bounded(monkeypatch): + service, _alpha, _beta, _gamma = _seed_service() + monkeypatch.setattr(service_module, "MAX_GRAPH_ANALYSIS_ENTITIES", 2) + + with pytest.raises(ValidationError, match="entity candidate limit"): + service.graph_scene(workspace="acme") + + +def test_graph_route_bounds_csv_filter_cardinality(): + service, _alpha, _beta, _gamma = _seed_service() + app = FastAPI() + app.include_router(v2_api.router) + v2_api.set_service(service) + client = TestClient(app) + + response = client.get("/api/graph/scene", params={ + "workspace": "acme", + "entity_types": ",".join(f"type-{index}" for index in range(65)), + }) + + assert response.status_code == 422 + + +def test_workspace_copy_remaps_canonical_and_support_ids(): + service, alpha, _beta, _gamma = _seed_service() + copied = service.copy_workspace("acme", "acme-copy") + copied_workspace_id = copied["id"] + copied_entities = [dict(row) for row in service.store.conn.execute( + "SELECT id, canonical_id FROM entities WHERE workspace_id=? ORDER BY id", + (copied_workspace_id,), + ).fetchall()] + source_ids = {row["id"] for row in service.store.conn.execute( + "SELECT id FROM entities WHERE workspace_id<>(?)", (copied_workspace_id,) + ).fetchall()} + copied_edges = [dict(row) for row in service.store.conn.execute( + "SELECT id, provenance FROM edges WHERE workspace_id=? ORDER BY id", + (copied_workspace_id,), + ).fetchall()] + copied_supports = [dict(row) for row in service.store.conn.execute( + "SELECT s.edge_id, s.memory_id FROM edge_supports s " + "JOIN edges e ON e.id=s.edge_id WHERE e.workspace_id=? ORDER BY s.id", + (copied_workspace_id,), + ).fetchall()] + copied_memory_ids = {row["id"] for row in service.store.conn.execute( + "SELECT id FROM memories WHERE workspace_id=?", (copied_workspace_id,) + ).fetchall()} + + assert alpha in source_ids + assert all(row["canonical_id"] not in source_ids for row in copied_entities) + assert {row["edge_id"] for row in copied_supports} == { + row["id"] for row in copied_edges + } + assert {row["memory_id"] for row in copied_supports} <= copied_memory_ids + for edge in copied_edges: + provenance = json.loads(edge["provenance"]) + assert provenance["memory_id"] in copied_memory_ids diff --git a/tests/test_graph_extractor.py b/tests/test_graph_extractor.py index dd91787..1558fd6 100644 --- a/tests/test_graph_extractor.py +++ b/tests/test_graph_extractor.py @@ -10,6 +10,7 @@ from __future__ import annotations +from engraphis.backends import graph_extractor as graph_extractor_module from engraphis.backends.graph_extractor import ( GraphExtraction, NullGraphExtractor, RegexGraphExtractor, feed, get_graph_extractor, ) @@ -60,6 +61,22 @@ def test_factory_and_null_default(): assert NullGraphExtractor().extract("Alice Johnson works at Acme.").entities == [] +def test_regex_and_feed_bound_per_memory_entity_fanout(): + content = " ".join(f"person{index}@example.com" for index in range(5_000)) + extraction = RegexGraphExtractor().extract(content) + store = Store(":memory:") + + written = feed( + store, content, workspace_id="w1", extractor=RegexGraphExtractor() + ) + + assert len(extraction.entities) == graph_extractor_module._MAX_ENTITIES + assert written["entities"] == graph_extractor_module._MAX_ENTITIES + assert store.conn.execute( + "SELECT COUNT(*) AS n FROM entities WHERE workspace_id='w1'" + ).fetchone()["n"] == graph_extractor_module._MAX_ENTITIES + + # ── the feed() writer (scoped graph) ────────────────────────────────────────── def test_feed_writes_scoped_entities_and_edges(): diff --git a/tests/test_graph_scene_contract.py b/tests/test_graph_scene_contract.py new file mode 100644 index 0000000..d515c8f --- /dev/null +++ b/tests/test_graph_scene_contract.py @@ -0,0 +1,64 @@ +"""Shared contract checks for the dashboard and graph-scene service.""" +from __future__ import annotations + +import json +from pathlib import Path + + +FIXTURE = Path(__file__).with_name("graph_scene_fixture.json") + + +def _scene() -> dict: + return json.loads(FIXTURE.read_text(encoding="utf-8")) + + +def test_graph_scene_fixture_has_stable_public_shape(): + scene = _scene() + assert set(scene) == { + "meta", "nodes", "edges", "communities", "community_bridges", "facets" + } + assert { + "workspace", "level", "scene_hash", "index_generation", "total_nodes", + "total_edges", "shown_nodes", "shown_edges", "truncated", "query_ms", + "layout_seed", "index_state", "filters", + } <= set(scene["meta"]) + assert { + "id", "canonical_id", "label", "type", "member_ids", "repo_ids", + "mass_score", "gravity_mass", "visual_radius", "community_id", + "anchor_role", "scene_rank", + } <= set(scene["nodes"][0]) + assert { + "id", "source", "target", "relation", "layer", "directed", "confidence", + "support_count", "strength", "rest_length", "spring_strength", "tier", + "visible_by_default", "bundled_edge_count", + } <= set(scene["edges"][0]) + + +def test_graph_scene_fixture_encodes_galaxy_invariants(): + scene = _scene() + nodes = {node["id"]: node for node in scene["nodes"]} + black_holes = [node for node in scene["nodes"] if node["anchor_role"] == "global"] + assert len(black_holes) == 1 + assert black_holes[0]["mass_score"] == max(node["mass_score"] for node in scene["nodes"]) + assert (black_holes[0]["x"], black_holes[0]["y"]) == (0, 0) + + strengths_and_lengths = sorted( + (edge["strength"], edge["rest_length"]) for edge in scene["edges"] + ) + for (weaker, longer), (stronger, shorter) in zip( + strengths_and_lengths, strengths_and_lengths[1:] + ): + assert weaker <= stronger + assert longer >= shorter + + for edge in scene["edges"]: + cross_system = ( + nodes[edge["source"]]["community_id"] + != nodes[edge["target"]]["community_id"] + ) + if cross_system and scene["meta"]["level"] == "overview": + assert not edge["visible_by_default"] + assert edge["tier"] in {"context", "ambient"} + + assert scene["community_bridges"] + diff --git a/tests/test_http_security.py b/tests/test_http_security.py index 604c010..13e84fb 100644 --- a/tests/test_http_security.py +++ b/tests/test_http_security.py @@ -3,7 +3,7 @@ pytest.importorskip("fastapi") from fastapi import FastAPI -from fastapi.responses import JSONResponse +from fastapi.responses import HTMLResponse, JSONResponse from fastapi.testclient import TestClient from engraphis import http_security @@ -29,6 +29,12 @@ def root(): def custom(): return JSONResponse({"ok": True}, headers={"Referrer-Policy": "no-referrer"}) + @app.get("/dashboard", response_class=HTMLResponse) + def dashboard(): + # The real dashboard is an inline single-file app: inline