feat(evals): M1 CI skeleton — run MCP evals end to end in GitHub Actions

.changeset/mcp-allowed-hosts.md

@@ -0,0 +1,5 @@
+---
+"@hyperdx/api": patch
+---
+
+Add `HYPERDX_MCP_ALLOWED_HOSTS` to allow the MCP HTTP endpoint to accept additional `Host` header values beyond the SDK's localhost defaults. The MCP transport enables DNS-rebinding protection by default and rejects any non-localhost `Host` with "Invalid Host", which prevents reaching the MCP over a service DNS name (e.g. when a separate container connects to `http://hyperdx:8000/mcp` over a Docker network). Set this env var (comma/space separated) to the hostname(s) that should be accepted; the localhost defaults remain allowed. Unset by default, so behavior is unchanged.

.github/workflows/evals.yml

@@ -0,0 +1,185 @@
+name: MCP Evals
+
+# HDX-4756 — Run the MCP-server AI evals end to end in CI.
+#
+# Milestone 1 (CI skeleton): boot the HyperDX all-in-one image + the pre-baked
+# eval runner container, seed a LOW-volume dataset, and run all five stages
+# (Setup → Seed → Run → Grade → Report) to completion, then post the verdict as
+# a PR comment. This job is ADVISORY — it never gates merges (see the final
+# step which always exits 0). Speed is not a goal for M1.
+
+on:
+  pull_request:
+    branches: [main]
+  workflow_dispatch:
+    inputs:
+      scenario:
+        description: 'Scenario to run'
+        type: string
+        default: latency-spike
+      volume_factor:
+        description: 'Seed volume factor (fraction of full volume)'
+        type: string
+        default: '0.01'
+      runs:
+        description: 'Runs per (scenario, MCP) cell'
+        type: string
+        default: '1'
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  evals:
+    name: Run MCP evals (advisory)
+    runs-on: ubuntu-24.04
+    # Generous budget — M1 is slow by design (heavy image build + agent run).
+    timeout-minutes: 90
+    permissions:
+      contents: read
+      pull-requests: write
+    env:
+      # M1 defaults; workflow_dispatch inputs override on manual runs.
+      HDX_EVAL_SCENARIO: ${{ github.event.inputs.scenario || 'latency-spike' }}
+      HDX_EVAL_VOLUME_FACTOR: ${{ github.event.inputs.volume_factor || '0.01' }}
+      HDX_EVAL_RUNS: ${{ github.event.inputs.runs || '1' }}
+      HDX_EVAL_OUTPUT_DIR: ${{ github.workspace }}/eval-output
+      HDX_EVALS_OUTPUT_DIR: ${{ github.workspace }}/eval-output
+      HDX_EVALS_RUNS_DIR: ${{ github.workspace }}/eval-output/runs
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v6
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+        with:
+          driver: docker-container
+
+      - name: Expose GitHub Actions cache to BuildKit
+        uses: crazy-max/ghaction-github-runtime@v4
+
+      - name: Prepare output dir
+        # World-writable so the runner container's non-root `node` user (uid
+        # 1000) can write runs/ + verdict.md into these bind mounts regardless
+        # of the host runner's uid.
+        run: |
+          mkdir -p "${HDX_EVAL_OUTPUT_DIR}/runs"
+          chmod -R 777 "${HDX_EVAL_OUTPUT_DIR}"
+
+      # Build both DISTINCT images with GHA layer caching. The HyperDX
+      # all-in-one build is heavy (OTel collector + Next.js); caching keeps
+      # re-runs reasonable. Slow first build is acceptable for M1.
+      - name: Build HyperDX image
+        run: |
+          docker buildx bake \
+            -f docker-compose.evals.yml \
+            --set hyperdx.cache-to=type=gha,scope=evals-hyperdx,mode=max \
+            --set hyperdx.cache-from=type=gha,scope=evals-hyperdx \
+            --load \
+            hyperdx
+
+      - name: Build eval runner image
+        run: |
+          docker buildx bake \
+            -f docker-compose.evals.yml \
+            --set eval-runner.cache-to=type=gha,scope=evals-runner,mode=max \
+            --set eval-runner.cache-from=type=gha,scope=evals-runner \
+            --load \
+            eval-runner
+
+      - name: Boot HyperDX instance
+        run: |
+          docker compose -f docker-compose.evals.yml up -d --no-build hyperdx
+
+      - name: Wait for HyperDX to be healthy
+        run: |
+          echo "Waiting for the HyperDX container healthcheck to pass..."
+          for i in $(seq 1 60); do
+            status=$(docker inspect -f '{{.State.Health.Status}}' \
+              "$(docker compose -f docker-compose.evals.yml ps -q hyperdx)" 2>/dev/null || echo "starting")
+            echo "  [$i] health=$status"
+            if [ "$status" = "healthy" ]; then
+              echo "HyperDX is healthy."
+              exit 0
+            fi
+            sleep 5
+          done
+          echo "::error::HyperDX did not become healthy in time"
+          docker compose -f docker-compose.evals.yml logs hyperdx | tail -n 200
+          exit 1
+
+      - name: Run evals (Setup → Seed → Run → Grade → Report)
+        env:
+          ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
+          HDX_EVAL_RUN_URL:
+            ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{
+            github.run_id }}
+          HDX_EVAL_COMMIT_SHA:
+            ${{ github.event.pull_request.head.sha || github.sha }}
+        run: |
+          docker compose -f docker-compose.evals.yml run --rm \
+            -e ANTHROPIC_API_KEY \
+            -e HDX_EVAL_RUN_URL \
+            -e HDX_EVAL_COMMIT_SHA \
+            eval-runner
+
+      - name: Dump HyperDX logs on failure
+        if: failure()
+        run:
+          docker compose -f docker-compose.evals.yml logs hyperdx | tail -n 300
+
+      - name: Tear down
+        if: always()
+        run: docker compose -f docker-compose.evals.yml down -v || true
+
+      - name: Upload eval artifacts
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: mcp-eval-results
+          path: ${{ env.HDX_EVAL_OUTPUT_DIR }}
+          if-no-files-found: warn
+          retention-days: 14
+
+      - name: Read verdict comment
+        id: verdict
+        if: always() && github.event_name == 'pull_request'
+        run: |
+          COMMENT_FILE="${HDX_EVAL_OUTPUT_DIR}/verdict.md"
+          if [ -f "$COMMENT_FILE" ]; then
+            {
+              echo 'body<<HDX_EOF'
+              cat "$COMMENT_FILE"
+              echo 'HDX_EOF'
+            } >> "$GITHUB_OUTPUT"
+          else
+            {
+              echo 'body<<HDX_EOF'
+              echo '<!-- hdx-eval-verdict -->'
+              echo '## MCP Eval Results — ❌ FAIL'
+              echo ''
+              echo '_Advisory only (Milestone 1 CI skeleton) — this check does not block merges._'
+              echo ''
+              echo 'The eval pipeline did not produce a verdict (no `verdict.md`). Check the workflow logs.'
+              echo 'HDX_EOF'
+            } >> "$GITHUB_OUTPUT"
+          fi
+
+      # Sticky comment: `message-id` makes re-runs UPDATE the existing comment
+      # instead of posting duplicates (same pattern as the E2E results comment).
+      - name: Post verdict as PR comment
+        if: >-
+          always() && github.event_name == 'pull_request' &&
+          github.event.pull_request.head.repo.fork != true
+        uses: mshick/add-pr-comment@v3
+        with:
+          message: ${{ steps.verdict.outputs.body }}
+          message-id: mcp-eval-verdict
+
+      # Advisory: this workflow never fails the build on the eval verdict. Only
+      # genuine infra breakage (image build, boot, or the runner erroring) fails
+      # via the earlier steps' own non-zero exits.
+      - name: Advisory note
+        if: always()
+        run: echo "MCP evals are advisory for M1 — not gating merges."

docker-compose.evals.yml

@@ -0,0 +1,120 @@
+# HDX-4754 / HDX-4756 — Eval pipeline stack for CI (and local reproduction).
+#
+# Two DISTINCT images on one shared Docker network:
+#   - `hyperdx`      : the all-in-one HyperDX image (ClickHouse + Mongo + OTel +
+#                      API + App in one container) — the isolated feature-branch
+#                      instance under test. Built from docker/hyperdx/Dockerfile,
+#                      target all-in-one-auth (auth is required so the eval
+#                      account/accessKey bootstrap works for the MCP).
+#   - `eval-runner`  : the pre-baked runner (docker/hdx-eval-runner/Dockerfile)
+#                      that drives Setup → Seed → Run → Grade → Report.
+#
+# Networking reconciliation (the subtle bit):
+#   - The runner reaches ClickHouse + API over the shared network via the
+#     `hyperdx` service DNS name: http://hyperdx:8123 and http://hyperdx:8000.
+#   - The HyperDX API queries its OWN bundled ClickHouse at http://localhost:8123
+#     (same container). So the Connection stored during setup uses that self-view
+#     — passed through HDX_EVAL_CONNECTION_CH_URL.
+#
+# Usage (CI drives this via .github/workflows/evals.yml):
+#   docker compose -f docker-compose.evals.yml build
+#   docker compose -f docker-compose.evals.yml up -d hyperdx
+#   # wait for hyperdx healthcheck ...
+#   docker compose -f docker-compose.evals.yml run --rm eval-runner
+
+name: hdx-evals
+
+services:
+  hyperdx:
+    build:
+      context: .
+      dockerfile: docker/hyperdx/Dockerfile
+      target: all-in-one-auth
+      additional_contexts:
+        clickhouse: ./docker/clickhouse
+        otel-collector: ./docker/otel-collector
+        hyperdx: ./docker/hyperdx
+        api: ./packages/api
+        app: ./packages/app
+      args:
+        OTEL_COLLECTOR_VERSION: ${OTEL_COLLECTOR_VERSION:-0.155.0}
+        OTEL_COLLECTOR_CORE_VERSION: ${OTEL_COLLECTOR_CORE_VERSION:-1.61.0}
+        CODE_VERSION: ${CODE_VERSION:-evals-ci}
+    image: hdx-evals-hyperdx:latest
+    environment:
+      HYPERDX_API_PORT: '8000'
+      HYPERDX_APP_PORT: '8080'
+      # Fresh, isolated instance each run — no external state.
+      HYPERDX_LOG_LEVEL: ${HYPERDX_LOG_LEVEL:-error}
+      # Let the MCP HTTP endpoint accept the `hyperdx` service DNS name in the
+      # Host header — the eval-runner container reaches it at http://hyperdx:8000
+      # over the shared network, and the MCP SDK's default DNS-rebinding
+      # protection would otherwise reject any non-localhost Host.
+      HYPERDX_MCP_ALLOWED_HOSTS: hyperdx
+    volumes:
+      # Re-open the ClickHouse `default` user to the internal Docker network so
+      # the separate eval-runner container can seed CH directly. The image's own
+      # users.d/default-user.xml clamps `default` to localhost; this override
+      # sorts after it (z-) and wins. CI-only; see the file header.
+      - ./docker/hdx-eval-runner/clickhouse-evals-user.xml:/etc/clickhouse-server/users.d/z-evals-user.xml:ro
+    # Expose to the host too, so the pipeline can also run with the runner as
+    # the GitHub job itself if ever desired. Not required for the compose-run
+    # path (service DNS is used there).
+    ports:
+      - '${HDX_EVALS_API_PORT:-18000}:8000'
+      - '${HDX_EVALS_APP_PORT:-18080}:8080'
+      - '${HDX_EVALS_CH_PORT:-18123}:8123'
+    healthcheck:
+      # The all-in-one auth image serves the app health endpoint on 8080 and the
+      # API on 8000. Gate on the API since that is what the eval harness talks to.
+      test:
+        - CMD-SHELL
+        - >-
+          curl -fsS http://localhost:8000/health >/dev/null && curl -fsS
+          http://localhost:8123/ping >/dev/null || exit 1
+      interval: 5s
+      timeout: 5s
+      retries: 60
+      start_period: 90s
+    networks:
+      - evals
+
+  eval-runner:
+    build:
+      context: .
+      dockerfile: docker/hdx-eval-runner/Dockerfile
+      args:
+        NODE_VERSION: ${NODE_VERSION:-22.23.1}
+    image: hdx-evals-runner:latest
+    depends_on:
+      hyperdx:
+        condition: service_healthy
+    environment:
+      ANTHROPIC_API_KEY: ${ANTHROPIC_API_KEY:-}
+      AI_API_KEY: ${AI_API_KEY:-}
+      # Runner's view of the HyperDX instance over the shared network.
+      HDX_EVAL_API_URL: http://hyperdx:8000
+      HDX_EVAL_CH_URL: http://hyperdx:8123
+      # The API's self-view of its bundled ClickHouse (all-in-one container).
+      HDX_EVAL_CONNECTION_CH_URL: http://localhost:8123
+      # M1 knobs — low volume so the skeleton finishes in minutes.
+      HDX_EVAL_SCENARIO: ${HDX_EVAL_SCENARIO:-latency-spike}
+      HDX_EVAL_VOLUME_FACTOR: ${HDX_EVAL_VOLUME_FACTOR:-0.01}
+      HDX_EVAL_RUNS: ${HDX_EVAL_RUNS:-1}
+      HDX_EVAL_MAX_TURNS: ${HDX_EVAL_MAX_TURNS:-15}
+      HDX_EVAL_TIMEOUT_MS: ${HDX_EVAL_TIMEOUT_MS:-600000}
+      HDX_EVAL_MCP: ${HDX_EVAL_MCP:-hyperdx}
+      HDX_EVAL_OUT_COMMENT: /work/eval-output/verdict.md
+      HDX_EVAL_RUN_URL: ${HDX_EVAL_RUN_URL:-}
+      HDX_EVAL_COMMIT_SHA: ${HDX_EVAL_COMMIT_SHA:-}
+    volumes:
+      # Persist runs + rendered verdict to the host so CI can upload artifacts
+      # and post the PR comment.
+      - ${HDX_EVALS_OUTPUT_DIR:-./eval-output}:/work/eval-output
+      - ${HDX_EVALS_RUNS_DIR:-./eval-output/runs}:/work/packages/hdx-eval/runs
+    networks:
+      - evals
+
+networks:
+  evals:
+    driver: bridge