live-e2e.yml

name: Live E2E

# The real-model browser suite of #204: four task checks driven through the FlyBot web
# UI on a fresh isolated bench, against the real DeepSeek endpoint. Separate from the
# compatibility matrix (nightly.yml) and from every PR check; never added to `tests-ok`
# or branch protection. This is the repository's only workflow that reads a secret: the
# live job runs on main only, the key is mapped step-scoped, and everything uploaded has
# passed the runner's evidence sanitizer (no `token=`, no key material, no cookies) and
# is an explicit allow-list — never a raw bench home.
#
# Without the repository secret DEEPSEEK_API_KEY the suite cannot run at all, and a run
# that is red every night for a reason nobody can act on teaches the team to ignore red
# (#794). So a missing key warns, skips the six steps that need it, and leaves the run
# green while the step summary says plainly that nothing ran and nothing was measured. A
# real failure of the suite still fails the run: the skip is never a pass.

on:
  schedule:
    - cron: "30 16 * * *" # 00:30 Asia/Shanghai daily; clear of compat's 16:00Z
  workflow_dispatch:

concurrency:
  group: live-e2e
  cancel-in-progress: false

permissions:
  contents: read

jobs:
  window:
    name: window check
    runs-on: ubuntu-latest
    timeout-minutes: 5
    outputs:
      run: ${{ steps.gate.outputs.run }}
    steps:
      - name: Check out main with full history
        if: github.event_name == 'schedule'
        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1  # v7.0.1
        with:
          fetch-depth: 0
          persist-credentials: false
      - name: Decide whether main has new commits
        id: gate
        run: |
          run=false
          reason=""
          if [ "${{ github.event_name }}" = "workflow_dispatch" ]; then
            run=true
            reason="manual dispatch always runs"
          elif [ -n "$(git log --since='24 hours ago' --pretty=oneline)" ]; then
            run=true
            reason="main had commits in the last 24 hours"
          else
            reason="main had no commits in the window; no tokens spent"
          fi
          echo "run=$run" >> "$GITHUB_OUTPUT"
          {
            echo "### Live E2E run decision"
            echo ""
            echo "- event: \`${{ github.event_name }}\`"
            echo "- run: \`${run}\`: ${reason}"
          } >> "$GITHUB_STEP_SUMMARY"

  live:
    name: live · ubuntu · Node 24 · real model
    needs: window
    if: needs.window.outputs.run == 'true' && github.ref == 'refs/heads/main'
    runs-on: ubuntu-latest
    timeout-minutes: 20
    env:
      WEB_PORT: '3196'
      TAP_PORT: '8898'
      BENCH: live-e2e
      MODEL: deepseek-official/deepseek-flash
    steps:
      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1  # v7.0.1
        with:
          persist-credentials: false
      - uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1  # v7.1.0
        with:
          node-version: 24
          cache: npm
          cache-dependency-path: core/package-lock.json
      # The key decides whether anything can run, so it is checked once and published as an
      # output that every step below gates on. The decision itself lives in a script with tests
      # (.github/scripts/live-e2e-preflight.test.mjs): a missing key warns and skips, never fails.
      - name: Preflight (a missing key skips every step, and says so)
        id: preflight
        env:
          DEEPSEEK_API_KEY: ${{ secrets.DEEPSEEK_API_KEY }}
        run: node .github/scripts/live-e2e-preflight.mjs
      - name: npm ci (core)
        if: steps.preflight.outputs.key == 'true'
        working-directory: core
        run: npm ci --no-fund --no-audit
        timeout-minutes: 5
      - name: Install the pinned DSH runtime
        if: steps.preflight.outputs.key == 'true'
        working-directory: core
        run: |
          DSH=$(node -p "require('./dev/harness/package.json').dependencies['@deepseek-ai/dsh']")
          npm run harness -- --no-save "@deepseek-ai/dsh@$DSH"
          echo "DSH=$DSH" >> "$GITHUB_ENV"
        timeout-minutes: 5
      - name: Clone DeepSeek Harness at the pinned tag
        if: steps.preflight.outputs.key == 'true'
        run: |
          git clone --depth 1 --branch "dsh-v$DSH" https://github.com/deepseek-ai/deepseek-harness.git deepseek-harness
          corepack enable
          pnpm install --dir deepseek-harness
        timeout-minutes: 10
      - name: Start the usage tap
        if: steps.preflight.outputs.key == 'true'
        run: |
          # The bench home lives outside the checkout (#398): a workspace inside the repository
          # would hand the grader to the Bot under test. bench.mjs and run.mjs read the same
          # variable, and the default is the same on both sides.
          LIVE_E2E_HOME="$HOME/.cache/flybot-live-e2e/$BENCH"
          echo "LIVE_E2E_HOME=$LIVE_E2E_HOME" >> "$GITHUB_ENV"
          mkdir -p "$LIVE_E2E_HOME"
          setsid node core/dev/llm-tap.mjs "$TAP_PORT" https://api.deepseek.com "$LIVE_E2E_HOME/tap.jsonl" \
            </dev/null >"$LIVE_E2E_HOME/tap.out" 2>&1 & echo $! >"$LIVE_E2E_HOME/tap.pid"
          i=0
          until (exec 3<>"/dev/tcp/127.0.0.1/$TAP_PORT") 2>/dev/null; do
            i=$((i + 1))
            [ $i -ge 30 ] && { echo "::error::the usage tap did not start"; cat "$LIVE_E2E_HOME/tap.out"; exit 1; }
            sleep 1
          done
      - name: Start the real-model bench
        if: steps.preflight.outputs.key == 'true'
        env:
          DEEPSEEK_API_KEY: ${{ secrets.DEEPSEEK_API_KEY }}
        run: |
          DEEPSEEK_BASE_URL="http://127.0.0.1:$TAP_PORT/anthropic" node dev/live-e2e/bench.mjs "$BENCH" "$WEB_PORT" reset
        timeout-minutes: 8
      - name: Run the four checks
        if: steps.preflight.outputs.key == 'true'
        id: run
        env:
          DEEPSEEK_API_KEY: ${{ secrets.DEEPSEEK_API_KEY }}
          DEEPSEEK_BASE_URL: http://127.0.0.1:8898/anthropic
          LIVE_E2E_TAP_UPSTREAM: https://api.deepseek.com
        run: |
          node dev/live-e2e/run.mjs "$LIVE_E2E_HOME" "$WEB_PORT" \
            --model "$MODEL" --harness deepseek-harness \
            --tap "$LIVE_E2E_HOME/tap.jsonl" --evidence evidence --tz Asia/Shanghai
      - name: Stop the bench and the tap
        if: always() && steps.preflight.outputs.key == 'true'
        run: |
          node dev/live-e2e/bench.mjs "$BENCH" "$WEB_PORT" stop || true
          if [ -f "$LIVE_E2E_HOME/tap.pid" ]; then
            kill -TERM "-$(cat "$LIVE_E2E_HOME/tap.pid")" 2>/dev/null || true
          fi
      - name: Sanitized web log (diagnostics only, never uploaded raw)
        if: failure()
        run: tail -80 "$LIVE_E2E_HOME/web.log" | grep -v 'token=' || true
      # Fail closed twice: the runner writes evidence/sanitized.txt only after a clean
      # scan, and the upload requires that stamp. A bundle the sanitizer flagged (or one
      # from a run that never reached the scan) never becomes a public artifact.
      - name: Evidence gate
        id: evidence-gate
        if: always() && steps.run.outcome != 'skipped'
        run: |
          if [ ! -f evidence/sanitized.txt ]; then
            echo "::error::no sanitizer-cleared evidence bundle (the run failed before the scan, or the scan found something); nothing is uploaded"
            exit 1
          fi
      # Only the stamped allow-list is uploaded: the report, the screenshots, the
      # transcript — never the bench home, the tap log or any raw log.
      - name: Upload evidence (allow-list, sanitizer-stamped)
        if: always() && steps.evidence-gate.outcome == 'success'
        uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9  # v7.0.2
        with:
          name: live-e2e-evidence
          path: evidence/
          if-no-files-found: ignore
          retention-days: 14