test: init e2e

.github/workflows/main-ci.yml

@@ -42,6 +42,7 @@ jobs:
     runs-on: ubuntu-latest
     outputs:
       api-changed: ${{ steps.changes.outputs.api }}
+      e2e-changed: ${{ steps.changes.outputs.e2e }}
       web-changed: ${{ steps.changes.outputs.web }}
       vdb-changed: ${{ steps.changes.outputs.vdb }}
       migration-changed: ${{ steps.changes.outputs.migration }}
@@ -59,6 +60,16 @@ jobs:
               - 'web/**'
               - '.github/workflows/web-tests.yml'
               - '.github/actions/setup-web/**'
+            e2e:
+              - 'api/**'
+              - 'api/pyproject.toml'
+              - 'api/uv.lock'
+              - 'e2e/**'
+              - 'web/**'
+              - 'docker/docker-compose.middleware.yaml'
+              - 'docker/middleware.env.example'
+              - '.github/workflows/web-e2e.yml'
+              - '.github/actions/setup-web/**'
             vdb:
               - 'api/core/rag/datasource/**'
               - 'docker/**'
@@ -190,6 +201,65 @@ jobs:
           echo "Web tests were not required, but the skip job finished with result: $SKIP_RESULT" >&2
           exit 1
 
+  web-e2e-run:
+    name: Run Web Full-Stack E2E
+    needs:
+      - pre_job
+      - check-changes
+    if: needs.pre_job.outputs.should_skip != 'true' && needs.check-changes.outputs.e2e-changed == 'true'
+    uses: ./.github/workflows/web-e2e.yml
+
+  web-e2e-skip:
+    name: Skip Web Full-Stack E2E
+    needs:
+      - pre_job
+      - check-changes
+    if: needs.pre_job.outputs.should_skip != 'true' && needs.check-changes.outputs.e2e-changed != 'true'
+    runs-on: ubuntu-latest
+    steps:
+      - name: Report skipped web full-stack e2e
+        run: echo "No E2E-related changes detected; skipping web full-stack E2E."
+
+  web-e2e:
+    name: Web Full-Stack E2E
+    if: ${{ always() }}
+    needs:
+      - pre_job
+      - check-changes
+      - web-e2e-run
+      - web-e2e-skip
+    runs-on: ubuntu-latest
+    steps:
+      - name: Finalize Web Full-Stack E2E status
+        env:
+          SHOULD_SKIP_WORKFLOW: ${{ needs.pre_job.outputs.should_skip }}
+          TESTS_CHANGED: ${{ needs.check-changes.outputs.e2e-changed }}
+          RUN_RESULT: ${{ needs.web-e2e-run.result }}
+          SKIP_RESULT: ${{ needs.web-e2e-skip.result }}
+        run: |
+          if [[ "$SHOULD_SKIP_WORKFLOW" == 'true' ]]; then
+            echo "Web full-stack E2E was skipped because this workflow run duplicated a successful or newer run."
+            exit 0
+          fi
+
+          if [[ "$TESTS_CHANGED" == 'true' ]]; then
+            if [[ "$RUN_RESULT" == 'success' ]]; then
+              echo "Web full-stack E2E ran successfully."
+              exit 0
+            fi
+
+            echo "Web full-stack E2E was required but finished with result: $RUN_RESULT" >&2
+            exit 1
+          fi
+
+          if [[ "$SKIP_RESULT" == 'success' ]]; then
+            echo "Web full-stack E2E was skipped because no E2E-related files changed."
+            exit 0
+          fi
+
+          echo "Web full-stack E2E was not required, but the skip job finished with result: $SKIP_RESULT" >&2
+          exit 1
+
   style-check:
     name: Style Check
     needs: pre_job

.github/workflows/web-e2e.yml

@@ -0,0 +1,93 @@
+name: Web Full-Stack E2E
+
+on:
+  workflow_call:
+
+permissions:
+  contents: read
+
+concurrency:
+  group: web-e2e-${{ github.head_ref || github.run_id }}
+  cancel-in-progress: true
+
+jobs:
+  test:
+    name: Web Full-Stack E2E
+    runs-on: ubuntu-latest
+    defaults:
+      run:
+        shell: bash
+
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
+        with:
+          persist-credentials: false
+
+      - name: Setup web dependencies
+        uses: ./.github/actions/setup-web
+
+      - name: Install E2E package dependencies
+        working-directory: ./e2e
+        run: pnpm install --frozen-lockfile
+
+      - name: Setup UV and Python
+        uses: astral-sh/setup-uv@37802adc94f370d6bfd71619e3f0bf239e1f3b78 # v7.6.0
+        with:
+          enable-cache: true
+          python-version: "3.12"
+          cache-dependency-glob: api/uv.lock
+
+      - name: Install API dependencies
+        run: uv sync --project api --dev
+
+      - name: Prepare built web app env
+        run: |
+          cp web/.env.example web/.env.local
+          sed -i 's|http://localhost:5001|http://127.0.0.1:5001|g' web/.env.local
+
+      - name: Prepare middleware stack env
+        run: |
+          cp docker/middleware.env.example docker/middleware.env
+          cat <<'EOF' >> docker/middleware.env
+          COMPOSE_PROFILES=postgresql,weaviate
+          EOF
+
+      - name: Start middleware stack
+        working-directory: ./e2e
+        run: pnpm e2e:middleware:up
+
+      - name: Install Playwright browser
+        working-directory: ./e2e
+        run: pnpm e2e:install
+
+      - name: Run source-api and built-web E2E tests
+        working-directory: ./e2e
+        env:
+          E2E_ADMIN_EMAIL: e2e-admin@example.com
+          E2E_ADMIN_NAME: E2E Admin
+          E2E_ADMIN_PASSWORD: E2eAdmin12345
+          E2E_FORCE_WEB_BUILD: "1"
+          E2E_INIT_PASSWORD: E2eInit12345
+        run: pnpm e2e
+
+      - name: Upload Playwright report
+        if: ${{ !cancelled() }}
+        uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0
+        with:
+          name: playwright-report
+          path: e2e/playwright-report
+          retention-days: 7
+
+      - name: Upload Playwright test results
+        if: ${{ !cancelled() }}
+        uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0
+        with:
+          name: playwright-test-results
+          path: e2e/test-results
+          retention-days: 7
+
+      - name: Stop middleware stack
+        if: ${{ always() }}
+        working-directory: ./e2e
+        run: pnpm e2e:middleware:down

docker/docker-compose.middleware.yaml

@@ -127,6 +127,8 @@ services:
     restart: always
     env_file:
       - ./middleware.env
+    extra_hosts:
+      - "host.docker.internal:host-gateway"
     environment:
       # Use the shared environment variables.
       LOG_OUTPUT_FORMAT: ${LOG_OUTPUT_FORMAT:-text}

e2e/.gitignore

@@ -0,0 +1,6 @@
+node_modules/
+.auth/
+playwright-report/
+test-results/
+cucumber-report/
+.logs/

e2e/AGENTS.md

@@ -0,0 +1,177 @@
+# E2E
+
+This package contains the repository-level end-to-end tests for Dify.
+
+This file is the canonical package guide for `e2e/`. Keep detailed workflow, architecture, debugging, and reporting documentation here. Keep `README.md` as a minimal pointer to this file so the two documents do not drift.
+
+The suite uses Cucumber for scenario definitions and Playwright as the browser execution layer.
+
+It tests:
+
+- backend API started from source
+- frontend served from the production artifact
+- middleware services started from Docker
+
+## Prerequisites
+
+- Node.js `^22.22.1`
+- `pnpm`
+- `uv`
+- Docker
+
+Install Playwright browsers once:
+
+```bash
+cd e2e
+pnpm install
+pnpm e2e:install
+pnpm check
+```
+
+Use `pnpm check` as the default local verification step after editing E2E TypeScript, Cucumber support code, or feature glue. It runs formatting, linting, and type checks for this package.
+
+Common commands:
+
+```bash
+# authenticated-only regression (default excludes @fresh)
+pnpm e2e
+
+# full reset + fresh install + authenticated scenarios
+pnpm e2e:full
+
+# run a tagged subset
+pnpm e2e -- --tags @smoke
+
+# headed browser
+pnpm e2e:headed -- --tags @smoke
+
+# slow down browser actions for local debugging
+E2E_SLOW_MO=500 pnpm e2e:headed -- --tags @smoke
+```
+
+Frontend artifact behavior:
+
+- if `web/.next/BUILD_ID` exists, E2E reuses the existing build by default
+- if you set `E2E_FORCE_WEB_BUILD=1`, E2E rebuilds the frontend before starting it
+
+## Lifecycle
+
+```mermaid
+flowchart TD
+  A["Start E2E run"] --> B["run-cucumber.sh orchestrates reset/middleware/API"]
+  B --> C["typed web-server manager starts or reuses frontend"]
+  C --> D["Cucumber loads config, steps, and support modules"]
+  D --> E["BeforeAll bootstraps shared auth state via /install"]
+  E --> F{"Which command is running?"}
+  F -->|`pnpm e2e`| G["Run config default tags: not @fresh and not @skip"]
+  F -->|`pnpm e2e:full*`| H["Override tags to not @skip"]
+  G --> I["Per-scenario BrowserContext from shared browser"]
+  H --> I
+  I --> J["Failure artifacts written to cucumber-report/artifacts"]
+```
+
+Ownership is split like this:
+
+- shell scripts orchestrate reset, middleware, and backend startup
+- `support/web-server.ts` manages frontend reuse, startup, readiness, and shutdown
+- `features/support/hooks.ts` manages auth bootstrap, scenario lifecycle, and diagnostics
+- `features/support/world.ts` owns per-scenario typed context
+- `features/step-definitions/` holds domain-oriented glue so the official VS Code Cucumber plugin works with default conventions when `e2e/` is opened as the workspace root
+
+Package layout:
+
+- `features/`: Gherkin scenarios grouped by capability
+- `features/step-definitions/`: domain-oriented step definitions
+- `features/support/hooks.ts`: suite lifecycle, auth-state bootstrap, diagnostics
+- `features/support/world.ts`: shared scenario context
+- `support/web-server.ts`: typed frontend startup/reuse logic
+- `scripts/*.sh`: middleware and backend orchestration
+
+Behavior depends on instance state:
+
+- uninitialized instance: completes install and stores authenticated state
+- initialized instance: signs in and reuses authenticated state
+
+Because of that, the `@fresh` install scenario only runs in the `pnpm e2e:full*` flows. The default `pnpm e2e*` flows exclude `@fresh` via Cucumber config tags so they can be re-run against an already initialized instance.
+
+Reset all persisted E2E state:
+
+```bash
+pnpm e2e:reset
+```
+
+This removes:
+
+- `docker/volumes/db/data`
+- `docker/volumes/redis/data`
+- `docker/volumes/weaviate`
+- `docker/volumes/plugin_daemon`
+- `e2e/.auth`
+- `e2e/.logs`
+- `e2e/cucumber-report`
+
+Start the full middleware stack:
+
+```bash
+pnpm e2e:middleware:up
+```
+
+Stop the full middleware stack:
+
+```bash
+pnpm e2e:middleware:down
+```
+
+The middleware stack includes:
+
+- PostgreSQL
+- Redis
+- Weaviate
+- Sandbox
+- SSRF proxy
+- Plugin daemon
+
+Fresh install verification:
+
+```bash
+pnpm e2e:full
+```
+
+Repeat authenticated regression without clearing data:
+
+```bash
+pnpm e2e:middleware:up
+pnpm e2e
+pnpm e2e:middleware:down
+```
+
+Run the Cucumber suite against an already running middleware stack:
+
+```bash
+pnpm e2e:middleware:up
+pnpm e2e
+pnpm e2e:middleware:down
+```
+
+Interactive local debugging:
+
+```bash
+pnpm e2e:full:ui
+```
+
+Artifacts and diagnostics:
+
+- `cucumber-report/report.html`: HTML report
+- `cucumber-report/report.json`: JSON report
+- `cucumber-report/artifacts/`: failure screenshots and HTML captures
+- `.logs/cucumber-api.log`: backend startup log
+- `.logs/cucumber-web.log`: frontend startup log
+- `.logs/web-server-manager.log`: typed web-server manager log
+
+Open the HTML report locally with:
+
+```bash
+open cucumber-report/report.html
+```
+
+`pnpm e2e:ui` and `pnpm e2e:full:ui` currently run the suite in headed mode. Cucumber does not provide a Playwright-style UI mode.

e2e/README.md

@@ -0,0 +1,3 @@
+# E2E
+
+Canonical documentation for this package lives in [AGENTS.md](./AGENTS.md).