ci: benchmark built suites with unsharded parallel Endform

.github/workflows/ci.yml

@@ -4,6 +4,7 @@ on:
   push:
     branches: [main]
   pull_request:
+  workflow_dispatch:
   merge_group:
 
 permissions:
@@ -225,6 +226,7 @@ jobs:
         test-backend,
         test-browser,
         test-e2e,
+        test-e2e-endform,
         test-e2e-table,
       ]
     runs-on: ubuntu-latest
@@ -246,6 +248,7 @@ jobs:
           BROWSER_RESULT: ${{ needs.test-browser.result }}
           E2E_SELECTED: ${{ needs.change-plan.outputs.e2e_node == 'true' || needs.change-plan.outputs.e2e_cloudflare == 'true' }}
           E2E_RESULT: ${{ needs.test-e2e.result }}
+          ENDFORM_RESULT: ${{ needs.test-e2e-endform.result }}
           TABLE_SELECTED: ${{ needs.change-plan.outputs.e2e_table }}
           TABLE_RESULT: ${{ needs.test-e2e-table.result }}
         run: |
@@ -273,6 +276,7 @@ jobs:
           check_lane "Backend tests" "$BACKEND_SELECTED" "$BACKEND_RESULT"
           check_lane "Browser and playground tests" "$BROWSER_SELECTED" "$BROWSER_RESULT"
           check_lane "Node/Cloudflare E2E" "$E2E_SELECTED" "$E2E_RESULT"
+          check_lane "Endform E2E tests" "$E2E_SELECTED" "$ENDFORM_RESULT"
           check_lane "Table E2E" "$TABLE_SELECTED" "$TABLE_RESULT"
           exit "$failed"
 
@@ -412,13 +416,15 @@ jobs:
       - run: echo "E2E results are aggregated by the Tests job"
 
   test-e2e:
-    name: E2E shard (${{ matrix.shardIndex }}/${{ matrix.shardTotal }})
+    name: E2E built native (${{ matrix.shardIndex }}/${{ matrix.shardTotal }})
     needs: change-plan
     if: needs.change-plan.outputs.e2e_node == 'true' || needs.change-plan.outputs.e2e_cloudflare == 'true'
     runs-on: ubuntu-latest
-    timeout-minutes: 40
+    timeout-minutes: 90
+    permissions:
+      contents: read
     strategy:
-      fail-fast: true
+      fail-fast: false
       matrix:
         shardIndex: [1, 2, 3, 4, 5, 6, 7, 8]
         shardTotal: [8]
@@ -451,25 +457,86 @@ jobs:
       - name: Run Node E2E
         id: node-e2e
         if: ${{ !cancelled() && steps.node-build.outcome == 'success' }}
-        run: pnpm exec playwright test --shard=${{ matrix.shardIndex }}/${{ matrix.shardTotal }}
+        run: ENDFORM=false pnpm exec playwright test --shard=${{ matrix.shardIndex }}/${{ matrix.shardTotal }}
       - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
         if: failure() && steps.node-e2e.outcome == 'failure'
         with:
-          name: playwright-report-node-${{ matrix.shardIndex }}
+          name: playwright-report-node-native-${{ matrix.shardIndex }}
           path: |
             playwright-report/
             test-results/
           retention-days: 7
       - name: Run Cloudflare E2E
         id: cloudflare-e2e
         if: ${{ !cancelled() && steps.cloudflare-build.outcome == 'success' }}
-        run: pnpm exec playwright test --shard=${{ matrix.shardIndex }}/${{ matrix.shardTotal }}
+        run: ENDFORM=false pnpm exec playwright test --shard=${{ matrix.shardIndex }}/${{ matrix.shardTotal }}
+        env:
+          EMDASH_E2E_TARGET: cloudflare
+      - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
+        if: failure() && steps.cloudflare-e2e.outcome == 'failure'
+        with:
+          name: playwright-report-cloudflare-native-${{ matrix.shardIndex }}
+          path: |
+            playwright-report/
+            test-results/
+          retention-days: 7
+
+  test-e2e-endform:
+    name: E2E built Endform (full suite)
+    needs: change-plan
+    if: needs.change-plan.outputs.e2e_node == 'true' || needs.change-plan.outputs.e2e_cloudflare == 'true'
+    runs-on: ubuntu-latest
+    timeout-minutes: 90
+    permissions:
+      contents: read
+      id-token: write
+    env:
+      ENDFORM_GIT_BRANCH: ${{ github.head_ref || github.ref_name }}
+      ENDFORM_GIT_COMMIT_SHA: ${{ github.event.pull_request.head.sha || github.sha }}
+    steps:
+      - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
+        with:
+          persist-credentials: false
+      - uses: pnpm/action-setup@0e279bb959325dab635dd2c09392533439d90093 # v6.0.8
+      - uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0
+        with:
+          node-version: 22
+          cache: pnpm
+      - run: pnpm install --frozen-lockfile
+      - name: Build Node fixture
+        id: node-build
+        if: ${{ !cancelled() && needs.change-plan.outputs.e2e_node == 'true' }}
+        run: pnpm run --filter emdash... build
+      - name: Build Cloudflare fixture
+        id: cloudflare-build
+        if: ${{ !cancelled() && needs.change-plan.outputs.e2e_cloudflare == 'true' }}
+        run: pnpm run --filter "emdash-e2e-fixture-cloudflare..." build
+      - name: Run Node E2E
+        id: node-e2e
+        if: ${{ !cancelled() && steps.node-build.outcome == 'success' }}
+        run: |
+          pnpm exec endform --version
+          pnpm exec tsx e2e/run-endform.ts test --organization-id 2G1ZCj7X --timeout 240000
+      - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
+        if: failure() && steps.node-e2e.outcome == 'failure'
+        with:
+          name: playwright-report-node-endform
+          path: |
+            playwright-report/
+            test-results/
+          retention-days: 7
+      - name: Run Cloudflare E2E
+        id: cloudflare-e2e
+        if: ${{ !cancelled() && steps.cloudflare-build.outcome == 'success' }}
+        run: |
+          pnpm exec endform --version
+          pnpm exec tsx e2e/run-endform.ts test --organization-id 2G1ZCj7X --timeout 240000
         env:
           EMDASH_E2E_TARGET: cloudflare
       - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
         if: failure() && steps.cloudflare-e2e.outcome == 'failure'
         with:
-          name: playwright-report-cloudflare-${{ matrix.shardIndex }}
+          name: playwright-report-cloudflare-endform
           path: |
             playwright-report/
             test-results/

e2e/endform-server.ts

@@ -0,0 +1,56 @@
+import { readFileSync, writeFileSync } from "node:fs";
+import { createServer } from "node:http";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+
+import globalSetup from "./global-setup.js";
+import globalTeardown from "./global-teardown.js";
+
+const serverInfoPath = join(tmpdir(), "emdash-pw-server.json");
+
+await globalSetup();
+
+const server = createServer(async (request, response) => {
+	if (request.url === "/ready" && request.method === "GET") {
+		response.end("ready");
+		return;
+	}
+	if (request.url !== "/server-info") {
+		response.writeHead(404).end();
+		return;
+	}
+	if (request.method === "GET") {
+		response.setHeader("Content-Type", "application/json");
+		response.end(readFileSync(serverInfoPath));
+		return;
+	}
+	if (request.method === "POST") {
+		let body = "";
+		for await (const chunk of request) body += chunk;
+		const update: { token?: unknown } = JSON.parse(body);
+		if (typeof update.token !== "string") {
+			response.writeHead(400).end();
+			return;
+		}
+		const info = JSON.parse(readFileSync(serverInfoPath, "utf8"));
+		info.token = update.token;
+		writeFileSync(serverInfoPath, JSON.stringify(info));
+		response.end();
+		return;
+	}
+	response.writeHead(405).end();
+});
+
+server.listen(4446, "127.0.0.1");
+
+let stopping = false;
+async function stop() {
+	if (stopping) return;
+	stopping = true;
+	server.close();
+	await globalTeardown();
+	process.exit(0);
+}
+
+process.on("SIGINT", stop);
+process.on("SIGTERM", stop);

e2e/fixture-cloudflare/astro.config.mjs

@@ -16,6 +16,8 @@ import registryTestPlugin from "@emdash-cms/plugin-marketplace-test";
 import { defineConfig } from "astro/config";
 import emdash from "emdash/astro";
 
+import { builtTestHelpers } from "../fixtures/built-test-helpers.mjs";
+
 const marketplaceUrl = process.env.EMDASH_MARKETPLACE_URL || undefined;
 const registryUrl = process.env.EMDASH_REGISTRY_URL || undefined;
 const registryFixturePath = process.env.EMDASH_REGISTRY_FIXTURE;
@@ -80,17 +82,32 @@ export default defineConfig({
 	integrations: [
 		react(),
 		emdash({
+			siteUrl: "http://localhost:4444",
 			database: d1({ binding: "DB" }),
 			middleware: { outer: "./src/outer-middleware.ts" },
 			storage: r2({ binding: "MEDIA" }),
-			plugins: [colorPlugin()],
+			plugins: [
+				colorPlugin(),
+				...(process.env.EMDASH_E2E_SERVER === "dev"
+					? []
+					: [
+							{
+								id: "emdash-console-email",
+								version: "0.0.0",
+								entrypoint: fileURLToPath(
+									new URL("../fixtures/email-capture-plugin.mjs", import.meta.url),
+								),
+							},
+						]),
+			],
 			sandboxed: [{ ...registryTestPlugin, hooks: e2eHooks }, deniedPlugin],
 			marketplace: marketplaceUrl,
-			registry: registryUrl,
+			registry: process.env.EMDASH_E2E_SERVER === "dev" ? registryUrl : false,
 			sandboxRunner: sandbox(),
 		}),
 	],
 	i18n: {
+		routing: "manual",
 		defaultLocale: "en",
 		locales: ["en", "fr", "es"],
 		fallback: { fr: "en", es: "en" },
@@ -100,7 +117,7 @@ export default defineConfig({
 		define: {
 			__EMDASH_REGISTRY_FIXTURE__: JSON.stringify(registryFixture),
 		},
-		plugins: [lateManifestImport],
+		plugins: [builtTestHelpers(), lateManifestImport],
 		server: {
 			fs: { strict: false },
 		},

e2e/fixture-cloudflare/src/middleware.ts

@@ -0,0 +1 @@
+export { onRequest } from "../../fixtures/locale-middleware.js";

e2e/fixture/astro.config.mjs

@@ -16,13 +16,16 @@ import emdash from "emdash/astro";
 import { sqlite } from "emdash/db";
 import { installRegistryAuthoritativeFixture } from "emdash/internal/testing/registry";
 
+import { builtTestHelpers } from "../fixtures/built-test-helpers.mjs";
+
 const dbUrl = process.env.EMDASH_TEST_DB || "file:./test.db";
 const marketplaceUrl = process.env.EMDASH_MARKETPLACE_URL || undefined;
 const registryUrl = process.env.EMDASH_REGISTRY_URL || undefined;
 const registryFixturePath = process.env.EMDASH_REGISTRY_FIXTURE;
-if (registryFixturePath) {
-	installRegistryAuthoritativeFixture(JSON.parse(readFileSync(registryFixturePath, "utf8")));
-}
+const registryFixture = registryFixturePath
+	? JSON.parse(readFileSync(registryFixturePath, "utf8"))
+	: null;
+if (registryFixture) installRegistryAuthoritativeFixture(registryFixture);
 const e2eHookNames = new Set([
 	"content:afterSave",
 	"content:beforeDelete",
@@ -108,22 +111,39 @@ export default defineConfig({
 	integrations: [
 		react(),
 		emdash({
+			siteUrl: "http://localhost:4444",
 			database: sqlite({ url: dbUrl }),
 			middleware: { outer: "./src/outer-middleware.ts" },
-			plugins: [colorPlugin()],
+			plugins: [
+				colorPlugin(),
+				...(process.env.EMDASH_E2E_SERVER === "dev"
+					? []
+					: [
+							{
+								id: "emdash-console-email",
+								version: "0.0.0",
+								entrypoint: fileURLToPath(
+									new URL("../fixtures/email-capture-plugin.mjs", import.meta.url),
+								),
+							},
+						]),
+			],
 			sandboxed: [editorExtensionsPlugin, { ...registryTestPlugin, hooks: e2eHooks }, deniedPlugin],
 			marketplace: marketplaceUrl,
-			registry: registryUrl,
+			registry: process.env.EMDASH_E2E_SERVER === "dev" ? registryUrl : false,
 			sandboxRunner: "@emdash-cms/sandbox-workerd",
 		}),
 	],
 	i18n: {
+		routing: "manual",
 		defaultLocale: "en",
 		locales: ["en", "fr", "es"],
 		fallback: { fr: "en", es: "en" },
 	},
 	devToolbar: { enabled: false },
 	vite: {
+		define: { __EMDASH_REGISTRY_FIXTURE__: JSON.stringify(registryFixture) },
+		plugins: [builtTestHelpers()],
 		server: {
 			fs: { strict: false },
 		},

e2e/fixture/package.json

@@ -12,6 +12,7 @@
 		"astro": "catalog:",
 		"emdash": "workspace:*",
 		"react": "catalog:",
-		"react-dom": "catalog:"
+		"react-dom": "catalog:",
+		"sharp": "0.35.4"
 	}
 }

e2e/fixture/src/middleware.ts

@@ -0,0 +1 @@
+export { onRequest } from "../../fixtures/locale-middleware.js";

e2e/fixture/src/outer-middleware.ts

@@ -1,6 +1,13 @@
 import { defineMiddleware } from "astro:middleware";
+import { installRegistryAuthoritativeFixture } from "emdash/internal/testing/registry";
 import { getRequestContext } from "emdash/request-context";
 
+declare const __EMDASH_REGISTRY_FIXTURE__: unknown;
+
+if (__EMDASH_REGISTRY_FIXTURE__ !== null) {
+	installRegistryAuthoritativeFixture(__EMDASH_REGISTRY_FIXTURE__);
+}
+
 const FINALIZER_NONCE = "outer-finalizer";
 
 function preNextState(locals: App.Locals): string {