chore: Add visual regression testing

.github/workflows/visual-regression.yml

@@ -0,0 +1,101 @@
+name: Visual Regression Tests
+
+on:
+  pull_request:
+    branches:
+      - main
+
+defaults:
+  run:
+    shell: bash
+
+permissions:
+  id-token: write
+  contents: read
+
+jobs:
+  visual:
+    name: Visual regression
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+
+      - name: Setup Node.js
+        uses: actions/setup-node@v4
+        with:
+          node-version: 20
+          cache: npm
+
+      - name: Setup Chrome and ChromeDriver
+        uses: browser-actions/setup-chrome@v1
+        with:
+          chrome-version: stable
+
+      # ── Build PR (test) pages ──────────────────────────────────────────────
+      # Install the PR's dependencies and build its pages.
+      - name: Install PR dependencies
+        run: npm i
+
+      - name: Build PR pages
+        run: npx gulp quick-build
+        env:
+          NODE_ENV: production
+
+      - name: Bundle PR pages
+        run: node_modules/.bin/webpack --config pages/webpack.config.integ.cjs --output-path pages/lib/static-default
+        env:
+          NODE_ENV: production
+
+      # ── Build baseline (main) pages ────────────────────────────────────────
+      # Use a git worktree so the baseline has its own directory and its own
+      # node_modules. This means a PR that changes package-lock.json will still
+      # produce a correct baseline: the baseline installs from main's lockfile
+      # and the PR build installs from the PR's lockfile, so both sides use the
+      # dependency versions that are correct for their respective source trees.
+      - name: Create baseline worktree from origin/main
+        run: git worktree add /tmp/baseline origin/main
+
+      - name: Install baseline dependencies
+        run: npm i
+        working-directory: /tmp/baseline
+
+      - name: Build baseline pages
+        run: npx gulp quick-build
+        working-directory: /tmp/baseline
+        env:
+          NODE_ENV: production
+
+      - name: Bundle baseline pages
+        run: node_modules/.bin/webpack --config pages/webpack.config.integ.cjs --output-path ${{ github.workspace }}/pages/lib/static-visual-baseline
+        working-directory: /tmp/baseline
+        env:
+          NODE_ENV: production
+
+      # ── Run tests ─────────────────────────────────────────────────────────
+      - name: Start test server (port 8080)
+        run: node_modules/.bin/webpack serve --config pages/webpack.config.integ.cjs --port 8080 --static pages/lib/static-default --no-hot &
+        env:
+          NODE_ENV: development
+
+      - name: Start baseline server (port 8081)
+        run: node_modules/.bin/webpack serve --config pages/webpack.config.integ.cjs --port 8081 --static pages/lib/static-visual-baseline --no-hot &
+        env:
+          NODE_ENV: development
+
+      - name: Wait for servers to be ready
+        run: node_modules/.bin/wait-on http://localhost:8080 http://localhost:8081
+
+      - name: Run visual regression tests
+        run: NODE_OPTIONS=--experimental-vm-modules node_modules/.bin/jest -c jest.visual.config.js
+        env:
+          TZ: UTC
+
+      - name: Upload diff artifacts
+        if: failure()
+        uses: actions/upload-artifact@v4
+        with:
+          name: visual-regression-diffs
+          path: visual-regression-output/
+          retention-days: 14

build-tools/tasks/index.js

@@ -21,4 +21,5 @@ module.exports = {
   themeableSource: require('./themeable-source'),
   bundleVendorFiles: require('./bundle-vendor-files'),
   sizeLimit: require('./size-limit'),
+  visual: require('./visual'),
 };

build-tools/tasks/visual.js

@@ -0,0 +1,107 @@
+// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+// SPDX-License-Identifier: Apache-2.0
+const execa = require('execa');
+const path = require('path');
+const fs = require('fs');
+const waitOn = require('wait-on');
+const { task } = require('../utils/gulp-utils.js');
+const { parseArgs } = require('node:util');
+
+const BASELINE_WORKTREE = '/tmp/visual-baseline';
+const BASELINE_OUTPUT = path.resolve('pages/lib/static-visual-baseline');
+const TEST_OUTPUT = path.resolve('pages/lib/static-default');
+
+// Port assignments:
+//   8080 — test build (PR / local changes)
+//   8081 — baseline build (main branch)
+const TEST_PORT = 8080;
+const BASELINE_PORT = 8081;
+
+/**
+ * Serves a pre-built static directory using webpack-dev-server in static mode.
+ */
+function serveStatic(dir, port) {
+  return execa(
+    'node_modules/.bin/webpack',
+    ['serve', '--config', 'pages/webpack.config.integ.cjs', '--port', String(port), '--static', dir, '--no-hot'],
+    { env: { ...process.env, NODE_ENV: 'development' } }
+  );
+}
+
+/**
+ * Builds the dev pages from the source tree at `cwd` into `outputPath`.
+ * Uses the node_modules present in `cwd`.
+ */
+async function buildPages(cwd, outputPath) {
+  await execa('npx', ['gulp', 'quick-build'], {
+    stdio: 'inherit',
+    cwd,
+    env: { ...process.env, NODE_ENV: 'production' },
+  });
+  await execa(
+    path.join(cwd, 'node_modules/.bin/webpack'),
+    ['--config', 'pages/webpack.config.integ.cjs', '--output-path', outputPath],
+    { stdio: 'inherit', cwd, env: { ...process.env, NODE_ENV: 'production' } }
+  );
+}
+
+module.exports = task('test:visual', async () => {
+  const options = {
+    shard: { type: 'string' },
+    // Pass --skip-build to skip the build steps when artifacts are already present.
+    skipBuild: { type: 'boolean' },
+  };
+  const { shard, skipBuild } = parseArgs({ options, strict: false }).values;
+
+  const cwd = process.cwd();
+
+  if (!skipBuild) {
+    // ── 1. Build the test (PR) pages ────────────────────────────────────────
+    console.log('Building test pages (current branch)…');
+    await buildPages(cwd, TEST_OUTPUT);
+
+    // ── 2. Build the baseline (main) pages ──────────────────────────────────
+    // Create a worktree for origin/main so it gets its own node_modules.
+    // This correctly handles PRs that change package-lock.json: each side
+    // installs from its own lockfile.
+    console.log('Setting up baseline worktree from origin/main…');
+    if (fs.existsSync(BASELINE_WORKTREE)) {
+      await execa('git', ['worktree', 'remove', '--force', BASELINE_WORKTREE]);
+    }
+    await execa('git', ['worktree', 'add', BASELINE_WORKTREE, 'origin/main']);
+
+    try {
+      console.log('Installing baseline dependencies…');
+      await execa('npm', ['ci'], { stdio: 'inherit', cwd: BASELINE_WORKTREE });
+
+      console.log('Building baseline pages (origin/main)…');
+      await buildPages(BASELINE_WORKTREE, BASELINE_OUTPUT);
+    } finally {
+      await execa('git', ['worktree', 'remove', '--force', BASELINE_WORKTREE]);
+    }
+  }
+
+  // ── 3. Start both static servers ──────────────────────────────────────────
+  console.log(`Starting test server on :${TEST_PORT} (${TEST_OUTPUT})…`);
+  const testServer = serveStatic(TEST_OUTPUT, TEST_PORT);
+
+  console.log(`Starting baseline server on :${BASELINE_PORT} (${BASELINE_OUTPUT})…`);
+  const baselineServer = serveStatic(BASELINE_OUTPUT, BASELINE_PORT);
+
+  try {
+    await waitOn({ resources: [`http://localhost:${TEST_PORT}`, `http://localhost:${BASELINE_PORT}`] });
+
+    // ── 4. Run visual tests ──────────────────────────────────────────────────
+    const jestArgs = ['-c', 'jest.visual.config.js'];
+    if (shard) {
+      jestArgs.push(`--shard=${shard}`);
+    }
+    await execa('jest', jestArgs, {
+      stdio: 'inherit',
+      env: { ...process.env, NODE_OPTIONS: '--experimental-vm-modules' },
+    });
+  } finally {
+    testServer.cancel();
+    baselineServer.cancel();
+  }
+});

build-tools/visual/global-setup.js

@@ -0,0 +1,4 @@
+// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+// SPDX-License-Identifier: Apache-2.0
+const { startWebdriver } = require('@cloudscape-design/browser-test-tools/chrome-launcher');
+module.exports = () => startWebdriver();

build-tools/visual/global-teardown.js

@@ -0,0 +1,4 @@
+// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+// SPDX-License-Identifier: Apache-2.0
+const { shutdownWebdriver } = require('@cloudscape-design/browser-test-tools/chrome-launcher');
+module.exports = () => shutdownWebdriver();

build-tools/visual/setup.js

@@ -0,0 +1,18 @@
+// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+// SPDX-License-Identifier: Apache-2.0
+/* global jest */
+const { configure } = require('@cloudscape-design/browser-test-tools/use-browser');
+
+// The PR build (the code under test) is served on port 8080.
+// The baseline build (main branch, same node_modules) is served on port 8081.
+configure({
+  browserName: 'ChromeHeadlessIntegration',
+  browserCreatorOptions: {
+    seleniumUrl: 'http://localhost:9515',
+  },
+  webdriverOptions: {
+    baseUrl: 'http://localhost:8080',
+  },
+});
+
+jest.retryTimes(2, { logErrorsBeforeRetry: true });

docs/RUNNING_TESTS.md

@@ -60,11 +60,60 @@ TZ=UTC npx jest -u -c jest.unit.config.js src/
 ```
 ## Visual Regression Tests
 
-> **Note:** The components repository does not have visual regression tests on GitHub. This section applies to other repositories such as chat-components, code-view, chart-components, and board-components.
+Visual regression tests run automatically when opening a pull request in GitHub (see `.github/workflows/visual-regression.yml`).
 
-Visual regression tests for permutation pages run automatically when opening a pull request in GitHub.
+They compare permutation pages between the PR build and a baseline build of `main`, both served locally in the same CI job. Each side installs from its own `package-lock.json` via a git worktree, so dependency changes in the PR are handled correctly and unpinned updates in sister repositories affect both sides equally.
 
-To check results: look at the "Visual Regression Tests" action in the PR. The "Test for regressions" step logs which pages failed. For a full report, download the `visual-regression-snapshots-results` artifact from the action summary.
+### How it works
 
-If there are unexpected regressions, fix your pull request.
-If the changes are expected, call this out in your pull request comments.
+1. The PR pages are built and served on port 8080.
+2. A git worktree of `origin/main` is created, its dependencies installed, and its pages built and served on port 8081.
+3. The single test runner (`test/visual/visual.test.ts`) iterates over all test definitions, captures the `.screenshot-area` element from both servers for each test, and fails if any pixels differ.
+
+### Running locally
+
+```
+npm run test:visual
+```
+
+This handles the full build and comparison in one command. If both outputs are already built, skip the build step:
+
+```
+NODE_OPTIONS=--experimental-vm-modules node_modules/.bin/jest -c jest.visual.config.js
+```
+
+(Requires both servers to be running — start the PR build with `npm run start:integ` on port 8080 and the baseline build on port 8081, or set `NEW_HOST` / `OLD_HOST` env vars to point at different hosts.)
+
+### Adding tests for a new component
+
+Create `test/visual/definitions/<component>.ts`:
+
+```ts
+import { TestSuite } from '../types';
+
+const suite: TestSuite = {
+  description: 'my-component',
+  tests: [
+    {
+      description: 'permutations',
+      path: 'my-component/permutations',
+    },
+  ],
+};
+
+export default suite;
+```
+
+Then import and add it to `test/visual/definitions/index.ts`:
+
+```ts
+import myComponent from './my-component';
+
+export const allSuites: TestSuite[] = [..., myComponent];
+```
+
+### Reviewing failures
+
+If the CI job fails, download the `visual-regression-diffs` artifact from the Actions summary.
+
+If the diff is expected (intentional visual change), note it in your PR description.

eslint.config.mjs

@@ -225,7 +225,7 @@ export default tsEslint.config(
     },
   },
   {
-    files: ['**/__integ__/**', '**/__motion__/**', '**/__a11y__/**'],
+    files: ['**/__integ__/**', '**/__motion__/**', '**/__a11y__/**', 'test/visual/**'],
     rules: {
       // useBrowser is not a hook
       'react-hooks/rules-of-hooks': 'off',

gulpfile.js

@@ -19,6 +19,7 @@ const {
   generateI18nMessages,
   integ,
   motion,
+  visual,
   copyFiles,
   themeableSource,
   bundleVendorFiles,
@@ -41,6 +42,7 @@ exports['test:unit'] = unit;
 exports['test:integ'] = integ;
 exports['test:a11y'] = a11y;
 exports['test:motion'] = motion;
+exports['test:visual'] = visual;
 
 exports.watch = () => {
   watch(

jest.visual.config.js

@@ -0,0 +1,25 @@
+// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+// SPDX-License-Identifier: Apache-2.0
+const path = require('path');
+const os = require('os');
+
+module.exports = {
+  verbose: true,
+  testEnvironment: 'node',
+  transform: {
+    '^.+\\.tsx?$': [
+      'ts-jest',
+      {
+        tsconfig: 'tsconfig.integ.json',
+      },
+    ],
+  },
+  reporters: ['default', 'github-actions'],
+  testTimeout: 120_000, // 2min — pages can be tall and slow to capture
+  maxWorkers: os.cpus().length * (process.env.GITHUB_ACTION ? 3 : 1),
+  globalSetup: '<rootDir>/build-tools/visual/global-setup.js',
+  globalTeardown: '<rootDir>/build-tools/visual/global-teardown.js',
+  setupFilesAfterEnv: [path.join(__dirname, 'build-tools', 'visual', 'setup.js')],
+  moduleFileExtensions: ['js', 'ts'],
+  testMatch: ['<rootDir>/test/visual/visual.test.ts'],
+};