diff --git a/.github/workflows/visual-regression.yml b/.github/workflows/visual-regression.yml new file mode 100644 index 0000000000..7ee7d42669 --- /dev/null +++ b/.github/workflows/visual-regression.yml @@ -0,0 +1,101 @@ +name: Visual Regression Tests + +on: + pull_request: + branches: + - main + +defaults: + run: + shell: bash + +permissions: + id-token: write + contents: read + +jobs: + visual: + name: Visual regression + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Setup Node.js + uses: actions/setup-node@v4 + with: + node-version: 20 + cache: npm + + - name: Setup Chrome and ChromeDriver + uses: browser-actions/setup-chrome@v1 + with: + chrome-version: stable + + # ── Build PR (test) pages ────────────────────────────────────────────── + # Install the PR's dependencies and build its pages. + - name: Install PR dependencies + run: npm i + + - name: Build PR pages + run: npx gulp quick-build + env: + NODE_ENV: production + + - name: Bundle PR pages + run: node_modules/.bin/webpack --config pages/webpack.config.integ.cjs --output-path pages/lib/static-default + env: + NODE_ENV: production + + # ── Build baseline (main) pages ──────────────────────────────────────── + # Use a git worktree so the baseline has its own directory and its own + # node_modules. This means a PR that changes package-lock.json will still + # produce a correct baseline: the baseline installs from main's lockfile + # and the PR build installs from the PR's lockfile, so both sides use the + # dependency versions that are correct for their respective source trees. + - name: Create baseline worktree from origin/main + run: git worktree add /tmp/baseline origin/main + + - name: Install baseline dependencies + run: npm i + working-directory: /tmp/baseline + + - name: Build baseline pages + run: npx gulp quick-build + working-directory: /tmp/baseline + env: + NODE_ENV: production + + - name: Bundle baseline pages + run: node_modules/.bin/webpack --config pages/webpack.config.integ.cjs --output-path ${{ github.workspace }}/pages/lib/static-visual-baseline + working-directory: /tmp/baseline + env: + NODE_ENV: production + + # ── Run tests ───────────────────────────────────────────────────────── + - name: Start test server (port 8080) + run: node_modules/.bin/webpack serve --config pages/webpack.config.integ.cjs --port 8080 --static pages/lib/static-default --no-hot & + env: + NODE_ENV: development + + - name: Start baseline server (port 8081) + run: node_modules/.bin/webpack serve --config pages/webpack.config.integ.cjs --port 8081 --static pages/lib/static-visual-baseline --no-hot & + env: + NODE_ENV: development + + - name: Wait for servers to be ready + run: node_modules/.bin/wait-on http://localhost:8080 http://localhost:8081 + + - name: Run visual regression tests + run: NODE_OPTIONS=--experimental-vm-modules node_modules/.bin/jest -c jest.visual.config.js + env: + TZ: UTC + + - name: Upload diff artifacts + if: failure() + uses: actions/upload-artifact@v4 + with: + name: visual-regression-diffs + path: visual-regression-output/ + retention-days: 14 diff --git a/build-tools/tasks/index.js b/build-tools/tasks/index.js index 982a8f6a72..c653c71684 100644 --- a/build-tools/tasks/index.js +++ b/build-tools/tasks/index.js @@ -21,4 +21,5 @@ module.exports = { themeableSource: require('./themeable-source'), bundleVendorFiles: require('./bundle-vendor-files'), sizeLimit: require('./size-limit'), + visual: require('./visual'), }; diff --git a/build-tools/tasks/visual.js b/build-tools/tasks/visual.js new file mode 100644 index 0000000000..0864790afb --- /dev/null +++ b/build-tools/tasks/visual.js @@ -0,0 +1,107 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 +const execa = require('execa'); +const path = require('path'); +const fs = require('fs'); +const waitOn = require('wait-on'); +const { task } = require('../utils/gulp-utils.js'); +const { parseArgs } = require('node:util'); + +const BASELINE_WORKTREE = '/tmp/visual-baseline'; +const BASELINE_OUTPUT = path.resolve('pages/lib/static-visual-baseline'); +const TEST_OUTPUT = path.resolve('pages/lib/static-default'); + +// Port assignments: +// 8080 — test build (PR / local changes) +// 8081 — baseline build (main branch) +const TEST_PORT = 8080; +const BASELINE_PORT = 8081; + +/** + * Serves a pre-built static directory using webpack-dev-server in static mode. + */ +function serveStatic(dir, port) { + return execa( + 'node_modules/.bin/webpack', + ['serve', '--config', 'pages/webpack.config.integ.cjs', '--port', String(port), '--static', dir, '--no-hot'], + { env: { ...process.env, NODE_ENV: 'development' } } + ); +} + +/** + * Builds the dev pages from the source tree at `cwd` into `outputPath`. + * Uses the node_modules present in `cwd`. + */ +async function buildPages(cwd, outputPath) { + await execa('npx', ['gulp', 'quick-build'], { + stdio: 'inherit', + cwd, + env: { ...process.env, NODE_ENV: 'production' }, + }); + await execa( + path.join(cwd, 'node_modules/.bin/webpack'), + ['--config', 'pages/webpack.config.integ.cjs', '--output-path', outputPath], + { stdio: 'inherit', cwd, env: { ...process.env, NODE_ENV: 'production' } } + ); +} + +module.exports = task('test:visual', async () => { + const options = { + shard: { type: 'string' }, + // Pass --skip-build to skip the build steps when artifacts are already present. + skipBuild: { type: 'boolean' }, + }; + const { shard, skipBuild } = parseArgs({ options, strict: false }).values; + + const cwd = process.cwd(); + + if (!skipBuild) { + // ── 1. Build the test (PR) pages ──────────────────────────────────────── + console.log('Building test pages (current branch)…'); + await buildPages(cwd, TEST_OUTPUT); + + // ── 2. Build the baseline (main) pages ────────────────────────────────── + // Create a worktree for origin/main so it gets its own node_modules. + // This correctly handles PRs that change package-lock.json: each side + // installs from its own lockfile. + console.log('Setting up baseline worktree from origin/main…'); + if (fs.existsSync(BASELINE_WORKTREE)) { + await execa('git', ['worktree', 'remove', '--force', BASELINE_WORKTREE]); + } + await execa('git', ['worktree', 'add', BASELINE_WORKTREE, 'origin/main']); + + try { + console.log('Installing baseline dependencies…'); + await execa('npm', ['ci'], { stdio: 'inherit', cwd: BASELINE_WORKTREE }); + + console.log('Building baseline pages (origin/main)…'); + await buildPages(BASELINE_WORKTREE, BASELINE_OUTPUT); + } finally { + await execa('git', ['worktree', 'remove', '--force', BASELINE_WORKTREE]); + } + } + + // ── 3. Start both static servers ────────────────────────────────────────── + console.log(`Starting test server on :${TEST_PORT} (${TEST_OUTPUT})…`); + const testServer = serveStatic(TEST_OUTPUT, TEST_PORT); + + console.log(`Starting baseline server on :${BASELINE_PORT} (${BASELINE_OUTPUT})…`); + const baselineServer = serveStatic(BASELINE_OUTPUT, BASELINE_PORT); + + try { + await waitOn({ resources: [`http://localhost:${TEST_PORT}`, `http://localhost:${BASELINE_PORT}`] }); + + // ── 4. Run visual tests ────────────────────────────────────────────────── + const jestArgs = ['-c', 'jest.visual.config.js']; + if (shard) { + jestArgs.push(`--shard=${shard}`); + } + await execa('jest', jestArgs, { + stdio: 'inherit', + env: { ...process.env, NODE_OPTIONS: '--experimental-vm-modules' }, + }); + } finally { + testServer.cancel(); + baselineServer.cancel(); + } +}); diff --git a/build-tools/visual/global-setup.js b/build-tools/visual/global-setup.js new file mode 100644 index 0000000000..075ee6e398 --- /dev/null +++ b/build-tools/visual/global-setup.js @@ -0,0 +1,4 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 +const { startWebdriver } = require('@cloudscape-design/browser-test-tools/chrome-launcher'); +module.exports = () => startWebdriver(); diff --git a/build-tools/visual/global-teardown.js b/build-tools/visual/global-teardown.js new file mode 100644 index 0000000000..57ad21b454 --- /dev/null +++ b/build-tools/visual/global-teardown.js @@ -0,0 +1,4 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 +const { shutdownWebdriver } = require('@cloudscape-design/browser-test-tools/chrome-launcher'); +module.exports = () => shutdownWebdriver(); diff --git a/build-tools/visual/setup.js b/build-tools/visual/setup.js new file mode 100644 index 0000000000..2625a43809 --- /dev/null +++ b/build-tools/visual/setup.js @@ -0,0 +1,18 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 +/* global jest */ +const { configure } = require('@cloudscape-design/browser-test-tools/use-browser'); + +// The PR build (the code under test) is served on port 8080. +// The baseline build (main branch, same node_modules) is served on port 8081. +configure({ + browserName: 'ChromeHeadlessIntegration', + browserCreatorOptions: { + seleniumUrl: 'http://localhost:9515', + }, + webdriverOptions: { + baseUrl: 'http://localhost:8080', + }, +}); + +jest.retryTimes(2, { logErrorsBeforeRetry: true }); diff --git a/docs/RUNNING_TESTS.md b/docs/RUNNING_TESTS.md index 525cdf181d..c5c483e981 100644 --- a/docs/RUNNING_TESTS.md +++ b/docs/RUNNING_TESTS.md @@ -60,11 +60,60 @@ TZ=UTC npx jest -u -c jest.unit.config.js src/ ``` ## Visual Regression Tests -> **Note:** The components repository does not have visual regression tests on GitHub. This section applies to other repositories such as chat-components, code-view, chart-components, and board-components. +Visual regression tests run automatically when opening a pull request in GitHub (see `.github/workflows/visual-regression.yml`). -Visual regression tests for permutation pages run automatically when opening a pull request in GitHub. +They compare permutation pages between the PR build and a baseline build of `main`, both served locally in the same CI job. Each side installs from its own `package-lock.json` via a git worktree, so dependency changes in the PR are handled correctly and unpinned updates in sister repositories affect both sides equally. -To check results: look at the "Visual Regression Tests" action in the PR. The "Test for regressions" step logs which pages failed. For a full report, download the `visual-regression-snapshots-results` artifact from the action summary. +### How it works -If there are unexpected regressions, fix your pull request. -If the changes are expected, call this out in your pull request comments. +1. The PR pages are built and served on port 8080. +2. A git worktree of `origin/main` is created, its dependencies installed, and its pages built and served on port 8081. +3. The single test runner (`test/visual/visual.test.ts`) iterates over all test definitions, captures the `.screenshot-area` element from both servers for each test, and fails if any pixels differ. + +### Running locally + +``` +npm run test:visual +``` + +This handles the full build and comparison in one command. If both outputs are already built, skip the build step: + +``` +NODE_OPTIONS=--experimental-vm-modules node_modules/.bin/jest -c jest.visual.config.js +``` + +(Requires both servers to be running — start the PR build with `npm run start:integ` on port 8080 and the baseline build on port 8081, or set `NEW_HOST` / `OLD_HOST` env vars to point at different hosts.) + +### Adding tests for a new component + +Create `test/visual/definitions/.ts`: + +```ts +import { TestSuite } from '../types'; + +const suite: TestSuite = { + description: 'my-component', + tests: [ + { + description: 'permutations', + path: 'my-component/permutations', + }, + ], +}; + +export default suite; +``` + +Then import and add it to `test/visual/definitions/index.ts`: + +```ts +import myComponent from './my-component'; + +export const allSuites: TestSuite[] = [..., myComponent]; +``` + +### Reviewing failures + +If the CI job fails, download the `visual-regression-diffs` artifact from the Actions summary. + +If the diff is expected (intentional visual change), note it in your PR description. diff --git a/eslint.config.mjs b/eslint.config.mjs index 0d9423aa5b..f03eb9ce60 100644 --- a/eslint.config.mjs +++ b/eslint.config.mjs @@ -225,7 +225,7 @@ export default tsEslint.config( }, }, { - files: ['**/__integ__/**', '**/__motion__/**', '**/__a11y__/**'], + files: ['**/__integ__/**', '**/__motion__/**', '**/__a11y__/**', 'test/visual/**'], rules: { // useBrowser is not a hook 'react-hooks/rules-of-hooks': 'off', diff --git a/gulpfile.js b/gulpfile.js index 9d290b74f1..216581b858 100644 --- a/gulpfile.js +++ b/gulpfile.js @@ -19,6 +19,7 @@ const { generateI18nMessages, integ, motion, + visual, copyFiles, themeableSource, bundleVendorFiles, @@ -41,6 +42,7 @@ exports['test:unit'] = unit; exports['test:integ'] = integ; exports['test:a11y'] = a11y; exports['test:motion'] = motion; +exports['test:visual'] = visual; exports.watch = () => { watch( diff --git a/jest.visual.config.js b/jest.visual.config.js new file mode 100644 index 0000000000..7f1d020880 --- /dev/null +++ b/jest.visual.config.js @@ -0,0 +1,25 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 +const path = require('path'); +const os = require('os'); + +module.exports = { + verbose: true, + testEnvironment: 'node', + transform: { + '^.+\\.tsx?$': [ + 'ts-jest', + { + tsconfig: 'tsconfig.integ.json', + }, + ], + }, + reporters: ['default', 'github-actions'], + testTimeout: 120_000, // 2min — pages can be tall and slow to capture + maxWorkers: os.cpus().length * (process.env.GITHUB_ACTION ? 3 : 1), + globalSetup: '/build-tools/visual/global-setup.js', + globalTeardown: '/build-tools/visual/global-teardown.js', + setupFilesAfterEnv: [path.join(__dirname, 'build-tools', 'visual', 'setup.js')], + moduleFileExtensions: ['js', 'ts'], + testMatch: ['/test/visual/visual.test.ts'], +}; diff --git a/package-lock.json b/package-lock.json index c8880d1fc4..ae3d2d5545 100644 --- a/package-lock.json +++ b/package-lock.json @@ -52,6 +52,7 @@ "@types/jest": "^29.5.13", "@types/lodash": "^4.14.176", "@types/node": "^20.17.14", + "@types/pixelmatch": "^5.2.6", "@types/react": "^16.14.20", "@types/react-dom": "^16.9.14", "@types/react-is": "^18.2.0", @@ -96,6 +97,7 @@ "mockdate": "^3.0.5", "npm-run-all": "^4.1.5", "prettier": "^3.6.1", + "puppeteer-core": "^24.43.1", "react": "^16.14.0", "react-dom": "^16.14.0", "react-dom18": "npm:react-dom@^18.3.1", @@ -3521,9 +3523,9 @@ } }, "node_modules/@puppeteer/browsers": { - "version": "2.13.0", - "resolved": "https://registry.npmjs.org/@puppeteer/browsers/-/browsers-2.13.0.tgz", - "integrity": "sha512-46BZJYJjc/WwmKjsvDFykHtXrtomsCIrwYQPOP7VfMJoZY2bsDF9oROBABR3paDjDcmkUye1Pb1BqdcdiipaWA==", + "version": "2.13.2", + "resolved": "https://registry.npmjs.org/@puppeteer/browsers/-/browsers-2.13.2.tgz", + "integrity": "sha512-5EUZSUIc37H6aIXyWO0Z4y8NlF8NnjgmqeQgOGiswAU7pY0HOo16ho4+alIWmSfdZnjqBRawMsP3I5YqLSn6kw==", "dev": true, "license": "Apache-2.0", "dependencies": { @@ -4844,6 +4846,16 @@ "undici-types": "~6.21.0" } }, + "node_modules/@types/pixelmatch": { + "version": "5.2.6", + "resolved": "https://registry.npmjs.org/@types/pixelmatch/-/pixelmatch-5.2.6.tgz", + "integrity": "sha512-wC83uexE5KGuUODn6zkm9gMzTwdY5L0chiK+VrKcDfEjzxh1uadlWTvOmAbCpnM9zx/Ww3f8uKlYQVnO/TrqVg==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/node": "*" + } + }, "node_modules/@types/pngjs": { "version": "6.0.5", "dev": true, @@ -7260,6 +7272,20 @@ "node": ">=6.0" } }, + "node_modules/chromium-bidi": { + "version": "14.0.0", + "resolved": "https://registry.npmjs.org/chromium-bidi/-/chromium-bidi-14.0.0.tgz", + "integrity": "sha512-9gYlLtS6tStdRWzrtXaTMnqcM4dudNegMXJxkR0I/CXObHalYeYcAMPrL19eroNZHtJ8DQmu1E+ZNOYu/IXMXw==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "mitt": "^3.0.1", + "zod": "^3.24.1" + }, + "peerDependencies": { + "devtools-protocol": "*" + } + }, "node_modules/ci-info": { "version": "3.9.0", "dev": true, @@ -8776,6 +8802,13 @@ "dev": true, "license": "MIT" }, + "node_modules/devtools-protocol": { + "version": "0.0.1608973", + "resolved": "https://registry.npmjs.org/devtools-protocol/-/devtools-protocol-0.0.1608973.tgz", + "integrity": "sha512-Tpm17fxYzt+J7VrGdc1k8YdRqS3YV7se/M6KeemEqvUbq/n7At1rWVuXMxQgpWkdwSdIEKYbU//Bve+Shm4YNQ==", + "dev": true, + "license": "BSD-3-Clause" + }, "node_modules/diff-sequences": { "version": "29.6.3", "dev": true, @@ -17732,6 +17765,25 @@ "node": ">=6" } }, + "node_modules/puppeteer-core": { + "version": "24.43.1", + "resolved": "https://registry.npmjs.org/puppeteer-core/-/puppeteer-core-24.43.1.tgz", + "integrity": "sha512-T5ScUMAsmhdNbgDR41AGESYeS6V9MSgetkSnVhhW+gXvzC42VesKCn5ld87gAZDJ6vLHL9GkRvY9WtQWSnwFbw==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "@puppeteer/browsers": "2.13.2", + "chromium-bidi": "14.0.0", + "debug": "^4.4.3", + "devtools-protocol": "0.0.1608973", + "typed-query-selector": "^2.12.2", + "webdriver-bidi-protocol": "0.4.1", + "ws": "^8.20.0" + }, + "engines": { + "node": ">=18" + } + }, "node_modules/pure-rand": { "version": "6.1.0", "dev": true, @@ -21144,6 +21196,13 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/typed-query-selector": { + "version": "2.12.2", + "resolved": "https://registry.npmjs.org/typed-query-selector/-/typed-query-selector-2.12.2.tgz", + "integrity": "sha512-EOPFbyIub4ngnEdqi2yOcNeDLaX/0jcE1JoAXQDDMIthap7FoN795lc/SHfIq2d416VufXpM8z/lD+WRm2gfOQ==", + "dev": true, + "license": "MIT" + }, "node_modules/typedarray": { "version": "0.0.6", "dev": true, @@ -21661,6 +21720,13 @@ "node": ">=18.20.0" } }, + "node_modules/webdriver-bidi-protocol": { + "version": "0.4.1", + "resolved": "https://registry.npmjs.org/webdriver-bidi-protocol/-/webdriver-bidi-protocol-0.4.1.tgz", + "integrity": "sha512-ARrjNjtWRRs2w4Tk7nqrf2gBI0QXWuOmMCx2hU+1jUt6d00MjMxURrhxhGbrsoiZKJrhTSTzbIrc554iKI10qw==", + "dev": true, + "license": "Apache-2.0" + }, "node_modules/webdriver/node_modules/agent-base": { "version": "7.1.4", "resolved": "https://registry.npmjs.org/agent-base/-/agent-base-7.1.4.tgz", @@ -22267,7 +22333,9 @@ } }, "node_modules/ws": { - "version": "8.18.2", + "version": "8.20.0", + "resolved": "https://registry.npmjs.org/ws/-/ws-8.20.0.tgz", + "integrity": "sha512-sAt8BhgNbzCtgGbt2OxmpuryO63ZoDk/sqaB/znQm94T4fCEsy/yV+7CdC1kJhOU9lboAEU7R3kquuycDoibVA==", "dev": true, "license": "MIT", "engines": { @@ -22420,6 +22488,16 @@ "engines": { "node": "^12.22.0 || ^14.17.0 || >=16.0.0" } + }, + "node_modules/zod": { + "version": "3.25.76", + "resolved": "https://registry.npmjs.org/zod/-/zod-3.25.76.tgz", + "integrity": "sha512-gzUt/qt81nXsFGKIFcC3YnfEAx5NkunCfnDlvuBSSFS02bcXu4Lmea0AFIUwbLWxWPx3d9p8S5QoaujKcNQxcQ==", + "dev": true, + "license": "MIT", + "funding": { + "url": "https://github.com/sponsors/colinhacks" + } } } } diff --git a/package.json b/package.json index 39b9206218..22336b244c 100644 --- a/package.json +++ b/package.json @@ -15,6 +15,7 @@ "test:a11y": "gulp test:a11y", "test:integ": "gulp test:integ", "test:motion": "gulp test:motion", + "test:visual": "gulp test:visual", "lint": "npm-run-all --parallel lint:*", "lint:eslint": "eslint .", "lint:stylelint": "stylelint --ignore-path .gitignore '{src,pages}/**/*.{css,scss}'", @@ -75,6 +76,7 @@ "@types/jest": "^29.5.13", "@types/lodash": "^4.14.176", "@types/node": "^20.17.14", + "@types/pixelmatch": "^5.2.6", "@types/react": "^16.14.20", "@types/react-dom": "^16.9.14", "@types/react-is": "^18.2.0", @@ -119,6 +121,7 @@ "mockdate": "^3.0.5", "npm-run-all": "^4.1.5", "prettier": "^3.6.1", + "puppeteer-core": "^24.43.1", "react": "^16.14.0", "react-dom": "^16.14.0", "react-dom18": "npm:react-dom@^18.3.1", diff --git a/test/visual/compare-screenshots.ts b/test/visual/compare-screenshots.ts new file mode 100644 index 0000000000..8847e10d21 --- /dev/null +++ b/test/visual/compare-screenshots.ts @@ -0,0 +1,113 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 +import pixelmatch from 'pixelmatch'; +import { PNG } from 'pngjs'; + +import { parsePng } from '@cloudscape-design/browser-test-tools/image-utils'; +import { ScreenshotPageObject } from '@cloudscape-design/browser-test-tools/page-objects'; +import useBrowser from '@cloudscape-design/browser-test-tools/use-browser'; + +import { TestDefinition, TestSuite } from './types'; + +const screenshotAreaSelector = '.screenshot-area'; +const defaultWindowSize = { width: 1600, height: 800 }; + +// NEW_HOST serves the PR's pages, OLD_HOST serves the baseline (main) pages. +const newHost = process.env.NEW_HOST || 'http://localhost:8080'; +const oldHost = process.env.OLD_HOST || 'http://localhost:8081'; + +/** + * Captures the .screenshot-area element on a focused page. + * Uses a standard ScreenshotPageObject (no forced scroll-and-merge). + */ +async function captureScreenshotArea(browser: WebdriverIO.Browser, url: string): Promise { + await browser.url(url); + const page = new ScreenshotPageObject(browser); + await page.waitForVisible(screenshotAreaSelector); + const { image } = await page.captureBySelector(screenshotAreaSelector); + return image; +} + +/** + * Captures the full page as a PNG for permutation pages. + * Uses fullPageScreenshot which handles pages taller than the viewport. + */ +async function capturePermutations(browser: WebdriverIO.Browser, url: string): Promise { + await browser.url(url); + const page = new ScreenshotPageObject(browser); + await page.waitForVisible(screenshotAreaSelector); + const base64 = await page.fullPageScreenshot(); + return parsePng(base64); +} + +async function captureScreenshot( + browser: WebdriverIO.Browser, + url: string, + testDef: TestDefinition, + setup?: (page: ScreenshotPageObject) => Promise +): Promise { + if (setup) { + await browser.url(url); + const page = new ScreenshotPageObject(browser); + await page.waitForVisible(screenshotAreaSelector); + await setup(page); + if (testDef.screenshotType === 'permutations') { + const base64 = await page.fullPageScreenshot(); + return parsePng(base64); + } + const { image } = await page.captureBySelector(screenshotAreaSelector); + return image; + } + if (testDef.screenshotType === 'permutations') { + return capturePermutations(browser, url); + } + return captureScreenshotArea(browser, url); +} + +function buildUrl(host: string, path: string, queryParams?: Record): string { + const params = new URLSearchParams(queryParams); + const qs = params.toString(); + return `${host}/#/${path}${qs ? `?${qs}` : ''}`; +} + +function compareImages(newImage: PNG, oldImage: PNG): number { + const { width, height } = newImage; + const diff = new PNG({ width, height }); + return pixelmatch(newImage.data, oldImage.data, diff.data, width, height, { threshold: 0.1 }); +} + +function isTestDefinition(item: TestDefinition | TestSuite): item is TestDefinition { + return (item as TestDefinition).path !== undefined; +} + +export function runTestSuites(suites: Array) { + for (const item of suites) { + if (isTestDefinition(item)) { + runSingleTest(item); + } else { + describe(item.description, () => { + runTestSuites(item.tests); + }); + } + } +} + +function runSingleTest(testDef: TestDefinition) { + const windowSize = { ...defaultWindowSize, ...testDef.configuration }; + + test( + testDef.description, + useBrowser(windowSize, async browser => { + const newUrl = buildUrl(newHost, testDef.path, testDef.queryParams); + const newScreenshot = await captureScreenshot(browser, newUrl, testDef, testDef.setup); + + const oldUrl = buildUrl(oldHost, testDef.path, testDef.queryParams); + const oldScreenshot = await captureScreenshot(browser, oldUrl, testDef, testDef.setup); + const diffPixels = compareImages(newScreenshot, oldScreenshot); + expect(diffPixels).toBe(0); + }) + ); +} + +// Export the capture functions for use in custom setup callbacks if needed. +export { captureScreenshotArea, capturePermutations }; diff --git a/test/visual/definitions/action-card.ts b/test/visual/definitions/action-card.ts new file mode 100644 index 0000000000..8730ac9504 --- /dev/null +++ b/test/visual/definitions/action-card.ts @@ -0,0 +1,31 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 +import { TestSuite } from '../types'; + +const suite: TestSuite = { + description: 'action-card', + tests: [ + { + description: 'permutations', + path: 'action-card/permutations', + screenshotType: 'permutations', + }, + { + description: 'variant permutations', + path: 'action-card/variant-permutations', + screenshotType: 'permutations', + }, + { + description: 'padding permutations', + path: 'action-card/padding-permutations', + screenshotType: 'permutations', + }, + { + description: 'simple', + path: 'action-card/simple', + screenshotType: 'screenshotArea', + }, + ], +}; + +export default suite; diff --git a/test/visual/definitions/alert.ts b/test/visual/definitions/alert.ts new file mode 100644 index 0000000000..30792c0d2d --- /dev/null +++ b/test/visual/definitions/alert.ts @@ -0,0 +1,37 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { TestSuite } from '../types'; + +const suite: TestSuite = { + description: 'alert', + tests: [ + { + description: 'simple', + path: 'alert/simple', + screenshotType: 'screenshotArea', + }, + { + description: 'style custom page', + path: 'alert/style-custom-types', + screenshotType: 'screenshotArea', + }, + ...[600, 1280].map(width => ({ + description: `width ${width}px`, + tests: [ + { + description: 'permutations', + path: 'alert/permutations', + screenshotType: 'permutations' as const, + }, + { + description: 'custom types', + path: 'alert/style-custom-types', + screenshotType: 'screenshotArea' as const, + }, + ], + })), + ], +}; + +export default suite; diff --git a/test/visual/definitions/index.ts b/test/visual/definitions/index.ts new file mode 100644 index 0000000000..318ce7c68b --- /dev/null +++ b/test/visual/definitions/index.ts @@ -0,0 +1,10 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +// Each component has its own test definition file. +// Import them here manually to form the full test suite. +import { TestSuite } from '../types'; +import actionCard from './action-card'; +import alert from './alert'; + +export const allSuites: TestSuite[] = [actionCard, alert]; diff --git a/test/visual/types.ts b/test/visual/types.ts new file mode 100644 index 0000000000..f0fa665863 --- /dev/null +++ b/test/visual/types.ts @@ -0,0 +1,28 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 +import { ScreenshotPageObject } from '@cloudscape-design/browser-test-tools/page-objects'; + +export interface ScreenshotTestConfiguration { + width?: number; + height?: number; +} + +export type TestCallback = (page: ScreenshotPageObject) => Promise; + +// 'screenshotArea' — captures the .screenshot-area element on a focused page. +// 'permutations' — captures the entire page and crops permutations out of it. +export type ScreenshotType = 'screenshotArea' | 'permutations'; + +export interface TestDefinition { + description: string; + path: string; + screenshotType: ScreenshotType; + queryParams?: Record; + configuration?: ScreenshotTestConfiguration; + setup?: TestCallback; +} + +export interface TestSuite { + description: string; + tests: Array; +} diff --git a/test/visual/visual.test.ts b/test/visual/visual.test.ts new file mode 100644 index 0000000000..06ef0fa8a4 --- /dev/null +++ b/test/visual/visual.test.ts @@ -0,0 +1,6 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 +import { runTestSuites } from './compare-screenshots'; +import { allSuites } from './definitions'; + +runTestSuites(allSuites);