From f71ca450a676b889c27b40671de400b45ed55654 Mon Sep 17 00:00:00 2001 From: Simon Knott Date: Fri, 10 Jul 2026 16:01:34 +0200 Subject: [PATCH 1/6] fix(reporter): expose setup/teardown projects to preprocessSuite Closes https://github.com/microsoft/playwright/issues/41708 --- docs/src/test-reporter-api/class-reporter.md | 2 +- packages/playwright/src/common/test.ts | 13 ++++++ packages/playwright/src/runner/loadUtils.ts | 38 ++++++++------- packages/playwright/types/testReporter.d.ts | 3 +- .../reporter-preprocess-suite.spec.ts | 46 ++++++++++++++----- 5 files changed, 71 insertions(+), 31 deletions(-) diff --git a/docs/src/test-reporter-api/class-reporter.md b/docs/src/test-reporter-api/class-reporter.md index 26f8d619e22ca..67619c914e1f7 100644 --- a/docs/src/test-reporter-api/class-reporter.md +++ b/docs/src/test-reporter-api/class-reporter.md @@ -317,6 +317,6 @@ Resolved configuration. The root suite that contains the projects, files and test cases that will run. -The suite reflects `--project`, `--grep`/`--grep-invert` and `.only` filtering, so it only contains tests that match the current invocation. It contains only the top-level projects being run — setup and dependency projects are not included and cannot be excluded from here. +The suite reflects `--project`, `--grep`/`--grep-invert` and `.only` filtering, so it only contains tests that match the current invocation. Setup and dependency projects are readonly and cannot be excluded from here. The suite ignores the `--shard` argument: it always contains the full, un-sharded corpus. Playwright applies its built-in sharding after [`method: Reporter.preprocessSuite`] returns, unless the returned `implementsSharding` is `true`. diff --git a/packages/playwright/src/common/test.ts b/packages/playwright/src/common/test.ts index 991ac8304ef0d..c5ba38a2d7b25 100644 --- a/packages/playwright/src/common/test.ts +++ b/packages/playwright/src/common/test.ts @@ -59,6 +59,7 @@ export class Suite extends Base { _fullProject: FullProjectInternal | undefined; _fileId: string | undefined; _preprocessing = false; + _isDependency = false; readonly _type: 'root' | 'project' | 'file' | 'describe'; skip: (reason?: string) => void; @@ -272,6 +273,8 @@ export class Suite extends Base { private _modifier(type: 'skip' | 'fixme' | 'fail', location: Location, reason: string | undefined): void { if (!this._rootSuite()._preprocessing) throw new Error(`Suite.${type}() can only be called from Reporter.preprocessSuite().`); + if (this._isInDependencyProject()) + throw new Error(`Suite.${type}() cannot be called on a setup or teardown project; these always run in full.`); for (const test of this.allTests()) test._applyPlanAnnotation({ type, description: reason, location }); } @@ -281,9 +284,15 @@ export class Suite extends Base { throw new Error(`Suite.exclude() can only be called from Reporter.preprocessSuite().`); if (!this.parent) throw new Error(`Suite.exclude() cannot be called on the root suite.`); + if (this._isInDependencyProject()) + throw new Error(`Suite.exclude() cannot be called on a setup or teardown project; these always run in full.`); this.parent._detach(this); } + _isInDependencyProject(): boolean { + return this._isDependency || (this.parent?._isInDependencyProject() ?? false); + } + _rootSuite(): Suite { return this.parent?._rootSuite() ?? this; } @@ -355,6 +364,8 @@ export class TestCase extends Base implements reporterTypes.TestCase { private _modifier(type: 'skip' | 'fixme' | 'fail', location: Location, reason: string | undefined): void { if (!this._rootSuite()._preprocessing) throw new Error(`TestCase.${type}() can only be called from Reporter.preprocessSuite().`); + if (this.parent._isInDependencyProject()) + throw new Error(`TestCase.${type}() cannot be called on a setup or teardown project test; these always run in full.`); this._applyPlanAnnotation({ type, description: reason, location }); } @@ -370,6 +381,8 @@ export class TestCase extends Base implements reporterTypes.TestCase { exclude(): void { if (!this._rootSuite()._preprocessing) throw new Error(`TestCase.exclude() can only be called from Reporter.preprocessSuite().`); + if (this.parent._isInDependencyProject()) + throw new Error(`TestCase.exclude() cannot be called on a setup or teardown project test; these always run in full.`); this.parent._detach(this); } diff --git a/packages/playwright/src/runner/loadUtils.ts b/packages/playwright/src/runner/loadUtils.ts index 9485a923ba5a1..ed5b24acc0280 100644 --- a/packages/playwright/src/runner/loadUtils.ts +++ b/packages/playwright/src/runner/loadUtils.ts @@ -165,12 +165,23 @@ export async function createRootSuite(testRun: TestRun, errors: TestError[], sho } } + // Prepend dependency projects without filtration. + for (const [project, type] of [...projectClosure].reverse()) { + if (type !== 'dependency') + continue; + const dependencySuite = buildProjectSuite(project, projectSuites.get(project)!); + dependencySuite._isDependency = true; + rootSuite._prependSuite(dependencySuite); + } + const preprocessResult = await testRun.reporter.preprocessSuite(config.config, rootSuite); // Shard only the top-level projects. if (config.config.shard && !preprocessResult?.implementsSharding) { // Create test groups for top-level projects. const testGroups: TestGroup[] = []; for (const projectSuite of rootSuite.suites) { + if (projectSuite._isDependency) + continue; // Split beforeAll-grouped tests into "config.shard.total" groups when needed. // Later on, we'll re-split them between workers by using "config.workers" instead. for (const group of createTestGroups(projectSuite, config.config.shard.total)) @@ -186,26 +197,19 @@ export async function createRootSuite(testRun: TestRun, errors: TestError[], sho } // Update project suites, removing empty ones. - suiteUtils.filterTestsRemoveEmptySuites(rootSuite, test => testsInThisShard.has(test)); + suiteUtils.filterTestsRemoveEmptySuites(rootSuite, test => test.parent._isInDependencyProject() || testsInThisShard.has(test)); } if (testRun.postShardTestFilters.length) - suiteUtils.filterTestsRemoveEmptySuites(rootSuite, test => testRun.postShardTestFilters.every(filter => filter(test))); - - const topLevelProjects = []; - // Now prepend dependency projects without filtration. - { - // Filtering 'only' and sharding might have reduced the number of top-level projects. - // Build the project closure to only include dependencies that are still needed. - const projectClosure = new Map(buildProjectsClosure(rootSuite.suites.map(suite => suite._fullProject!))); - - // Clone file suites for dependency projects. - for (const [project, level] of projectClosure.entries()) { - if (level === 'dependency') - rootSuite._prependSuite(buildProjectSuite(project, projectSuites.get(project)!)); - else - topLevelProjects.push(project); - } + suiteUtils.filterTestsRemoveEmptySuites(rootSuite, test => test.parent._isInDependencyProject() || testRun.postShardTestFilters.every(filter => filter(test))); + + // Filtering 'only' and sharding might have reduced the number of top-level projects. + // Prune the project closure to only include dependencies that are still needed. + const topLevelProjects = rootSuite.suites.filter(suite => !suite._isDependency).map(suite => suite._fullProject!); + const neededClosure = buildProjectsClosure(topLevelProjects); + for (const projectSuite of [...rootSuite.suites]) { + if (projectSuite._isDependency && !neededClosure.has(projectSuite._fullProject!)) + rootSuite._detach(projectSuite); } testRun.rootSuite = rootSuite; diff --git a/packages/playwright/types/testReporter.d.ts b/packages/playwright/types/testReporter.d.ts index 47e406bcd1d82..f7809c29c65fe 100644 --- a/packages/playwright/types/testReporter.d.ts +++ b/packages/playwright/types/testReporter.d.ts @@ -153,8 +153,7 @@ export interface Reporter { * @param suite The root suite that contains the projects, files and test cases that will run. * * The suite reflects `--project`, `--grep`/`--grep-invert` and `.only` filtering, so it only contains tests that - * match the current invocation. It contains only the top-level projects being run — setup and dependency projects are - * not included and cannot be excluded from here. + * match the current invocation. Setup and dependency projects are readonly and cannot be excluded from here. * * The suite ignores the `--shard` argument: it always contains the full, un-sharded corpus. Playwright applies its * built-in sharding after diff --git a/tests/playwright-test/reporter-preprocess-suite.spec.ts b/tests/playwright-test/reporter-preprocess-suite.spec.ts index 7e2f16342851e..52a43acb3ee77 100644 --- a/tests/playwright-test/reporter-preprocess-suite.spec.ts +++ b/tests/playwright-test/reporter-preprocess-suite.spec.ts @@ -392,15 +392,29 @@ test('multiple reporters declaring implementsSharding throws', async ({ runInlin expect(result.outputLines.join('\n')).toContain(`Multiple reporters declare 'implementsSharding'`); }); -test('plan.suite contains only top-level projects, not dependency/setup projects', async ({ runInlineTest }) => { +test('plan.suite exposes setup/teardown dependency projects but they are read-only', async ({ runInlineTest }) => { const result = await runInlineTest({ 'reporter.ts': ` class Reporter { async preprocessSuite(config, suite) { - // The suite only exposes top-level projects, so a reporter has no handle on - // setup/dependency project tests and therefore cannot exclude them. console.log('%% plan projects: ' + suite.suites.map(s => s.title).join(',')); console.log('%% plan tests: ' + suite.allTests().map(t => t.title).join(',')); + const setupTest = suite.allTests().find(t => t.title === 'setup-test'); + for (const method of ['skip', 'fixme', 'fail', 'exclude']) { + try { + setupTest[method](); + console.log('%% dep-' + method + ': no-throw'); + } catch (e) { + console.log('%% dep-' + method + ': ' + e.message); + } + } + const setupProject = suite.suites.find(s => s.title === 'setup'); + try { + setupProject.exclude(); + console.log('%% dep-suite-exclude: no-throw'); + } catch (e) { + console.log('%% dep-suite-exclude: ' + e.message); + } } onTestEnd(test, result) { console.log('%% ran ' + test.parent.project().name + '/' + test.title); @@ -412,7 +426,8 @@ test('plan.suite contains only top-level projects, not dependency/setup projects module.exports = { reporter: './reporter.ts', projects: [ - { name: 'setup', testMatch: /a\\.setup\\.ts/ }, + { name: 'setup', testMatch: /a\\.setup\\.ts/, teardown: 'teardown' }, + { name: 'teardown', testMatch: /a\\.teardown\\.ts/ }, { name: 'main', testMatch: /a\\.test\\.ts/, dependencies: ['setup'] }, ], }; @@ -421,6 +436,10 @@ test('plan.suite contains only top-level projects, not dependency/setup projects import { test } from '@playwright/test'; test('setup-test', async () => {}); `, + 'a.teardown.ts': ` + import { test } from '@playwright/test'; + test('teardown-test', async () => {}); + `, 'a.test.ts': ` import { test } from '@playwright/test'; test('main-test', async () => {}); @@ -428,11 +447,16 @@ test('plan.suite contains only top-level projects, not dependency/setup projects }, { reporter: '', workers: 1 }, undefined, { additionalArgs: ['--project=main'] }); expect(result.exitCode).toBe(0); - // plan only sees the top-level 'main' project; the 'setup' dependency is prepended afterwards. - expect(result.outputLines).toContain('plan projects: main'); - // 'setup-test' is absent from the plan suite, proving setup/dependency tests are not exposed. - expect(result.outputLines).toContain('plan tests: main-test'); - // Both the dependency and the main project still run. - expect(result.outputLines).toContain('ran setup/setup-test'); - expect(result.outputLines).toContain('ran main/main-test'); + expect(result.outputLines).toEqual([ + 'plan projects: setup,teardown,main', + 'plan tests: setup-test,teardown-test,main-test', + 'dep-skip: TestCase.skip() cannot be called on a setup or teardown project test; these always run in full.', + 'dep-fixme: TestCase.fixme() cannot be called on a setup or teardown project test; these always run in full.', + 'dep-fail: TestCase.fail() cannot be called on a setup or teardown project test; these always run in full.', + 'dep-exclude: TestCase.exclude() cannot be called on a setup or teardown project test; these always run in full.', + 'dep-suite-exclude: Suite.exclude() cannot be called on a setup or teardown project; these always run in full.', + 'ran setup/setup-test', + 'ran main/main-test', + 'ran teardown/teardown-test', + ]); }); From 22a60540e40dc5d14b9a7bb678e03ebdf408f05c Mon Sep 17 00:00:00 2001 From: Simon Knott Date: Fri, 10 Jul 2026 16:53:23 +0200 Subject: [PATCH 2/6] fix(reporter): preserve dependency project order Dropped the reverse() so setup/teardown projects keep the same order in the root suite as before, avoiding a spurious UI mode filter reorder. --- packages/playwright/src/runner/loadUtils.ts | 2 +- tests/playwright-test/reporter-preprocess-suite.spec.ts | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/packages/playwright/src/runner/loadUtils.ts b/packages/playwright/src/runner/loadUtils.ts index ed5b24acc0280..444aef454d67a 100644 --- a/packages/playwright/src/runner/loadUtils.ts +++ b/packages/playwright/src/runner/loadUtils.ts @@ -166,7 +166,7 @@ export async function createRootSuite(testRun: TestRun, errors: TestError[], sho } // Prepend dependency projects without filtration. - for (const [project, type] of [...projectClosure].reverse()) { + for (const [project, type] of projectClosure) { if (type !== 'dependency') continue; const dependencySuite = buildProjectSuite(project, projectSuites.get(project)!); diff --git a/tests/playwright-test/reporter-preprocess-suite.spec.ts b/tests/playwright-test/reporter-preprocess-suite.spec.ts index 52a43acb3ee77..1f1b5d2b7a1ef 100644 --- a/tests/playwright-test/reporter-preprocess-suite.spec.ts +++ b/tests/playwright-test/reporter-preprocess-suite.spec.ts @@ -448,8 +448,8 @@ test('plan.suite exposes setup/teardown dependency projects but they are read-on expect(result.exitCode).toBe(0); expect(result.outputLines).toEqual([ - 'plan projects: setup,teardown,main', - 'plan tests: setup-test,teardown-test,main-test', + 'plan projects: teardown,setup,main', + 'plan tests: teardown-test,setup-test,main-test', 'dep-skip: TestCase.skip() cannot be called on a setup or teardown project test; these always run in full.', 'dep-fixme: TestCase.fixme() cannot be called on a setup or teardown project test; these always run in full.', 'dep-fail: TestCase.fail() cannot be called on a setup or teardown project test; these always run in full.', From 8a4b6f6042b4b47aa8c2cd9554fa69ef89636fd6 Mon Sep 17 00:00:00 2001 From: Simon Knott Date: Mon, 13 Jul 2026 14:20:16 +0200 Subject: [PATCH 3/6] chore(reporter): unify preprocess suite modes Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: af814592-e5ae-4edb-a8ef-dc73c2882097 --- packages/playwright/src/common/test.ts | 35 ++++++++----------- .../src/reporters/internalReporter.ts | 4 +-- packages/playwright/src/runner/loadUtils.ts | 14 ++++---- 3 files changed, 25 insertions(+), 28 deletions(-) diff --git a/packages/playwright/src/common/test.ts b/packages/playwright/src/common/test.ts index c5ba38a2d7b25..598a041d8a81e 100644 --- a/packages/playwright/src/common/test.ts +++ b/packages/playwright/src/common/test.ts @@ -58,8 +58,7 @@ export class Suite extends Base { _parallelMode: 'none' | 'default' | 'serial' | 'parallel' = 'none'; _fullProject: FullProjectInternal | undefined; _fileId: string | undefined; - _preprocessing = false; - _isDependency = false; + _preprocessMode: 'editable' | 'readonly' | undefined = undefined; readonly _type: 'root' | 'project' | 'file' | 'describe'; skip: (reason?: string) => void; @@ -271,30 +270,28 @@ export class Suite extends Base { } private _modifier(type: 'skip' | 'fixme' | 'fail', location: Location, reason: string | undefined): void { - if (!this._rootSuite()._preprocessing) + const mode = this._resolvePreprocessMode(); + if (!mode) throw new Error(`Suite.${type}() can only be called from Reporter.preprocessSuite().`); - if (this._isInDependencyProject()) + if (mode === 'readonly') throw new Error(`Suite.${type}() cannot be called on a setup or teardown project; these always run in full.`); for (const test of this.allTests()) test._applyPlanAnnotation({ type, description: reason, location }); } exclude(): void { - if (!this._rootSuite()._preprocessing) + const mode = this._resolvePreprocessMode(); + if (!mode) throw new Error(`Suite.exclude() can only be called from Reporter.preprocessSuite().`); if (!this.parent) throw new Error(`Suite.exclude() cannot be called on the root suite.`); - if (this._isInDependencyProject()) + if (mode === 'readonly') throw new Error(`Suite.exclude() cannot be called on a setup or teardown project; these always run in full.`); this.parent._detach(this); } - _isInDependencyProject(): boolean { - return this._isDependency || (this.parent?._isInDependencyProject() ?? false); - } - - _rootSuite(): Suite { - return this.parent?._rootSuite() ?? this; + _resolvePreprocessMode(): 'editable' | 'readonly' | undefined { + return this._preprocessMode ?? this.parent?._resolvePreprocessMode(); } } @@ -362,9 +359,10 @@ export class TestCase extends Base implements reporterTypes.TestCase { } private _modifier(type: 'skip' | 'fixme' | 'fail', location: Location, reason: string | undefined): void { - if (!this._rootSuite()._preprocessing) + const mode = this.parent._resolvePreprocessMode(); + if (!mode) throw new Error(`TestCase.${type}() can only be called from Reporter.preprocessSuite().`); - if (this.parent._isInDependencyProject()) + if (mode === 'readonly') throw new Error(`TestCase.${type}() cannot be called on a setup or teardown project test; these always run in full.`); this._applyPlanAnnotation({ type, description: reason, location }); } @@ -379,17 +377,14 @@ export class TestCase extends Base implements reporterTypes.TestCase { } exclude(): void { - if (!this._rootSuite()._preprocessing) + const mode = this.parent._resolvePreprocessMode(); + if (!mode) throw new Error(`TestCase.exclude() can only be called from Reporter.preprocessSuite().`); - if (this.parent._isInDependencyProject()) + if (mode === 'readonly') throw new Error(`TestCase.exclude() cannot be called on a setup or teardown project test; these always run in full.`); this.parent._detach(this); } - _rootSuite(): Suite { - return this.parent._rootSuite(); - } - _serialize(): any { return { kind: 'test', diff --git a/packages/playwright/src/reporters/internalReporter.ts b/packages/playwright/src/reporters/internalReporter.ts index 7766ec11f6f8b..3070aa64132de 100644 --- a/packages/playwright/src/reporters/internalReporter.ts +++ b/packages/playwright/src/reporters/internalReporter.ts @@ -55,11 +55,11 @@ export class InternalReporter implements ReporterV2 { } async preprocessSuite(config: FullConfig, suite: testNs.Suite) { - suite._preprocessing = true; + suite._preprocessMode = 'editable'; try { return await this._reporter.preprocessSuite?.(config, suite); } finally { - suite._preprocessing = false; + suite._preprocessMode = undefined; } } diff --git a/packages/playwright/src/runner/loadUtils.ts b/packages/playwright/src/runner/loadUtils.ts index 444aef454d67a..ead16f6a7ebc5 100644 --- a/packages/playwright/src/runner/loadUtils.ts +++ b/packages/playwright/src/runner/loadUtils.ts @@ -166,11 +166,13 @@ export async function createRootSuite(testRun: TestRun, errors: TestError[], sho } // Prepend dependency projects without filtration. + const dependencyProjectIds = new Set(); for (const [project, type] of projectClosure) { if (type !== 'dependency') continue; const dependencySuite = buildProjectSuite(project, projectSuites.get(project)!); - dependencySuite._isDependency = true; + dependencySuite._preprocessMode = 'readonly'; + dependencyProjectIds.add(project.id); rootSuite._prependSuite(dependencySuite); } @@ -180,7 +182,7 @@ export async function createRootSuite(testRun: TestRun, errors: TestError[], sho // Create test groups for top-level projects. const testGroups: TestGroup[] = []; for (const projectSuite of rootSuite.suites) { - if (projectSuite._isDependency) + if (projectClosure.get(projectSuite._fullProject!) === 'dependency') continue; // Split beforeAll-grouped tests into "config.shard.total" groups when needed. // Later on, we'll re-split them between workers by using "config.workers" instead. @@ -197,18 +199,18 @@ export async function createRootSuite(testRun: TestRun, errors: TestError[], sho } // Update project suites, removing empty ones. - suiteUtils.filterTestsRemoveEmptySuites(rootSuite, test => test.parent._isInDependencyProject() || testsInThisShard.has(test)); + suiteUtils.filterTestsRemoveEmptySuites(rootSuite, test => dependencyProjectIds.has(test._projectId) || testsInThisShard.has(test)); } if (testRun.postShardTestFilters.length) - suiteUtils.filterTestsRemoveEmptySuites(rootSuite, test => test.parent._isInDependencyProject() || testRun.postShardTestFilters.every(filter => filter(test))); + suiteUtils.filterTestsRemoveEmptySuites(rootSuite, test => dependencyProjectIds.has(test._projectId) || testRun.postShardTestFilters.every(filter => filter(test))); // Filtering 'only' and sharding might have reduced the number of top-level projects. // Prune the project closure to only include dependencies that are still needed. - const topLevelProjects = rootSuite.suites.filter(suite => !suite._isDependency).map(suite => suite._fullProject!); + const topLevelProjects = rootSuite.suites.filter(suite => !dependencyProjectIds.has(suite._fullProject!.id)).map(suite => suite._fullProject!); const neededClosure = buildProjectsClosure(topLevelProjects); for (const projectSuite of [...rootSuite.suites]) { - if (projectSuite._isDependency && !neededClosure.has(projectSuite._fullProject!)) + if (dependencyProjectIds.has(projectSuite._fullProject!.id) && !neededClosure.has(projectSuite._fullProject!)) rootSuite._detach(projectSuite); } From a04af33018edb0a3ec29204e0051398fc510151b Mon Sep 17 00:00:00 2001 From: Simon Knott Date: Tue, 14 Jul 2026 07:57:59 +0200 Subject: [PATCH 4/6] fix(reporter): preserve selected dependency projects Copilot-Session: af814592-e5ae-4edb-a8ef-dc73c2882097 --- .../src/reporters/internalReporter.ts | 7 +- packages/playwright/src/runner/loadUtils.ts | 27 ++++- .../reporter-preprocess-suite.spec.ts | 113 ++++++++++++++++++ 3 files changed, 138 insertions(+), 9 deletions(-) diff --git a/packages/playwright/src/reporters/internalReporter.ts b/packages/playwright/src/reporters/internalReporter.ts index 3070aa64132de..f4f092cd5d9ea 100644 --- a/packages/playwright/src/reporters/internalReporter.ts +++ b/packages/playwright/src/reporters/internalReporter.ts @@ -55,12 +55,7 @@ export class InternalReporter implements ReporterV2 { } async preprocessSuite(config: FullConfig, suite: testNs.Suite) { - suite._preprocessMode = 'editable'; - try { - return await this._reporter.preprocessSuite?.(config, suite); - } finally { - suite._preprocessMode = undefined; - } + return await this._reporter.preprocessSuite?.(config, suite); } onBegin(suite: testNs.Suite) { diff --git a/packages/playwright/src/runner/loadUtils.ts b/packages/playwright/src/runner/loadUtils.ts index ead16f6a7ebc5..472d25e4b21bb 100644 --- a/packages/playwright/src/runner/loadUtils.ts +++ b/packages/playwright/src/runner/loadUtils.ts @@ -135,11 +135,13 @@ export async function createRootSuite(testRun: TestRun, errors: TestError[], sho } } + let hasOnly = false; if (shouldFilterOnly) { // Create a fake root to execute the exclusive semantics across the projects. const filteredRoot = new testNs.Suite('', 'root'); for (const filteredProjectSuite of filteredProjectSuites.values()) filteredRoot._addSuite(filteredProjectSuite); + hasOnly = !!filteredRoot._getOnlyItems().length; suiteUtils.filterOnly(filteredRoot); for (const [project, filteredProjectSuite] of filteredProjectSuites) { if (!filteredRoot.suites.includes(filteredProjectSuite)) @@ -171,12 +173,21 @@ export async function createRootSuite(testRun: TestRun, errors: TestError[], sho if (type !== 'dependency') continue; const dependencySuite = buildProjectSuite(project, projectSuites.get(project)!); - dependencySuite._preprocessMode = 'readonly'; dependencyProjectIds.add(project.id); rootSuite._prependSuite(dependencySuite); } - const preprocessResult = await testRun.reporter.preprocessSuite(config.config, rootSuite); + const preprocessSuites = [rootSuite, ...rootSuite.suites]; + for (const suite of preprocessSuites) + suite._preprocessMode = suite === rootSuite || projectClosure.get(suite._fullProject!) === 'top-level' ? 'editable' : 'readonly'; + let preprocessResult: Awaited>; + try { + preprocessResult = await testRun.reporter.preprocessSuite(config.config, rootSuite); + } finally { + for (const suite of preprocessSuites) + suite._preprocessMode = undefined; + } + // Shard only the top-level projects. if (config.config.shard && !preprocessResult?.implementsSharding) { // Create test groups for top-level projects. @@ -207,7 +218,17 @@ export async function createRootSuite(testRun: TestRun, errors: TestError[], sho // Filtering 'only' and sharding might have reduced the number of top-level projects. // Prune the project closure to only include dependencies that are still needed. - const topLevelProjects = rootSuite.suites.filter(suite => !dependencyProjectIds.has(suite._fullProject!.id)).map(suite => suite._fullProject!); + const filteredProjects = new Set(testRun.filteredProjects); + const hasExplicitSelection = !!testRun.options.projectFilter?.length || !!testRun.preOnlyTestFilters.length || !!testRun.postShardTestFilters.length || hasOnly; + const isExplicitlySelected = (projectSuite: testNs.Suite) => { + const project = projectSuite._fullProject!; + if (!hasExplicitSelection || (testRun.options.projectFilter?.length && !filteredProjects.has(project))) + return false; + return !!filteredProjectSuites.get(project)?._hasTests() && + (!testRun.postShardTestFilters.length || projectSuite.allTests().some(test => testRun.postShardTestFilters.every(filter => filter(test)))); + }; + const topLevelProjects = rootSuite.suites.filter(suite => !dependencyProjectIds.has(suite._fullProject!.id) || + isExplicitlySelected(suite)).map(suite => suite._fullProject!); const neededClosure = buildProjectsClosure(topLevelProjects); for (const projectSuite of [...rootSuite.suites]) { if (dependencyProjectIds.has(projectSuite._fullProject!.id) && !neededClosure.has(projectSuite._fullProject!)) diff --git a/tests/playwright-test/reporter-preprocess-suite.spec.ts b/tests/playwright-test/reporter-preprocess-suite.spec.ts index 1f1b5d2b7a1ef..bce02858757fc 100644 --- a/tests/playwright-test/reporter-preprocess-suite.spec.ts +++ b/tests/playwright-test/reporter-preprocess-suite.spec.ts @@ -416,6 +416,15 @@ test('plan.suite exposes setup/teardown dependency projects but they are read-on console.log('%% dep-suite-exclude: ' + e.message); } } + onBegin(config, suite) { + const setupTest = suite.allTests().find(t => t.title === 'setup-test'); + try { + setupTest.skip(); + console.log('%% dep-after-preprocess: no-throw'); + } catch (e) { + console.log('%% dep-after-preprocess: ' + e.message); + } + } onTestEnd(test, result) { console.log('%% ran ' + test.parent.project().name + '/' + test.title); } @@ -455,8 +464,112 @@ test('plan.suite exposes setup/teardown dependency projects but they are read-on 'dep-fail: TestCase.fail() cannot be called on a setup or teardown project test; these always run in full.', 'dep-exclude: TestCase.exclude() cannot be called on a setup or teardown project test; these always run in full.', 'dep-suite-exclude: Suite.exclude() cannot be called on a setup or teardown project; these always run in full.', + 'dep-after-preprocess: TestCase.skip() can only be called from Reporter.preprocessSuite().', 'ran setup/setup-test', 'ran main/main-test', 'ran teardown/teardown-test', ]); }); + +const explicitDependencyVariants: { + name: string, + only: string, + projects?: string[], + additionalArgs?: string[], +}[] = [ + { name: 'file arguments', only: '', additionalArgs: ['setup.spec.ts', 'main.spec.ts'] }, + { name: 'project filters', only: '', projects: ['setup', 'main'] }, + { name: 'test.only', only: '.only' }, +]; + +for (const variant of explicitDependencyVariants) { + test(`plan.suite preserves dependency projects selected through ${variant.name}`, async ({ runInlineTest }) => { + const result = await runInlineTest({ + 'reporter.ts': ` + class Reporter { + async preprocessSuite(config, suite) { + suite.suites.find(suite => suite.title === 'main').exclude(); + } + onTestEnd(test, result) { + console.log('%% ran ' + test.parent.project().name + '/' + test.title); + } + } + module.exports = Reporter; + `, + 'playwright.config.ts': ` + module.exports = { + reporter: './reporter.ts', + projects: [ + { name: 'setup', testMatch: /setup\\.spec\\.ts/ }, + { name: 'main', testMatch: /main\\.spec\\.ts/, dependencies: ['setup'] }, + ], + }; + `, + 'setup.spec.ts': ` + import { test } from '@playwright/test'; + test${variant.only}('setup-test', async () => {}); + `, + 'main.spec.ts': ` + import { test } from '@playwright/test'; + test${variant.only}('main-test', async () => {}); + `, + }, { + reporter: '', + workers: 1, + ...(variant.projects ? { project: variant.projects } : {}), + }, undefined, variant.additionalArgs ? { additionalArgs: variant.additionalArgs } : {}); + + expect(result.exitCode).toBe(0); + expect(result.outputLines).toEqual([ + 'ran setup/setup-test', + ]); + }); +} + +test('plan.suite preserves dependency projects selected through --last-failed', async ({ runInlineTest }) => { + const files = { + 'reporter.ts': ` + class Reporter { + async preprocessSuite(config, suite) { + if (process.env.EXCLUDE_MAIN) + suite.suites.find(suite => suite.title === 'main').exclude(); + } + onTestEnd(test, result) { + console.log('%% ran ' + test.parent.project().name + '/' + test.title); + } + } + module.exports = Reporter; + `, + 'playwright.config.ts': ` + module.exports = { + reporter: './reporter.ts', + projects: [ + { name: 'setup', testMatch: /setup\\.spec\\.ts/ }, + { name: 'main', testMatch: /main\\.spec\\.ts/, dependencies: ['setup'] }, + ], + }; + `, + 'setup.spec.ts': ` + import { test, expect } from '@playwright/test'; + test('setup-test', async () => { + expect(1).toBe(2); + }); + `, + 'main.spec.ts': ` + import { test } from '@playwright/test'; + test('main-test', async () => {}); + `, + }; + + const firstRun = await runInlineTest(files, { reporter: '', workers: 1 }); + expect(firstRun.exitCode).toBe(1); + expect(firstRun.outputLines).toEqual([ + 'ran setup/setup-test', + ]); + + const lastFailedRun = await runInlineTest(files, { reporter: '', workers: 1 }, { EXCLUDE_MAIN: '1' }, { additionalArgs: ['--last-failed'] }); + expect(lastFailedRun.exitCode).toBe(1); + expect(lastFailedRun.outputLines).toEqual([ + 'ran setup/setup-test', + ]); +}); From 5e2b04ff7369105b589aa1912de1fe9f822567f1 Mon Sep 17 00:00:00 2001 From: Simon Knott Date: Tue, 14 Jul 2026 08:05:32 +0200 Subject: [PATCH 5/6] fix(reporter): keep dependency preprocessing additive Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: af814592-e5ae-4edb-a8ef-dc73c2882097 --- packages/playwright/src/runner/loadUtils.ts | 48 +++++----- .../reporter-preprocess-suite.spec.ts | 87 ++++--------------- 2 files changed, 38 insertions(+), 97 deletions(-) diff --git a/packages/playwright/src/runner/loadUtils.ts b/packages/playwright/src/runner/loadUtils.ts index 472d25e4b21bb..3736ba042255c 100644 --- a/packages/playwright/src/runner/loadUtils.ts +++ b/packages/playwright/src/runner/loadUtils.ts @@ -135,13 +135,11 @@ export async function createRootSuite(testRun: TestRun, errors: TestError[], sho } } - let hasOnly = false; if (shouldFilterOnly) { // Create a fake root to execute the exclusive semantics across the projects. const filteredRoot = new testNs.Suite('', 'root'); for (const filteredProjectSuite of filteredProjectSuites.values()) filteredRoot._addSuite(filteredProjectSuite); - hasOnly = !!filteredRoot._getOnlyItems().length; suiteUtils.filterOnly(filteredRoot); for (const [project, filteredProjectSuite] of filteredProjectSuites) { if (!filteredRoot.suites.includes(filteredProjectSuite)) @@ -167,13 +165,13 @@ export async function createRootSuite(testRun: TestRun, errors: TestError[], sho } } - // Prepend dependency projects without filtration. - const dependencyProjectIds = new Set(); + // Temporarily prepend unfiltered dependency projects for preprocessing. + const dependencySuites = new Map(); for (const [project, type] of projectClosure) { if (type !== 'dependency') continue; const dependencySuite = buildProjectSuite(project, projectSuites.get(project)!); - dependencyProjectIds.add(project.id); + dependencySuites.set(project, dependencySuite); rootSuite._prependSuite(dependencySuite); } @@ -188,13 +186,15 @@ export async function createRootSuite(testRun: TestRun, errors: TestError[], sho suite._preprocessMode = undefined; } + // Continue the existing sharding and filtering pipeline with top-level projects only. + for (const dependencySuite of dependencySuites.values()) + rootSuite._detach(dependencySuite); + // Shard only the top-level projects. if (config.config.shard && !preprocessResult?.implementsSharding) { // Create test groups for top-level projects. const testGroups: TestGroup[] = []; for (const projectSuite of rootSuite.suites) { - if (projectClosure.get(projectSuite._fullProject!) === 'dependency') - continue; // Split beforeAll-grouped tests into "config.shard.total" groups when needed. // Later on, we'll re-split them between workers by using "config.workers" instead. for (const group of createTestGroups(projectSuite, config.config.shard.total)) @@ -210,29 +210,23 @@ export async function createRootSuite(testRun: TestRun, errors: TestError[], sho } // Update project suites, removing empty ones. - suiteUtils.filterTestsRemoveEmptySuites(rootSuite, test => dependencyProjectIds.has(test._projectId) || testsInThisShard.has(test)); + suiteUtils.filterTestsRemoveEmptySuites(rootSuite, test => testsInThisShard.has(test)); } if (testRun.postShardTestFilters.length) - suiteUtils.filterTestsRemoveEmptySuites(rootSuite, test => dependencyProjectIds.has(test._projectId) || testRun.postShardTestFilters.every(filter => filter(test))); - - // Filtering 'only' and sharding might have reduced the number of top-level projects. - // Prune the project closure to only include dependencies that are still needed. - const filteredProjects = new Set(testRun.filteredProjects); - const hasExplicitSelection = !!testRun.options.projectFilter?.length || !!testRun.preOnlyTestFilters.length || !!testRun.postShardTestFilters.length || hasOnly; - const isExplicitlySelected = (projectSuite: testNs.Suite) => { - const project = projectSuite._fullProject!; - if (!hasExplicitSelection || (testRun.options.projectFilter?.length && !filteredProjects.has(project))) - return false; - return !!filteredProjectSuites.get(project)?._hasTests() && - (!testRun.postShardTestFilters.length || projectSuite.allTests().some(test => testRun.postShardTestFilters.every(filter => filter(test)))); - }; - const topLevelProjects = rootSuite.suites.filter(suite => !dependencyProjectIds.has(suite._fullProject!.id) || - isExplicitlySelected(suite)).map(suite => suite._fullProject!); - const neededClosure = buildProjectsClosure(topLevelProjects); - for (const projectSuite of [...rootSuite.suites]) { - if (dependencyProjectIds.has(projectSuite._fullProject!.id) && !neededClosure.has(projectSuite._fullProject!)) - rootSuite._detach(projectSuite); + suiteUtils.filterTestsRemoveEmptySuites(rootSuite, test => testRun.postShardTestFilters.every(filter => filter(test))); + + const topLevelProjects = []; + { + // Filtering 'only' and sharding might have reduced the number of top-level projects. + // Build the project closure to only include dependencies that are still needed. + const finalProjectClosure = buildProjectsClosure(rootSuite.suites.map(suite => suite._fullProject!)); + for (const [project, type] of finalProjectClosure) { + if (type === 'dependency') + rootSuite._prependSuite(dependencySuites.get(project)!); + else + topLevelProjects.push(project); + } } testRun.rootSuite = rootSuite; diff --git a/tests/playwright-test/reporter-preprocess-suite.spec.ts b/tests/playwright-test/reporter-preprocess-suite.spec.ts index bce02858757fc..d9139a34c518d 100644 --- a/tests/playwright-test/reporter-preprocess-suite.spec.ts +++ b/tests/playwright-test/reporter-preprocess-suite.spec.ts @@ -399,6 +399,7 @@ test('plan.suite exposes setup/teardown dependency projects but they are read-on async preprocessSuite(config, suite) { console.log('%% plan projects: ' + suite.suites.map(s => s.title).join(',')); console.log('%% plan tests: ' + suite.allTests().map(t => t.title).join(',')); + this.preprocessedTests = new Set(suite.allTests()); const setupTest = suite.allTests().find(t => t.title === 'setup-test'); for (const method of ['skip', 'fixme', 'fail', 'exclude']) { try { @@ -417,6 +418,7 @@ test('plan.suite exposes setup/teardown dependency projects but they are read-on } } onBegin(config, suite) { + console.log('%% same test objects: ' + suite.allTests().every(test => this.preprocessedTests.has(test))); const setupTest = suite.allTests().find(t => t.title === 'setup-test'); try { setupTest.skip(); @@ -464,6 +466,7 @@ test('plan.suite exposes setup/teardown dependency projects but they are read-on 'dep-fail: TestCase.fail() cannot be called on a setup or teardown project test; these always run in full.', 'dep-exclude: TestCase.exclude() cannot be called on a setup or teardown project test; these always run in full.', 'dep-suite-exclude: Suite.exclude() cannot be called on a setup or teardown project; these always run in full.', + 'same test objects: true', 'dep-after-preprocess: TestCase.skip() can only be called from Reporter.preprocessSuite().', 'ran setup/setup-test', 'ran main/main-test', @@ -471,68 +474,13 @@ test('plan.suite exposes setup/teardown dependency projects but they are read-on ]); }); -const explicitDependencyVariants: { - name: string, - only: string, - projects?: string[], - additionalArgs?: string[], -}[] = [ - { name: 'file arguments', only: '', additionalArgs: ['setup.spec.ts', 'main.spec.ts'] }, - { name: 'project filters', only: '', projects: ['setup', 'main'] }, - { name: 'test.only', only: '.only' }, -]; - -for (const variant of explicitDependencyVariants) { - test(`plan.suite preserves dependency projects selected through ${variant.name}`, async ({ runInlineTest }) => { - const result = await runInlineTest({ - 'reporter.ts': ` - class Reporter { - async preprocessSuite(config, suite) { - suite.suites.find(suite => suite.title === 'main').exclude(); - } - onTestEnd(test, result) { - console.log('%% ran ' + test.parent.project().name + '/' + test.title); - } - } - module.exports = Reporter; - `, - 'playwright.config.ts': ` - module.exports = { - reporter: './reporter.ts', - projects: [ - { name: 'setup', testMatch: /setup\\.spec\\.ts/ }, - { name: 'main', testMatch: /main\\.spec\\.ts/, dependencies: ['setup'] }, - ], - }; - `, - 'setup.spec.ts': ` - import { test } from '@playwright/test'; - test${variant.only}('setup-test', async () => {}); - `, - 'main.spec.ts': ` - import { test } from '@playwright/test'; - test${variant.only}('main-test', async () => {}); - `, - }, { - reporter: '', - workers: 1, - ...(variant.projects ? { project: variant.projects } : {}), - }, undefined, variant.additionalArgs ? { additionalArgs: variant.additionalArgs } : {}); - - expect(result.exitCode).toBe(0); - expect(result.outputLines).toEqual([ - 'ran setup/setup-test', - ]); - }); -} - -test('plan.suite preserves dependency projects selected through --last-failed', async ({ runInlineTest }) => { - const files = { +test('plan.suite temporarily exposes dependencies without changing final project selection', async ({ runInlineTest }) => { + const result = await runInlineTest({ 'reporter.ts': ` class Reporter { async preprocessSuite(config, suite) { - if (process.env.EXCLUDE_MAIN) - suite.suites.find(suite => suite.title === 'main').exclude(); + console.log('%% plan projects: ' + suite.suites.map(suite => suite.title).join(',')); + suite.suites.find(suite => suite.title === 'main').exclude(); } onTestEnd(test, result) { console.log('%% ran ' + test.parent.project().name + '/' + test.title); @@ -546,6 +494,7 @@ test('plan.suite preserves dependency projects selected through --last-failed', projects: [ { name: 'setup', testMatch: /setup\\.spec\\.ts/ }, { name: 'main', testMatch: /main\\.spec\\.ts/, dependencies: ['setup'] }, + { name: 'keep', testMatch: /keep\\.spec\\.ts/ }, ], }; `, @@ -559,17 +508,15 @@ test('plan.suite preserves dependency projects selected through --last-failed', import { test } from '@playwright/test'; test('main-test', async () => {}); `, - }; - - const firstRun = await runInlineTest(files, { reporter: '', workers: 1 }); - expect(firstRun.exitCode).toBe(1); - expect(firstRun.outputLines).toEqual([ - 'ran setup/setup-test', - ]); + 'keep.spec.ts': ` + import { test } from '@playwright/test'; + test('keep-test', async () => {}); + `, + }, { reporter: '', workers: 1 }, undefined, { additionalArgs: ['setup.spec.ts', 'main.spec.ts', 'keep.spec.ts'] }); - const lastFailedRun = await runInlineTest(files, { reporter: '', workers: 1 }, { EXCLUDE_MAIN: '1' }, { additionalArgs: ['--last-failed'] }); - expect(lastFailedRun.exitCode).toBe(1); - expect(lastFailedRun.outputLines).toEqual([ - 'ran setup/setup-test', + expect(result.exitCode).toBe(0); + expect(result.outputLines).toEqual([ + 'plan projects: setup,main,keep', + 'ran keep/keep-test', ]); }); From 8c413d81335a89231c8f46936b6d7389883b22d5 Mon Sep 17 00:00:00 2001 From: Simon Knott Date: Tue, 14 Jul 2026 08:23:45 +0200 Subject: [PATCH 6/6] chore(reporter): simplify preprocess suite modes Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: af814592-e5ae-4edb-a8ef-dc73c2882097 --- packages/playwright/src/runner/loadUtils.ts | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/packages/playwright/src/runner/loadUtils.ts b/packages/playwright/src/runner/loadUtils.ts index 3736ba042255c..9878b14702c8b 100644 --- a/packages/playwright/src/runner/loadUtils.ts +++ b/packages/playwright/src/runner/loadUtils.ts @@ -171,25 +171,24 @@ export async function createRootSuite(testRun: TestRun, errors: TestError[], sho if (type !== 'dependency') continue; const dependencySuite = buildProjectSuite(project, projectSuites.get(project)!); + dependencySuite._preprocessMode = 'readonly'; dependencySuites.set(project, dependencySuite); rootSuite._prependSuite(dependencySuite); } - const preprocessSuites = [rootSuite, ...rootSuite.suites]; - for (const suite of preprocessSuites) - suite._preprocessMode = suite === rootSuite || projectClosure.get(suite._fullProject!) === 'top-level' ? 'editable' : 'readonly'; + rootSuite._preprocessMode = 'editable'; let preprocessResult: Awaited>; try { preprocessResult = await testRun.reporter.preprocessSuite(config.config, rootSuite); } finally { - for (const suite of preprocessSuites) - suite._preprocessMode = undefined; + // Continue the existing sharding and filtering pipeline with top-level projects only. + rootSuite._preprocessMode = undefined; + for (const dependencySuite of dependencySuites.values()) { + dependencySuite._preprocessMode = undefined; + rootSuite._detach(dependencySuite); + } } - // Continue the existing sharding and filtering pipeline with top-level projects only. - for (const dependencySuite of dependencySuites.values()) - rootSuite._detach(dependencySuite); - // Shard only the top-level projects. if (config.config.shard && !preprocessResult?.implementsSharding) { // Create test groups for top-level projects.