fix(reporter): expose setup/teardown projects to preprocessSuite

docs/src/test-reporter-api/class-reporter.md

@@ -317,6 +317,6 @@ Resolved configuration.
 
 The root suite that contains the projects, files and test cases that will run.
 
-The suite reflects `--project`, `--grep`/`--grep-invert` and `.only` filtering, so it only contains tests that match the current invocation. It contains only the top-level projects being run — setup and dependency projects are not included and cannot be excluded from here.
+The suite reflects `--project`, `--grep`/`--grep-invert` and `.only` filtering, so it only contains tests that match the current invocation. Setup and dependency projects are readonly and cannot be excluded from here.
 
 The suite ignores the `--shard` argument: it always contains the full, un-sharded corpus. Playwright applies its built-in sharding after [`method: Reporter.preprocessSuite`] returns, unless the returned `implementsSharding` is `true`.

packages/playwright/src/common/test.ts

@@ -58,7 +58,7 @@ export class Suite extends Base {
   _parallelMode: 'none' | 'default' | 'serial' | 'parallel' = 'none';
   _fullProject: FullProjectInternal | undefined;
   _fileId: string | undefined;
-  _preprocessing = false;
+  _preprocessMode: 'editable' | 'readonly' | undefined = undefined;
   readonly _type: 'root' | 'project' | 'file' | 'describe';
 
   skip: (reason?: string) => void;
@@ -270,22 +270,28 @@ export class Suite extends Base {
   }
 
   private _modifier(type: 'skip' | 'fixme' | 'fail', location: Location, reason: string | undefined): void {
-    if (!this._rootSuite()._preprocessing)
+    const mode = this._resolvePreprocessMode();
+    if (!mode)
       throw new Error(`Suite.${type}() can only be called from Reporter.preprocessSuite().`);
+    if (mode === 'readonly')
+      throw new Error(`Suite.${type}() cannot be called on a setup or teardown project; these always run in full.`);
     for (const test of this.allTests())
       test._applyPlanAnnotation({ type, description: reason, location });
   }
 
   exclude(): void {
-    if (!this._rootSuite()._preprocessing)
+    const mode = this._resolvePreprocessMode();
+    if (!mode)
       throw new Error(`Suite.exclude() can only be called from Reporter.preprocessSuite().`);
     if (!this.parent)
       throw new Error(`Suite.exclude() cannot be called on the root suite.`);
+    if (mode === 'readonly')
+      throw new Error(`Suite.exclude() cannot be called on a setup or teardown project; these always run in full.`);
     this.parent._detach(this);
   }
 
-  _rootSuite(): Suite {
-    return this.parent?._rootSuite() ?? this;
+  _resolvePreprocessMode(): 'editable' | 'readonly' | undefined {
+    return this._preprocessMode ?? this.parent?._resolvePreprocessMode();
   }
 }
 
@@ -353,8 +359,11 @@ export class TestCase extends Base implements reporterTypes.TestCase {
   }
 
   private _modifier(type: 'skip' | 'fixme' | 'fail', location: Location, reason: string | undefined): void {
-    if (!this._rootSuite()._preprocessing)
+    const mode = this.parent._resolvePreprocessMode();
+    if (!mode)
       throw new Error(`TestCase.${type}() can only be called from Reporter.preprocessSuite().`);
+    if (mode === 'readonly')
+      throw new Error(`TestCase.${type}() cannot be called on a setup or teardown project test; these always run in full.`);
     this._applyPlanAnnotation({ type, description: reason, location });
   }
 
@@ -368,15 +377,14 @@ export class TestCase extends Base implements reporterTypes.TestCase {
   }
 
   exclude(): void {
-    if (!this._rootSuite()._preprocessing)
+    const mode = this.parent._resolvePreprocessMode();
+    if (!mode)
       throw new Error(`TestCase.exclude() can only be called from Reporter.preprocessSuite().`);
+    if (mode === 'readonly')
+      throw new Error(`TestCase.exclude() cannot be called on a setup or teardown project test; these always run in full.`);
     this.parent._detach(this);
   }
 
-  _rootSuite(): Suite {
-    return this.parent._rootSuite();
-  }
-
   _serialize(): any {
     return {
       kind: 'test',

packages/playwright/src/reporters/internalReporter.ts

@@ -55,12 +55,7 @@ export class InternalReporter implements ReporterV2 {
   }
 
   async preprocessSuite(config: FullConfig, suite: testNs.Suite) {
-    suite._preprocessing = true;
-    try {
-      return await this._reporter.preprocessSuite?.(config, suite);
-    } finally {
-      suite._preprocessing = false;
-    }
+    return await this._reporter.preprocessSuite?.(config, suite);
   }
 
   onBegin(suite: testNs.Suite) {

packages/playwright/src/runner/loadUtils.ts

@@ -165,7 +165,30 @@ export async function createRootSuite(testRun: TestRun, errors: TestError[], sho
     }
   }
 
-  const preprocessResult = await testRun.reporter.preprocessSuite(config.config, rootSuite);
+  // Temporarily prepend unfiltered dependency projects for preprocessing.
+  const dependencySuites = new Map<commonConfig.FullProjectInternal, testNs.Suite>();
+  for (const [project, type] of projectClosure) {
+    if (type !== 'dependency')
+      continue;
+    const dependencySuite = buildProjectSuite(project, projectSuites.get(project)!);
+    dependencySuite._preprocessMode = 'readonly';
+    dependencySuites.set(project, dependencySuite);
+    rootSuite._prependSuite(dependencySuite);
+  }
+
+  rootSuite._preprocessMode = 'editable';
+  let preprocessResult: Awaited<ReturnType<typeof testRun.reporter.preprocessSuite>>;
+  try {
+    preprocessResult = await testRun.reporter.preprocessSuite(config.config, rootSuite);
+  } finally {
+    // Continue the existing sharding and filtering pipeline with top-level projects only.
+    rootSuite._preprocessMode = undefined;
+    for (const dependencySuite of dependencySuites.values()) {
+      dependencySuite._preprocessMode = undefined;
+      rootSuite._detach(dependencySuite);
+    }
+  }
+
   // Shard only the top-level projects.
   if (config.config.shard && !preprocessResult?.implementsSharding) {
     // Create test groups for top-level projects.
@@ -193,16 +216,13 @@ export async function createRootSuite(testRun: TestRun, errors: TestError[], sho
     suiteUtils.filterTestsRemoveEmptySuites(rootSuite, test => testRun.postShardTestFilters.every(filter => filter(test)));
 
   const topLevelProjects = [];
-  // Now prepend dependency projects without filtration.
   {
     // Filtering 'only' and sharding might have reduced the number of top-level projects.
     // Build the project closure to only include dependencies that are still needed.
-    const projectClosure = new Map(buildProjectsClosure(rootSuite.suites.map(suite => suite._fullProject!)));
-
-    // Clone file suites for dependency projects.
-    for (const [project, level] of projectClosure.entries()) {
-      if (level === 'dependency')
-        rootSuite._prependSuite(buildProjectSuite(project, projectSuites.get(project)!));
+    const finalProjectClosure = buildProjectsClosure(rootSuite.suites.map(suite => suite._fullProject!));
+    for (const [project, type] of finalProjectClosure) {
+      if (type === 'dependency')
+        rootSuite._prependSuite(dependencySuites.get(project)!);
       else
         topLevelProjects.push(project);
     }

packages/playwright/types/testReporter.d.ts

@@ -153,8 +153,7 @@ export interface Reporter {
    * @param suite The root suite that contains the projects, files and test cases that will run.
    *
    * The suite reflects `--project`, `--grep`/`--grep-invert` and `.only` filtering, so it only contains tests that
-   * match the current invocation. It contains only the top-level projects being run — setup and dependency projects are
-   * not included and cannot be excluded from here.
+   * match the current invocation. Setup and dependency projects are readonly and cannot be excluded from here.
    *
    * The suite ignores the `--shard` argument: it always contains the full, un-sharded corpus. Playwright applies its
    * built-in sharding after

tests/playwright-test/reporter-preprocess-suite.spec.ts

@@ -392,15 +392,40 @@ test('multiple reporters declaring implementsSharding throws', async ({ runInlin
   expect(result.outputLines.join('\n')).toContain(`Multiple reporters declare 'implementsSharding'`);
 });
 
-test('plan.suite contains only top-level projects, not dependency/setup projects', async ({ runInlineTest }) => {
+test('plan.suite exposes setup/teardown dependency projects but they are read-only', async ({ runInlineTest }) => {
   const result = await runInlineTest({
     'reporter.ts': `
       class Reporter {
         async preprocessSuite(config, suite) {
-          // The suite only exposes top-level projects, so a reporter has no handle on
-          // setup/dependency project tests and therefore cannot exclude them.
           console.log('%% plan projects: ' + suite.suites.map(s => s.title).join(','));
           console.log('%% plan tests: ' + suite.allTests().map(t => t.title).join(','));
+          this.preprocessedTests = new Set(suite.allTests());
+          const setupTest = suite.allTests().find(t => t.title === 'setup-test');
+          for (const method of ['skip', 'fixme', 'fail', 'exclude']) {
+            try {
+              setupTest[method]();
+              console.log('%% dep-' + method + ': no-throw');
+            } catch (e) {
+              console.log('%% dep-' + method + ': ' + e.message);
+            }
+          }
+          const setupProject = suite.suites.find(s => s.title === 'setup');
+          try {
+            setupProject.exclude();
+            console.log('%% dep-suite-exclude: no-throw');
+          } catch (e) {
+            console.log('%% dep-suite-exclude: ' + e.message);
+          }
+        }
+        onBegin(config, suite) {
+          console.log('%% same test objects: ' + suite.allTests().every(test => this.preprocessedTests.has(test)));
+          const setupTest = suite.allTests().find(t => t.title === 'setup-test');
+          try {
+            setupTest.skip();
+            console.log('%% dep-after-preprocess: no-throw');
+          } catch (e) {
+            console.log('%% dep-after-preprocess: ' + e.message);
+          }
         }
         onTestEnd(test, result) {
           console.log('%% ran ' + test.parent.project().name + '/' + test.title);
@@ -412,7 +437,8 @@ test('plan.suite contains only top-level projects, not dependency/setup projects
       module.exports = {
         reporter: './reporter.ts',
         projects: [
-          { name: 'setup', testMatch: /a\\.setup\\.ts/ },
+          { name: 'setup', testMatch: /a\\.setup\\.ts/, teardown: 'teardown' },
+          { name: 'teardown', testMatch: /a\\.teardown\\.ts/ },
           { name: 'main', testMatch: /a\\.test\\.ts/, dependencies: ['setup'] },
         ],
       };
@@ -421,18 +447,76 @@ test('plan.suite contains only top-level projects, not dependency/setup projects
       import { test } from '@playwright/test';
       test('setup-test', async () => {});
     `,
+    'a.teardown.ts': `
+      import { test } from '@playwright/test';
+      test('teardown-test', async () => {});
+    `,
     'a.test.ts': `
       import { test } from '@playwright/test';
       test('main-test', async () => {});
     `,
   }, { reporter: '', workers: 1 }, undefined, { additionalArgs: ['--project=main'] });
 
   expect(result.exitCode).toBe(0);
-  // plan only sees the top-level 'main' project; the 'setup' dependency is prepended afterwards.
-  expect(result.outputLines).toContain('plan projects: main');
-  // 'setup-test' is absent from the plan suite, proving setup/dependency tests are not exposed.
-  expect(result.outputLines).toContain('plan tests: main-test');
-  // Both the dependency and the main project still run.
-  expect(result.outputLines).toContain('ran setup/setup-test');
-  expect(result.outputLines).toContain('ran main/main-test');
+  expect(result.outputLines).toEqual([
+    'plan projects: teardown,setup,main',
+    'plan tests: teardown-test,setup-test,main-test',
+    'dep-skip: TestCase.skip() cannot be called on a setup or teardown project test; these always run in full.',
+    'dep-fixme: TestCase.fixme() cannot be called on a setup or teardown project test; these always run in full.',
+    'dep-fail: TestCase.fail() cannot be called on a setup or teardown project test; these always run in full.',
+    'dep-exclude: TestCase.exclude() cannot be called on a setup or teardown project test; these always run in full.',
+    'dep-suite-exclude: Suite.exclude() cannot be called on a setup or teardown project; these always run in full.',
+    'same test objects: true',
+    'dep-after-preprocess: TestCase.skip() can only be called from Reporter.preprocessSuite().',
+    'ran setup/setup-test',
+    'ran main/main-test',
+    'ran teardown/teardown-test',
+  ]);
+});
+
+test('plan.suite temporarily exposes dependencies without changing final project selection', async ({ runInlineTest }) => {
+  const result = await runInlineTest({
+    'reporter.ts': `
+      class Reporter {
+        async preprocessSuite(config, suite) {
+          console.log('%% plan projects: ' + suite.suites.map(suite => suite.title).join(','));
+          suite.suites.find(suite => suite.title === 'main').exclude();
+        }
+        onTestEnd(test, result) {
+          console.log('%% ran ' + test.parent.project().name + '/' + test.title);
+        }
+      }
+      module.exports = Reporter;
+    `,
+    'playwright.config.ts': `
+      module.exports = {
+        reporter: './reporter.ts',
+        projects: [
+          { name: 'setup', testMatch: /setup\\.spec\\.ts/ },
+          { name: 'main', testMatch: /main\\.spec\\.ts/, dependencies: ['setup'] },
+          { name: 'keep', testMatch: /keep\\.spec\\.ts/ },
+        ],
+      };
+    `,
+    'setup.spec.ts': `
+      import { test, expect } from '@playwright/test';
+      test('setup-test', async () => {
+        expect(1).toBe(2);
+      });
+    `,
+    'main.spec.ts': `
+      import { test } from '@playwright/test';
+      test('main-test', async () => {});
+    `,
+    'keep.spec.ts': `
+      import { test } from '@playwright/test';
+      test('keep-test', async () => {});
+    `,
+  }, { reporter: '', workers: 1 }, undefined, { additionalArgs: ['setup.spec.ts', 'main.spec.ts', 'keep.spec.ts'] });
+
+  expect(result.exitCode).toBe(0);
+  expect(result.outputLines).toEqual([
+    'plan projects: setup,main,keep',
+    'ran keep/keep-test',
+  ]);
 });