From d2792c2d4b7186e6e09f887f0c0a2470c491db4d Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Thu, 22 Oct 2026 13:08:19 -0700 Subject: [PATCH 1/5] ci: balance desktop smoke shards by observed file durations --- .github/workflows/desktop-e2e.yml | 11 +++-- apps/desktop/e2e-durations.json | 20 +++++++++ scripts/desktop-e2e.ts | 70 +++++++++++++++++++++++++++++++ scripts/duration-shards.test.ts | 41 ++++++++++++++++++ scripts/duration-shards.ts | 28 +++++++++++++ 5 files changed, 167 insertions(+), 3 deletions(-) create mode 100644 apps/desktop/e2e-durations.json create mode 100644 scripts/desktop-e2e.ts create mode 100644 scripts/duration-shards.test.ts create mode 100644 scripts/duration-shards.ts diff --git a/.github/workflows/desktop-e2e.yml b/.github/workflows/desktop-e2e.yml index 88faa1918cd..a91e7010363 100644 --- a/.github/workflows/desktop-e2e.yml +++ b/.github/workflows/desktop-e2e.yml @@ -1,7 +1,7 @@ name: desktop-e2e # Smoke coverage of the real Electron shell on desktop changes. Three isolated runners -# partition the complete suite by Playwright test-file groups; workers remain serial per runner. +# partition complete test files by measured duration; workers remain serial per runner. # The weekly # advisory canary against electron@latest so Chromium-cadence breakage surfaces # before an upgrade is attempted. @@ -47,6 +47,8 @@ on: - 'packages/utils/**' - 'bun.lock' - 'package.json' + - 'scripts/desktop-e2e.ts' + - 'scripts/duration-shards.ts' schedule: - cron: '23 9 * * 1' workflow_dispatch: @@ -108,7 +110,7 @@ jobs: - name: Run Playwright _electron smoke suite working-directory: apps/desktop - run: bunx playwright test --shard=${{ matrix.shard }}/3 --reporter=line,json + run: bun run ../../scripts/desktop-e2e.ts ${{ matrix.shard }}/3 env: PLAYWRIGHT_JSON_OUTPUT_FILE: test-results/desktop-smoke.json BACKGROUND_EXECUTOR_REPORT_PATH: test-results/background-executor-report.json @@ -127,7 +129,10 @@ jobs: uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 with: name: desktop-smoke-reports-${{ matrix.shard }} - path: apps/desktop/test-results/desktop-smoke.json + path: | + apps/desktop/test-results/desktop-smoke.json + apps/desktop/test-results/desktop-shard-plan.json + apps/desktop/test-results/desktop-discovery.json if-no-files-found: ignore retention-days: 7 diff --git a/apps/desktop/e2e-durations.json b/apps/desktop/e2e-durations.json new file mode 100644 index 00000000000..2557a2730a8 --- /dev/null +++ b/apps/desktop/e2e-durations.json @@ -0,0 +1,20 @@ +{ + "background-executor.spec.ts": 131.12, + "browser-chrome.spec.ts": 15.28, + "browser-focus.spec.ts": 6.53, + "browser-page-dialogs.spec.ts": 11.81, + "browser-tools.spec.ts": 94.66, + "chat-panel.spec.ts": 31.08, + "check-report.spec.ts": 3.78, + "computer-use.spec.ts": 2.07, + "desktop-tools-live-sim.spec.ts": 0.0, + "local-files.spec.ts": 27.68, + "packaged-smoke.spec.ts": 0.0, + "password-autofill.spec.ts": 46.56, + "session-cookies.spec.ts": 11.45, + "smoke.spec.ts": 25.56, + "source-connect.spec.ts": 41.49, + "terminal-cancel.spec.ts": 199.76, + "update-notification.spec.ts": 19.55, + "updater.spec.ts": 2.48 +} diff --git a/scripts/desktop-e2e.ts b/scripts/desktop-e2e.ts new file mode 100644 index 00000000000..ea114003729 --- /dev/null +++ b/scripts/desktop-e2e.ts @@ -0,0 +1,70 @@ +import { spawnSync } from 'node:child_process' +import { mkdirSync, readFileSync, writeFileSync } from 'node:fs' +import { resolve } from 'node:path' +import { createLogger } from '@sim/logger' +import { escapeRegExp } from '@sim/utils/string' +import { partitionFiles } from './duration-shards' + +interface Suite { + specs?: { file: string }[] + suites?: Suite[] +} + +interface DiscoveryReport { + config: { rootDir: string } + suites: Suite[] + errors: unknown[] +} + +const logger = createLogger('DesktopE2E') +const match = /^(\d+)\/(\d+)$/.exec(process.argv[2] ?? '') +if (!match) throw new Error('Usage: bun run ../../scripts/desktop-e2e.ts /') +const index = Number(match[1]) +const count = Number(match[2]) +if (!Number.isSafeInteger(index) || !Number.isSafeInteger(count) || index < 1 || index > count) { + throw new Error('Invalid desktop shard index or count') +} + +mkdirSync('test-results', { recursive: true }) +const discoveryPath = resolve('test-results/desktop-discovery.json') +const discovery = spawnSync('bunx', ['playwright', 'test', '--list', '--reporter=json'], { + stdio: 'inherit', + env: { ...process.env, PLAYWRIGHT_JSON_OUTPUT_FILE: discoveryPath }, +}) +if (discovery.error) throw discovery.error +if (discovery.status !== 0) process.exit(discovery.status ?? 1) +const discoveryJson = readFileSync(discoveryPath, 'utf8') +const report = JSON.parse(discoveryJson) as DiscoveryReport +if (report.errors.length > 0) throw new Error('Playwright discovery reported errors') +const files = new Set() +function visit(suite: Suite): void { + for (const spec of suite.specs ?? []) files.add(spec.file) + for (const child of suite.suites ?? []) visit(child) +} +for (const suite of report.suites) visit(suite) +if (count > files.size) throw new Error('More desktop shards than discovered test files') +const durations = JSON.parse( + readFileSync(new URL('../apps/desktop/e2e-durations.json', import.meta.url), 'utf8') +) as Record +const shards = partitionFiles([...files], durations, count) +const selected = shards[index - 1] +if (selected.length === 0) throw new Error('Desktop shard selected no test files') +logger.info(`Running desktop shard ${index}/${count}: ${selected.join(', ')}`) +const run = spawnSync( + 'bunx', + [ + 'playwright', + 'test', + ...selected.map((file) => `^${escapeRegExp(resolve(report.config.rootDir, file))}$`), + '--reporter=line,json', + ], + { stdio: 'inherit', env: process.env } +) +mkdirSync('test-results', { recursive: true }) +writeFileSync(discoveryPath, discoveryJson) +writeFileSync( + 'test-results/desktop-shard-plan.json', + `${JSON.stringify({ index, count, discoveredFiles: files.size, shards }, null, 2)}\n` +) +if (run.error) throw run.error +process.exit(run.status ?? 1) diff --git a/scripts/duration-shards.test.ts b/scripts/duration-shards.test.ts new file mode 100644 index 00000000000..71c0b348664 --- /dev/null +++ b/scripts/duration-shards.test.ts @@ -0,0 +1,41 @@ +import { partitionFiles } from '@scripts/duration-shards' +import { describe, expect, it } from 'vitest' + +describe('duration-balanced file partitions', () => { + it('keeps whole files and assigns every discovered file exactly once, including unmeasured files', () => { + const files = ['a.spec.ts', 'b.spec.ts', 'new/nested.test.ts', 'unmeasured.spec.ts'] + const shards = partitionFiles( + files, + { 'a.spec.ts': 100, 'b.spec.ts': 1, 'deleted.spec.ts': 999 }, + 3 + ) + expect(shards.flat().sort()).toEqual([...files].sort()) + expect(shards.every((shard) => shard.length > 0)).toBe(true) + }) + + it('separates long files instead of packing them into the same shard', () => { + const weights = { a: 100, b: 90, c: 80, d: 20, e: 10 } + const shards = partitionFiles(Object.keys(weights), weights, 3) + const loads = shards.map((files) => + files.reduce((sum, file) => sum + weights[file as keyof typeof weights], 0) + ) + expect(Math.max(...loads)).toBe(100) + }) + + it('gives all runners the same partition regardless of discovery order and equal weights', () => { + const files = ['z', 'b', 'a', 'nested/c', 'x'] + expect(partitionFiles(files, {}, 3)).toEqual(partitionFiles([...files].reverse(), {}, 3)) + }) + + it('preserves zero-duration files without placing every skipped file in one empty shard', () => { + const shards = partitionFiles(['a', 'b', 'c'], { a: 0, b: 0, c: 0 }, 3) + expect(shards.map((files) => files.length)).toEqual([1, 1, 1]) + }) + + it.each([0, -1, 1.5, Number.NaN, Number.POSITIVE_INFINITY])( + 'rejects invalid shard count %s', + (count) => { + expect(() => partitionFiles(['a'], {}, count)).toThrow('shard count') + } + ) +}) diff --git a/scripts/duration-shards.ts b/scripts/duration-shards.ts new file mode 100644 index 00000000000..68273973533 --- /dev/null +++ b/scripts/duration-shards.ts @@ -0,0 +1,28 @@ +import { compareStrings } from '@sim/utils/string' + +/** Partitions discovered files without splitting their serial fixtures or omitting unmeasured files. */ +export function partitionFiles( + files: readonly string[], + durations: Record, + count: number +): string[][] { + if (!Number.isSafeInteger(count) || count < 1) throw new Error('Invalid shard count') + const measured = files + .map((file) => durations[file]) + .filter((duration) => Number.isFinite(duration) && duration > 0) + .sort((a, b) => a - b) + const fallback = measured[Math.floor(measured.length / 2)] ?? 1 + const weight = (file: string) => + Number.isFinite(durations[file]) && durations[file] >= 0 + ? Math.max(1, durations[file]) + : fallback + const shards = Array.from({ length: count }, () => ({ files: [] as string[], seconds: 0 })) + for (const file of [...files].sort((a, b) => weight(b) - weight(a) || compareStrings(a, b))) { + const target = shards.reduce((smallest, shard) => + shard.seconds < smallest.seconds ? shard : smallest + ) + target.files.push(file) + target.seconds += weight(file) + } + return shards.map((shard) => shard.files.sort(compareStrings)) +} From 6331b79ae532b0ebdc1349b6e36289c78d8bd38b Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Thu, 22 Oct 2026 13:13:06 -0700 Subject: [PATCH 2/5] test: verify balancing guards and document duration evidence --- .github/CI.md | 6 +++++- scripts/duration-shards.test.ts | 2 +- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/.github/CI.md b/.github/CI.md index bc9564169b6..aadac9d1fbe 100644 --- a/.github/CI.md +++ b/.github/CI.md @@ -43,4 +43,8 @@ The long-import and lease-expiration checks remain: they prove distinct survival Local planner validation is `bun run test:scripts scripts/ci-changes.test.ts scripts/ci-gate.test.ts`. It executes real Turbo queries in small Git fixtures; no application build or database is required. Full builds, services, browser suites, and Electron checks run on hosted CI. -The separate `desktop-e2e` workflow retains its desktop-specific path filters. Its macOS shell smoke suite is partitioned by Playwright across three isolated runners; serial test-file groups stay intact. Packaging smoke and the weekly Electron canary remain separate. Each shard uploads its small JSON coverage/timing report. +The separate `desktop-e2e` workflow retains its desktop-specific path filters. Its macOS shell smoke suite runs whole files across three isolated runners, with one worker per runner. `scripts/desktop-e2e.ts` asks Playwright to discover the complete suite, then assigns longer files first to the currently lightest shard using `apps/desktop/e2e-durations.json`. Exact escaped file filters preserve serial fixtures. Unmeasured files receive the median measured weight, deleted files cannot select tests, and invalid or empty selections fail. Normal local runs, packaging smoke and the weekly Electron canary keep their existing commands. + +The initial weights are median test durations in seconds from runs [39734295953](https://github.com/simstudioai/sim/actions/runs/39734295953), [39833640524](https://github.com/simstudioai/sim/actions/runs/39833640524), and [39836596108](https://github.com/simstudioai/sim/actions/runs/39836596108). Refresh them from complete `desktop-smoke-reports-*` artifacts when the workload changes: sum `tests[].results[].duration` by `spec.file` in each run, take the median across runs, and divide milliseconds by 1,000. The weights steer scheduling only; Playwright discovery owns coverage. Before adopting new weights, compare repeated paired runs and the complete case/status multiset, including skipped cases and retries. + +Each shard uploads its coverage/timing report, the complete discovery report, and its deterministic shard plan. `bun run test:scripts scripts/duration-shards.test.ts` checks partition coverage, unknown files, deterministic ordering, and balancing without launching Electron. diff --git a/scripts/duration-shards.test.ts b/scripts/duration-shards.test.ts index 71c0b348664..5a9477116ff 100644 --- a/scripts/duration-shards.test.ts +++ b/scripts/duration-shards.test.ts @@ -14,7 +14,7 @@ describe('duration-balanced file partitions', () => { }) it('separates long files instead of packing them into the same shard', () => { - const weights = { a: 100, b: 90, c: 80, d: 20, e: 10 } + const weights = { a: 90, b: 20, c: 10, d: 100, e: 80 } const shards = partitionFiles(Object.keys(weights), weights, 3) const loads = shards.map((files) => files.reduce((sum, file) => sum + weights[file as keyof typeof weights], 0) From 966673fda0ddcbd9ebe962dac2d23cd9cae1c9fb Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Thu, 22 Oct 2026 13:25:32 -0700 Subject: [PATCH 3/5] style: use the script alias for shard planning --- scripts/desktop-e2e.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/desktop-e2e.ts b/scripts/desktop-e2e.ts index ea114003729..3729696005e 100644 --- a/scripts/desktop-e2e.ts +++ b/scripts/desktop-e2e.ts @@ -1,9 +1,9 @@ import { spawnSync } from 'node:child_process' import { mkdirSync, readFileSync, writeFileSync } from 'node:fs' import { resolve } from 'node:path' +import { partitionFiles } from '@scripts/duration-shards' import { createLogger } from '@sim/logger' import { escapeRegExp } from '@sim/utils/string' -import { partitionFiles } from './duration-shards' interface Suite { specs?: { file: string }[] From 7cb16af8cab7d9a4fa4e98868afdb26cb3df3a0f Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Thu, 22 Oct 2026 13:30:54 -0700 Subject: [PATCH 4/5] fix: select Playwright file suites rather than test declaration locations --- scripts/desktop-e2e.ts | 15 +++------------ 1 file changed, 3 insertions(+), 12 deletions(-) diff --git a/scripts/desktop-e2e.ts b/scripts/desktop-e2e.ts index 3729696005e..fd28763429f 100644 --- a/scripts/desktop-e2e.ts +++ b/scripts/desktop-e2e.ts @@ -5,14 +5,9 @@ import { partitionFiles } from '@scripts/duration-shards' import { createLogger } from '@sim/logger' import { escapeRegExp } from '@sim/utils/string' -interface Suite { - specs?: { file: string }[] - suites?: Suite[] -} - interface DiscoveryReport { config: { rootDir: string } - suites: Suite[] + suites: { file: string }[] errors: unknown[] } @@ -36,12 +31,8 @@ if (discovery.status !== 0) process.exit(discovery.status ?? 1) const discoveryJson = readFileSync(discoveryPath, 'utf8') const report = JSON.parse(discoveryJson) as DiscoveryReport if (report.errors.length > 0) throw new Error('Playwright discovery reported errors') -const files = new Set() -function visit(suite: Suite): void { - for (const spec of suite.specs ?? []) files.add(spec.file) - for (const child of suite.suites ?? []) visit(child) -} -for (const suite of report.suites) visit(suite) +// A test factory can declare tests in a helper; the outer suite owns the discoverable test file. +const files = new Set(report.suites.map((suite) => suite.file)) if (count > files.size) throw new Error('More desktop shards than discovered test files') const durations = JSON.parse( readFileSync(new URL('../apps/desktop/e2e-durations.json', import.meta.url), 'utf8') From d718ac3eadc9b9c9ed77dcfb409db619094e6275 Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Thu, 22 Oct 2026 13:34:04 -0700 Subject: [PATCH 5/5] test: prove desktop discovery preserves imported and unmeasured cases --- .github/CI.md | 2 +- scripts/desktop-e2e.test.ts | 65 +++++++++++++++++++++++++++++++++++++ 2 files changed, 66 insertions(+), 1 deletion(-) create mode 100644 scripts/desktop-e2e.test.ts diff --git a/.github/CI.md b/.github/CI.md index aadac9d1fbe..271311ede03 100644 --- a/.github/CI.md +++ b/.github/CI.md @@ -47,4 +47,4 @@ The separate `desktop-e2e` workflow retains its desktop-specific path filters. I The initial weights are median test durations in seconds from runs [39734295953](https://github.com/simstudioai/sim/actions/runs/39734295953), [39833640524](https://github.com/simstudioai/sim/actions/runs/39833640524), and [39836596108](https://github.com/simstudioai/sim/actions/runs/39836596108). Refresh them from complete `desktop-smoke-reports-*` artifacts when the workload changes: sum `tests[].results[].duration` by `spec.file` in each run, take the median across runs, and divide milliseconds by 1,000. The weights steer scheduling only; Playwright discovery owns coverage. Before adopting new weights, compare repeated paired runs and the complete case/status multiset, including skipped cases and retries. -Each shard uploads its coverage/timing report, the complete discovery report, and its deterministic shard plan. `bun run test:scripts scripts/duration-shards.test.ts` checks partition coverage, unknown files, deterministic ordering, and balancing without launching Electron. +Each shard uploads its coverage/timing report, the complete discovery report, and its deterministic shard plan. `bun run test:scripts scripts/duration-shards.test.ts scripts/desktop-e2e.test.ts` checks partition coverage, unknown files, deterministic ordering, and balancing. The CLI regression executes tiny browser-free Playwright fixtures to prove that imported test factories and paths containing regex characters are selected correctly; it launches no Electron or browser. diff --git a/scripts/desktop-e2e.test.ts b/scripts/desktop-e2e.test.ts new file mode 100644 index 00000000000..e90d94a433c --- /dev/null +++ b/scripts/desktop-e2e.test.ts @@ -0,0 +1,65 @@ +import { spawnSync } from 'node:child_process' +import { mkdirSync, mkdtempSync, readFileSync, rmSync, symlinkSync, writeFileSync } from 'node:fs' +import { tmpdir } from 'node:os' +import { join, resolve } from 'node:path' +import { expect, it } from 'vitest' + +it('executes imported test factories and unmeasured files exactly once across desktop shards', () => { + const root = resolve(import.meta.dirname, '..') + const fixture = mkdtempSync(join(tmpdir(), 'desktop-shard-')) + try { + symlinkSync(join(root, 'node_modules'), join(fixture, 'node_modules'), 'dir') + writeFileSync(join(fixture, 'package.json'), '{"type":"module"}') + writeFileSync( + join(fixture, 'playwright.config.ts'), + "export default { testDir: './tests', workers: 1, retries: 0 }" + ) + mkdirSync(join(fixture, 'tests/nested'), { recursive: true }) + mkdirSync(join(fixture, 'tests/support'), { recursive: true }) + writeFileSync( + join(fixture, 'tests/support/factory.ts'), + "import { test } from '@playwright/test'; export function register() { test('factory', () => {}); }" + ) + writeFileSync( + join(fixture, 'tests/nested/a[1].spec.ts'), + "import { register } from '../support/factory'; register();" + ) + for (const file of ['brand-new.test.ts', 'ordinary.spec.ts']) { + writeFileSync( + join(fixture, 'tests', file), + `import { test } from '@playwright/test'; test('${file}', () => {});` + ) + } + const cases: { title: string; status: string }[] = [] + for (const index of [1, 2, 3]) { + const reportPath = join(fixture, `${index}.json`) + const result = spawnSync('bun', ['run', join(root, 'scripts/desktop-e2e.ts'), `${index}/3`], { + cwd: fixture, + env: { ...process.env, PLAYWRIGHT_JSON_OUTPUT_FILE: reportPath }, + encoding: 'utf8', + timeout: 10_000, + }) + expect(result.status, result.stdout + result.stderr).toBe(0) + const report = JSON.parse(readFileSync(reportPath, 'utf8')) as { + suites: { specs: { title: string; tests: { results: { status: string }[] }[] }[] }[] + } + for (const suite of report.suites) { + for (const spec of suite.specs) { + for (const test of spec.tests) { + for (const run of test.results) cases.push({ title: spec.title, status: run.status }) + } + } + } + } + expect(cases).toEqual( + expect.arrayContaining([ + { title: 'factory', status: 'passed' }, + { title: 'brand-new.test.ts', status: 'passed' }, + { title: 'ordinary.spec.ts', status: 'passed' }, + ]) + ) + expect(cases).toHaveLength(3) + } finally { + rmSync(fixture, { recursive: true, force: true }) + } +})