ci: balance macOS Electron smoke shards by measured duration

.github/CI.md

@@ -43,4 +43,8 @@ The long-import and lease-expiration checks remain: they prove distinct survival
 
 Local planner validation is `bun run test:scripts scripts/ci-changes.test.ts scripts/ci-gate.test.ts`. It executes real Turbo queries in small Git fixtures; no application build or database is required. Full builds, services, browser suites, and Electron checks run on hosted CI.
 
-The separate `desktop-e2e` workflow retains its desktop-specific path filters. Its macOS shell smoke suite is partitioned by Playwright across three isolated runners; serial test-file groups stay intact. Packaging smoke and the weekly Electron canary remain separate. Each shard uploads its small JSON coverage/timing report.
+The separate `desktop-e2e` workflow retains its desktop-specific path filters. Its macOS shell smoke suite runs whole files across three isolated runners, with one worker per runner. `scripts/desktop-e2e.ts` asks Playwright to discover the complete suite, then assigns longer files first to the currently lightest shard using `apps/desktop/e2e-durations.json`. Exact escaped file filters preserve serial fixtures. Unmeasured files receive the median measured weight, deleted files cannot select tests, and invalid or empty selections fail. Normal local runs, packaging smoke and the weekly Electron canary keep their existing commands.
+
+The initial weights are median test durations in seconds from runs [39734295953](https://github.com/simstudioai/sim/actions/runs/39734295953), [39833640524](https://github.com/simstudioai/sim/actions/runs/39833640524), and [39836596108](https://github.com/simstudioai/sim/actions/runs/39836596108). Refresh them from complete `desktop-smoke-reports-*` artifacts when the workload changes: sum `tests[].results[].duration` by `spec.file` in each run, take the median across runs, and divide milliseconds by 1,000. The weights steer scheduling only; Playwright discovery owns coverage. Before adopting new weights, compare repeated paired runs and the complete case/status multiset, including skipped cases and retries.
+
+Each shard uploads its coverage/timing report, the complete discovery report, and its deterministic shard plan. `bun run test:scripts scripts/duration-shards.test.ts scripts/desktop-e2e.test.ts` checks partition coverage, unknown files, deterministic ordering, and balancing. The CLI regression executes tiny browser-free Playwright fixtures to prove that imported test factories and paths containing regex characters are selected correctly; it launches no Electron or browser.

.github/workflows/desktop-e2e.yml

@@ -1,7 +1,7 @@
 name: desktop-e2e
 
 # Smoke coverage of the real Electron shell on desktop changes. Three isolated runners
-# partition the complete suite by Playwright test-file groups; workers remain serial per runner.
+# partition complete test files by measured duration; workers remain serial per runner.
 # The weekly
 # advisory canary against electron@latest so Chromium-cadence breakage surfaces
 # before an upgrade is attempted.
@@ -47,6 +47,8 @@ on:
       - 'packages/utils/**'
       - 'bun.lock'
       - 'package.json'
+      - 'scripts/desktop-e2e.ts'
+      - 'scripts/duration-shards.ts'
   schedule:
     - cron: '23 9 * * 1'
   workflow_dispatch:
@@ -108,7 +110,7 @@ jobs:
 
       - name: Run Playwright _electron smoke suite
         working-directory: apps/desktop
-        run: bunx playwright test --shard=${{ matrix.shard }}/3 --reporter=line,json
+        run: bun run ../../scripts/desktop-e2e.ts ${{ matrix.shard }}/3
         env:
           PLAYWRIGHT_JSON_OUTPUT_FILE: test-results/desktop-smoke.json
           BACKGROUND_EXECUTOR_REPORT_PATH: test-results/background-executor-report.json
@@ -127,7 +129,10 @@ jobs:
         uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2
         with:
           name: desktop-smoke-reports-${{ matrix.shard }}
-          path: apps/desktop/test-results/desktop-smoke.json
+          path: |
+            apps/desktop/test-results/desktop-smoke.json
+            apps/desktop/test-results/desktop-shard-plan.json
+            apps/desktop/test-results/desktop-discovery.json
           if-no-files-found: ignore
           retention-days: 7
 

apps/desktop/e2e-durations.json

@@ -0,0 +1,20 @@
+{
+  "background-executor.spec.ts": 131.12,
+  "browser-chrome.spec.ts": 15.28,
+  "browser-focus.spec.ts": 6.53,
+  "browser-page-dialogs.spec.ts": 11.81,
+  "browser-tools.spec.ts": 94.66,
+  "chat-panel.spec.ts": 31.08,
+  "check-report.spec.ts": 3.78,
+  "computer-use.spec.ts": 2.07,
+  "desktop-tools-live-sim.spec.ts": 0.0,
+  "local-files.spec.ts": 27.68,
+  "packaged-smoke.spec.ts": 0.0,
+  "password-autofill.spec.ts": 46.56,
+  "session-cookies.spec.ts": 11.45,
+  "smoke.spec.ts": 25.56,
+  "source-connect.spec.ts": 41.49,
+  "terminal-cancel.spec.ts": 199.76,
+  "update-notification.spec.ts": 19.55,
+  "updater.spec.ts": 2.48
+}

scripts/desktop-e2e.test.ts

@@ -0,0 +1,65 @@
+import { spawnSync } from 'node:child_process'
+import { mkdirSync, mkdtempSync, readFileSync, rmSync, symlinkSync, writeFileSync } from 'node:fs'
+import { tmpdir } from 'node:os'
+import { join, resolve } from 'node:path'
+import { expect, it } from 'vitest'
+
+it('executes imported test factories and unmeasured files exactly once across desktop shards', () => {
+  const root = resolve(import.meta.dirname, '..')
+  const fixture = mkdtempSync(join(tmpdir(), 'desktop-shard-'))
+  try {
+    symlinkSync(join(root, 'node_modules'), join(fixture, 'node_modules'), 'dir')
+    writeFileSync(join(fixture, 'package.json'), '{"type":"module"}')
+    writeFileSync(
+      join(fixture, 'playwright.config.ts'),
+      "export default { testDir: './tests', workers: 1, retries: 0 }"
+    )
+    mkdirSync(join(fixture, 'tests/nested'), { recursive: true })
+    mkdirSync(join(fixture, 'tests/support'), { recursive: true })
+    writeFileSync(
+      join(fixture, 'tests/support/factory.ts'),
+      "import { test } from '@playwright/test'; export function register() { test('factory', () => {}); }"
+    )
+    writeFileSync(
+      join(fixture, 'tests/nested/a[1].spec.ts'),
+      "import { register } from '../support/factory'; register();"
+    )
+    for (const file of ['brand-new.test.ts', 'ordinary.spec.ts']) {
+      writeFileSync(
+        join(fixture, 'tests', file),
+        `import { test } from '@playwright/test'; test('${file}', () => {});`
+      )
+    }
+    const cases: { title: string; status: string }[] = []
+    for (const index of [1, 2, 3]) {
+      const reportPath = join(fixture, `${index}.json`)
+      const result = spawnSync('bun', ['run', join(root, 'scripts/desktop-e2e.ts'), `${index}/3`], {
+        cwd: fixture,
+        env: { ...process.env, PLAYWRIGHT_JSON_OUTPUT_FILE: reportPath },
+        encoding: 'utf8',
+        timeout: 10_000,
+      })
+      expect(result.status, result.stdout + result.stderr).toBe(0)
+      const report = JSON.parse(readFileSync(reportPath, 'utf8')) as {
+        suites: { specs: { title: string; tests: { results: { status: string }[] }[] }[] }[]
+      }
+      for (const suite of report.suites) {
+        for (const spec of suite.specs) {
+          for (const test of spec.tests) {
+            for (const run of test.results) cases.push({ title: spec.title, status: run.status })
+          }
+        }
+      }
+    }
+    expect(cases).toEqual(
+      expect.arrayContaining([
+        { title: 'factory', status: 'passed' },
+        { title: 'brand-new.test.ts', status: 'passed' },
+        { title: 'ordinary.spec.ts', status: 'passed' },
+      ])
+    )
+    expect(cases).toHaveLength(3)
+  } finally {
+    rmSync(fixture, { recursive: true, force: true })
+  }
+})

scripts/desktop-e2e.ts

@@ -0,0 +1,61 @@
+import { spawnSync } from 'node:child_process'
+import { mkdirSync, readFileSync, writeFileSync } from 'node:fs'
+import { resolve } from 'node:path'
+import { partitionFiles } from '@scripts/duration-shards'
+import { createLogger } from '@sim/logger'
+import { escapeRegExp } from '@sim/utils/string'
+
+interface DiscoveryReport {
+  config: { rootDir: string }
+  suites: { file: string }[]
+  errors: unknown[]
+}
+
+const logger = createLogger('DesktopE2E')
+const match = /^(\d+)\/(\d+)$/.exec(process.argv[2] ?? '')
+if (!match) throw new Error('Usage: bun run ../../scripts/desktop-e2e.ts <shard>/<total>')
+const index = Number(match[1])
+const count = Number(match[2])
+if (!Number.isSafeInteger(index) || !Number.isSafeInteger(count) || index < 1 || index > count) {
+  throw new Error('Invalid desktop shard index or count')
+}
+
+mkdirSync('test-results', { recursive: true })
+const discoveryPath = resolve('test-results/desktop-discovery.json')
+const discovery = spawnSync('bunx', ['playwright', 'test', '--list', '--reporter=json'], {
+  stdio: 'inherit',
+  env: { ...process.env, PLAYWRIGHT_JSON_OUTPUT_FILE: discoveryPath },
+})
+if (discovery.error) throw discovery.error
+if (discovery.status !== 0) process.exit(discovery.status ?? 1)
+const discoveryJson = readFileSync(discoveryPath, 'utf8')
+const report = JSON.parse(discoveryJson) as DiscoveryReport
+if (report.errors.length > 0) throw new Error('Playwright discovery reported errors')
+// A test factory can declare tests in a helper; the outer suite owns the discoverable test file.
+const files = new Set(report.suites.map((suite) => suite.file))
+if (count > files.size) throw new Error('More desktop shards than discovered test files')
+const durations = JSON.parse(
+  readFileSync(new URL('../apps/desktop/e2e-durations.json', import.meta.url), 'utf8')
+) as Record<string, number>
+const shards = partitionFiles([...files], durations, count)
+const selected = shards[index - 1]
+if (selected.length === 0) throw new Error('Desktop shard selected no test files')
+logger.info(`Running desktop shard ${index}/${count}: ${selected.join(', ')}`)
+const run = spawnSync(
+  'bunx',
+  [
+    'playwright',
+    'test',
+    ...selected.map((file) => `^${escapeRegExp(resolve(report.config.rootDir, file))}$`),
+    '--reporter=line,json',
+  ],
+  { stdio: 'inherit', env: process.env }
+)
+mkdirSync('test-results', { recursive: true })
+writeFileSync(discoveryPath, discoveryJson)
+writeFileSync(
+  'test-results/desktop-shard-plan.json',
+  `${JSON.stringify({ index, count, discoveredFiles: files.size, shards }, null, 2)}\n`
+)
+if (run.error) throw run.error
+process.exit(run.status ?? 1)

scripts/duration-shards.test.ts

@@ -0,0 +1,41 @@
+import { partitionFiles } from '@scripts/duration-shards'
+import { describe, expect, it } from 'vitest'
+
+describe('duration-balanced file partitions', () => {
+  it('keeps whole files and assigns every discovered file exactly once, including unmeasured files', () => {
+    const files = ['a.spec.ts', 'b.spec.ts', 'new/nested.test.ts', 'unmeasured.spec.ts']
+    const shards = partitionFiles(
+      files,
+      { 'a.spec.ts': 100, 'b.spec.ts': 1, 'deleted.spec.ts': 999 },
+      3
+    )
+    expect(shards.flat().sort()).toEqual([...files].sort())
+    expect(shards.every((shard) => shard.length > 0)).toBe(true)
+  })
+
+  it('separates long files instead of packing them into the same shard', () => {
+    const weights = { a: 90, b: 20, c: 10, d: 100, e: 80 }
+    const shards = partitionFiles(Object.keys(weights), weights, 3)
+    const loads = shards.map((files) =>
+      files.reduce((sum, file) => sum + weights[file as keyof typeof weights], 0)
+    )
+    expect(Math.max(...loads)).toBe(100)
+  })
+
+  it('gives all runners the same partition regardless of discovery order and equal weights', () => {
+    const files = ['z', 'b', 'a', 'nested/c', 'x']
+    expect(partitionFiles(files, {}, 3)).toEqual(partitionFiles([...files].reverse(), {}, 3))
+  })
+
+  it('preserves zero-duration files without placing every skipped file in one empty shard', () => {
+    const shards = partitionFiles(['a', 'b', 'c'], { a: 0, b: 0, c: 0 }, 3)
+    expect(shards.map((files) => files.length)).toEqual([1, 1, 1])
+  })
+
+  it.each([0, -1, 1.5, Number.NaN, Number.POSITIVE_INFINITY])(
+    'rejects invalid shard count %s',
+    (count) => {
+      expect(() => partitionFiles(['a'], {}, count)).toThrow('shard count')
+    }
+  )
+})

scripts/duration-shards.ts

@@ -0,0 +1,28 @@
+import { compareStrings } from '@sim/utils/string'
+
+/** Partitions discovered files without splitting their serial fixtures or omitting unmeasured files. */
+export function partitionFiles(
+  files: readonly string[],
+  durations: Record<string, number>,
+  count: number
+): string[][] {
+  if (!Number.isSafeInteger(count) || count < 1) throw new Error('Invalid shard count')
+  const measured = files
+    .map((file) => durations[file])
+    .filter((duration) => Number.isFinite(duration) && duration > 0)
+    .sort((a, b) => a - b)
+  const fallback = measured[Math.floor(measured.length / 2)] ?? 1
+  const weight = (file: string) =>
+    Number.isFinite(durations[file]) && durations[file] >= 0
+      ? Math.max(1, durations[file])
+      : fallback
+  const shards = Array.from({ length: count }, () => ({ files: [] as string[], seconds: 0 }))
+  for (const file of [...files].sort((a, b) => weight(b) - weight(a) || compareStrings(a, b))) {
+    const target = shards.reduce((smallest, shard) =>
+      shard.seconds < smallest.seconds ? shard : smallest
+    )
+    target.files.push(file)
+    target.seconds += weight(file)
+  }
+  return shards.map((shard) => shard.files.sort(compareStrings))
+}