diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 6832882e..2fee9a73 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -67,16 +67,19 @@ jobs: - run: git diff --check tests: - name: Tests (${{ matrix.os }}, ${{ matrix.shard }}/2) + name: Tests (${{ matrix.os }}, ${{ matrix.shard }}/3) needs: changes if: needs.changes.outputs.code == 'true' runs-on: ${{ matrix.os }} - timeout-minutes: 10 + # Three shards per OS: test steps measured 4.9-6.6 minutes and whole jobs 6-7.8 minutes on 2026-10-22 (two shards had + # reached 8-10 minutes against a 10-minute cap). The cap is about 1.5x the slowest job, so slow macOS runners pass and a + # hang still fails the job; every test keeps its own 120 s timeout. + timeout-minutes: 12 strategy: fail-fast: false matrix: os: [macos-latest, ubuntu-24.04] - shard: [1, 2] + shard: [1, 2, 3] steps: - uses: actions/checkout@v4 - uses: actions/setup-node@v4 @@ -101,7 +104,7 @@ jobs: echo "NMSH_TEST_MERMAID=$dir/package" >> "$GITHUB_ENV" # Built-launcher tests must execute rather than silently skip on fresh runners. - run: npm run build - - run: npm test -- --shard=${{ matrix.shard }}/2 --test-timeout=120000 + - run: npm test -- --shard=${{ matrix.shard }}/3 --test-timeout=120000 - name: Process leak check if: always() run: bash scripts/check-process-leaks.sh diff --git a/docs/development-verification.md b/docs/development-verification.md index bc31082b..ee868ef2 100644 --- a/docs/development-verification.md +++ b/docs/development-verification.md @@ -22,13 +22,16 @@ Local commands reuse node_modules. Clean CI/release environments use ## Canonical suite and sharding -The only shard interface is `npm test -- --shard=1/2` (or `2/2`). No arguments +The only shard interface is `npm test -- --shard=/` (CI uses `1/3`, `2/3`, `3/3`). No arguments still runs every recursively discovered `tests/**/*.test.ts` file. File selection lives in `scripts/test-selection.mjs`; its tests verify deterministic assignment, exact union, no overlap, invalid-input rejection, and curated-list existence. Runtime files are partitioned as whole files by deterministic greedy scheduling. Rounded measured weights for slow files balance PTY work; ordinary/new files -default to one. Reprofile weights only when actual shard imbalance warrants it. `suggestionRanking.test.ts` runs +default to one. Reprofile weights only when actual shard imbalance warrants it: the per-test durations in the +CI logs, attributed to their files, are the source (last done 2026-10-22, when unweighted browser and live files had pushed +shards to 8–10 minutes of a 10-minute cap). Tests that drive a real Chromium share one browser per file where they test +pages rather than the browser's own lifecycle (`sharedLab()` in `tests/support/browserFixtures.ts`). `suggestionRanking.test.ts` runs separately after runtime files, only in shard one, preserving uncontested latency measurement. New files automatically enter the canonical suite and a shard. @@ -40,8 +43,8 @@ session scenarios run fully on Node 26 on both macOS and Ubuntu. ## CI gates -Code PRs run quality on Ubuntu/Node 26, two macOS/Node 26 shards, two Ubuntu/Node -26 shards, and curated Ubuntu/Node 22 compatibility. Each shard builds because +Code PRs run quality on Ubuntu/Node 26, three macOS/Node 26 shards, three Ubuntu/Node +26 shards (12-minute cap each, about 1.5 times the slowest measured job, so a hang still fails), and curated Ubuntu/Node 22 compatibility. Each shard builds because built-launcher tests otherwise skip on a clean checkout. Each integration shard and smoke job checks helper-process leaks, including after a test failure. Existing hard latency assertions remain in the canonical tests. diff --git a/scripts/test-selection.mjs b/scripts/test-selection.mjs index 3077682c..d248e884 100644 --- a/scripts/test-selection.mjs +++ b/scripts/test-selection.mjs @@ -18,36 +18,80 @@ const suites = { fedora: ['linuxPlatform', 'tools', 'update', 'hostProfiles', 'portabilityUninstall'], }; -// Rounded seconds from the macOS Node 26 canonical baseline (2026-10-04). +// Rounded seconds: the slower of the macOS and Ubuntu Node 26 CI shards (run 39756506927, 2026-10-22); files not +// measured there keep their 2026-10-04 macOS baseline. The two Chrome files are estimates after they began sharing one +// browser per file. // Only slow files need weights; new/ordinary files default to one. Greedy // whole-file scheduling avoids clustering the costly PTY files in one shard. const runtimeWeights = { - "bundledCatalog.test.ts": 5, - "compatibilityHarness.test.ts": 28, - "commandQueueLive.test.ts": 93, - "copySelectionLive.test.ts": 13, - "configuredCompletion.test.ts": 6, - "ctrlZJobControl.test.ts": 12, - "detachedOutput.test.ts": 22, + "activityStyleLive.test.ts": 16, + "ansiLiveRegion.test.ts": 15, + "arcadeAll.test.ts": 8, + "arcadeBackgroundLive.test.ts": 52, + "arcadeDashGame.test.ts": 26, + "arcadeDashPhysics.test.ts": 26, + "arcadePinball.test.ts": 20, + "arcadeQueensLogic.test.ts": 15, + "arcadeSudokuLogic.test.ts": 6, + "backgroundSupervisor.test.ts": 11, + "browserEngineChrome.test.ts": 50, + "browserPanel.test.ts": 6, + "bundledCatalog.test.ts": 11, + "chromaV2.test.ts": 6, + "clipsLive.test.ts": 6, + "commandQueueLive.test.ts": 97, + "compareOutputLive.test.ts": 10, + "compatibilityHarness.test.ts": 47, + "configuredCompletion.test.ts": 13, + "contextEngine.test.ts": 13, + "contextHostileWorkspace.test.ts": 9, + "copyExactLive.test.ts": 7, + "copyPayload.test.ts": 5, + "copyReportLive.test.ts": 5, + "copySelectionLive.test.ts": 51, + "ctrlZJobControl.test.ts": 24, + "detachedOutput.test.ts": 38, + "diagramsChrome.test.ts": 25, + "flowComposer.test.ts": 8, "fullChromaQa.test.ts": 5, - "hostProfiles.test.ts": 10, - "inputAwarenessLive.test.ts": 59, - "interactiveCli.test.ts": 14, - "liveHardening.test.ts": 45, - "liveStatus.test.ts": 5, - "muxInterop.test.ts": 31, - "nativeCaptureLifecycle.test.ts": 6, - "sessionLifecycle.test.ts": 28, - "sessionPresets.test.ts": 20, + "fullscreenFailure.test.ts": 8, + "fullscreenResizeSampling.test.ts": 10, + "historyReuseLive.test.ts": 11, + "hostProfiles.test.ts": 18, + "hybrid.test.ts": 6, + "inputAwarenessLive.test.ts": 52, + "inputOwnershipLive.test.ts": 46, + "interactiveCli.test.ts": 37, + "keepAwakeForeground.test.ts": 8, + "liveHardening.test.ts": 88, + "liveStatus.test.ts": 10, + "moduleDiagnosticsLive.test.ts": 7, + "muxInterop.test.ts": 47, + "nativeCaptureLifecycle.test.ts": 8, + "outputReferencesLive.test.ts": 7, "pasteBatchLive.test.ts": 21, + "pinsLive.test.ts": 7, "queueConditionsLive.test.ts": 24, + "queuePreviewLive.test.ts": 33, + "queueSummaryLive.test.ts": 23, + "sessionLifecycle.test.ts": 78, + "sessionPresets.test.ts": 45, "shellAdapters.test.ts": 35, + "shellIntelligence.test.ts": 22, "shellSwitchApp.test.ts": 23, - "startupBlocked.test.ts": 29, - "startupDiscovery.test.ts": 22, - "startupLaunch.test.ts": 8, + "startupBlocked.test.ts": 64, + "startupDiscovery.test.ts": 87, + "startupLaunch.test.ts": 10, + "startupLifecycle.test.ts": 7, + "terminalBaseline.test.ts": 9, + "terminalModeReconciliation.test.ts": 8, + "themeBridgeLive.test.ts": 5, + "tmpdirHygiene.test.ts": 9, + "toolConfig.test.ts": 6, + "trackLive.test.ts": 13, "typingAfterRead.test.ts": 17, - "themeBridgeLive.test.ts": 12 + "viewerLive.test.ts": 13, + "worktreeMutations.test.ts": 11 }; export function discoverTestFiles() { diff --git a/tests/browserEngineChrome.test.ts b/tests/browserEngineChrome.test.ts index 3656290a..baf74149 100644 --- a/tests/browserEngineChrome.test.ts +++ b/tests/browserEngineChrome.test.ts @@ -1,9 +1,9 @@ -import test from 'node:test'; +import test, {after} from 'node:test'; import assert from 'node:assert/strict'; import fs from 'node:fs'; import path from 'node:path'; import type {BrowserPage, Frame} from '../src/browser/types.js'; -import {chromeSkipReason, createLab, Events, localSite, pidAlive, scratchDir, waitFor} from './support/browserFixtures.js'; +import {chromeSkipReason, createLab, disposeSharedLab, Events, localSite, pidAlive, scratchDir, sharedLab, waitFor} from './support/browserFixtures.js'; import {createBrowserEngine} from '../src/browser/engine/index.js'; /** @@ -14,6 +14,7 @@ const skip = await chromeSkipReason(); const view = {width: 600, height: 400, scale: 1}; const sleep = (ms: number): Promise => new Promise(resolve => setTimeout(resolve, ms)); const settled = (page: BrowserPage, url?: string): Promise => waitFor(() => (url === undefined || page.url === url) && !page.loading && true, `${url ?? 'the page'} to load`); +after(disposeSharedLab); test('launch is lazy, uses a private profile, and shutdown leaves no process behind', {skip}, async () => { const lab = createLab(); @@ -93,7 +94,7 @@ test('a missing browser is reported in plain words and nothing is started', asyn }); test('web pages load from a local server; navigation, back, forward and reload work', {skip}, async () => { - const lab = createLab(); + const lab = sharedLab(); const site = await localSite({'/a': 'Page A

a

', '/b': 'Page B

b

'}); try { const page = await lab.engine.open({kind: 'web', viewport: view, url: `${site.origin}/a`}); @@ -122,7 +123,7 @@ test('web pages load from a local server; navigation, back, forward and reload w }); test('addresses that are not http, https, file or about:blank are refused and never navigated', {skip}, async () => { - const lab = createLab(); + const lab = sharedLab(); try { const page = await lab.engine.open({kind: 'web', viewport: view, url: 'about:blank'}); const events = new Events(page); @@ -138,7 +139,7 @@ test('addresses that are not http, https, file or about:blank are refused and ne }); test('a page that redirects or links to a refused scheme ends on about:blank with a blocked event', {skip}, async () => { - const lab = createLab(); + const lab = sharedLab(); const site = await localSite({'/': 'xstart'}); try { const page = await lab.engine.open({kind: 'web', viewport: view, url: site.origin + '/'}); @@ -158,7 +159,7 @@ test('a page that redirects or links to a refused scheme ends on about:blank wit const animation = '
'; test('screencast frames arrive at the viewport size, advance with a CSS animation, and a slow listener skips frames', {skip}, async () => { - const lab = createLab(); + const lab = sharedLab(); try { const handle = await lab.preview(animation); const page = await lab.engine.open({kind: 'preview', viewport: {width: 320, height: 200, scale: 1}, url: handle.url}); @@ -197,7 +198,7 @@ test('screencast frames arrive at the viewport size, advance with a CSS animatio }); test('zoom keeps the frame size and is clamped', {skip}, async () => { - const lab = createLab(); + const lab = sharedLab(); try { const handle = await lab.preview('

zoom

'); const page = await lab.engine.open({kind: 'preview', viewport: {width: 300, height: 200, scale: 1}, url: handle.url}); @@ -224,7 +225,7 @@ const form = `
'); const page = await lab.engine.open({kind: 'preview', viewport: view, url: handle.url}); @@ -276,7 +277,7 @@ test('wheel events scroll the page', {skip}, async () => { }); test('find works on a preview with scripts off, steps and wraps, and clears', {skip}, async () => { - const lab = createLab(); + const lab = sharedLab(); try { const handle = await lab.preview('

Needle one

nothing

second needle

needle hidden

NEEDLE last

'); const page = await lab.engine.open({kind: 'preview', viewport: view, url: handle.url}); @@ -299,7 +300,7 @@ test('find works on a preview with scripts off, steps and wraps, and clears', {s }); test('setScripts turns page scripts on and off and reloads', {skip}, async () => { - const lab = createLab(); + const lab = sharedLab(); try { const handle = await lab.preview('before', {trusted: true}); const page = await lab.engine.open({kind: 'preview', viewport: view, url: handle.url}); @@ -315,7 +316,7 @@ test('setScripts turns page scripts on and off and reloads', {skip}, async () => }); test('popups are closed and reported; dialogs are answered; downloads are refused', {skip}, async () => { - const lab = createLab(); + const lab = sharedLab(); const site = await localSite({ '/': `pop get @@ -352,7 +353,7 @@ test('popups are closed and reported; dialogs are answered; downloads are refuse }); test('a preview refuses remote resources and navigation, and no request leaves the machine', {skip}, async () => { - const lab = createLab(); + const lab = sharedLab(); const remote = await localSite(); try { const html = ` @@ -379,7 +380,7 @@ test('a preview refuses remote resources and navigation, and no request leaves t }); test('previews share nothing with browsing: cookies set in a web page are invisible to a preview', {skip}, async () => { - const lab = createLab(); + const lab = sharedLab(); const site = await localSite({'/': {type: 'text/html', body: 'set', headers: {'Set-Cookie': 'k=v; Path=/'}}}); try { const web = await lab.engine.open({kind: 'web', viewport: view, url: site.origin + '/'}); @@ -395,7 +396,7 @@ test('previews share nothing with browsing: cookies set in a web page are invisi }); test('closing a page closes its target and a preview page disposes its context', {skip}, async () => { - const lab = createLab(); + const lab = sharedLab(); try { const handle = await lab.preview('

x

'); const page = await lab.engine.open({kind: 'preview', viewport: view, url: handle.url}); diff --git a/tests/diagramsChrome.test.ts b/tests/diagramsChrome.test.ts index b56b19d8..6183af2e 100644 --- a/tests/diagramsChrome.test.ts +++ b/tests/diagramsChrome.test.ts @@ -1,4 +1,4 @@ -import test from 'node:test'; +import test, {after} from 'node:test'; import assert from 'node:assert/strict'; import http from 'node:http'; import type {AddressInfo} from 'node:net'; @@ -31,12 +31,25 @@ async function skip(): Promise { return (await chromeSkipReason()) ?? (await mermaidSkip); } -async function lab(): Promise<{engine: BrowserEngine; dir: string; done(): Promise}> { +interface DiagramLab {engine: BrowserEngine; dir: string; done(): Promise} + +function freshLab(): DiagramLab { const dir = scratchDir('nmsh-diagrams-'); const engine = createBrowserEngine({profileDir: path.join(dir, 'profile'), idleMs: 60_000}); return {engine, dir, done: async () => { await engine.shutdown(); fs.rmSync(dir, {recursive: true, force: true, maxRetries: 10, retryDelay: 200}); }}; } +// Launching Chromium costs seconds on CI runners: tests that only open renderer pages share one browser; each test closes the +// pages it left open, so the next starts with none. The crash test keeps a browser of its own. +let shared: DiagramLab | undefined; +after(async () => { await shared?.done(); shared = undefined; }); + +async function lab(options: {fresh?: boolean} = {}): Promise { + if (options.fresh) return freshLab(); + const base = shared ??= freshLab(); + return {...base, done: async () => { await Promise.all(base.engine.pages().map(page => page.close().catch(() => undefined))); }}; +} + const request = (source: string, extra: Partial = {}): DiagramRenderRequest => ({source, config: mermaidConfig(THEME), scale: 2, timeoutMs: 10_000, ...extra}); const TYPES: Record = { @@ -176,7 +189,7 @@ test('a library that never finishes is stopped at the timeout; the page is throw test('a crashed renderer page is reported, never hangs, and the browser stops cleanly afterwards', async t => { const reason = await skip(); if (reason) { t.skip(reason); return; } - const {engine, done} = await lab(); + const {engine, done} = await lab({fresh: true}); try { const renderer: DiagramRenderer = await engine.openDiagramRenderer!(library!); assert.ok((await renderer.render(request(TYPES.flowchart!))).ok); diff --git a/tests/support/browserFixtures.ts b/tests/support/browserFixtures.ts index dde0fdf8..ada058aa 100644 --- a/tests/support/browserFixtures.ts +++ b/tests/support/browserFixtures.ts @@ -93,6 +93,29 @@ export function createLab(options: {idleMs?: number} = {}): Lab { return lab; } +let shared: Lab | undefined; + +/** + * One browser for every test in a file that only needs pages: launching Chromium costs seconds on CI runners, so tests that do + * not exercise launch, idle shutdown or crashes share it. `dispose()` closes the pages the test left open (each test starts with + * none) and keeps the browser; call `disposeSharedLab()` from the file's `after` hook. Lifecycle tests keep `createLab()`. + */ +export function sharedLab(): Lab { + const lab = shared ??= createLab(); + return { + ...lab, + async dispose() { + await Promise.all(lab.engine.pages().map(page => page.close().catch(() => undefined))); + } + }; +} + +export async function disposeSharedLab(): Promise { + const lab = shared; + shared = undefined; + await lab?.dispose(); +} + /** Collects a page's events and lets a test wait for one. */ export class Events { readonly all: PageEvent[] = [];