From da26322a147f93fdf1d17405d2f9246d4ca60708 Mon Sep 17 00:00:00 2001 From: Rami Date: Wed, 21 Oct 2026 22:57:28 +0100 Subject: [PATCH 1/4] test(identity): the key test that reads an env file removes the folder it wrote the file in The test of an unquoted compact JSON line in an env file made an api-keys- folder under the temporary directory and never removed it, so each run of the suite left one there. It now removes it once the test finished. Co-Authored-By: Claude Fable 5.1 --- packages/identity/src/keys/api-keys-setting.test.ts | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/packages/identity/src/keys/api-keys-setting.test.ts b/packages/identity/src/keys/api-keys-setting.test.ts index c45a16a03..391b6cefc 100644 --- a/packages/identity/src/keys/api-keys-setting.test.ts +++ b/packages/identity/src/keys/api-keys-setting.test.ts @@ -1,9 +1,9 @@ import { spawnSync } from 'node:child_process'; -import { mkdtempSync, writeFileSync } from 'node:fs'; +import { mkdtempSync, rmSync, writeFileSync } from 'node:fs'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; -import { describe, expect, it } from 'vitest'; +import { describe, expect, it, onTestFinished } from 'vitest'; import { InvalidApiKeysError, createApiKey, readApiKeys, type ApiKey } from '../index.ts'; @@ -59,6 +59,9 @@ describe('readApiKeys', () => { it('reads an unquoted compact JSON line from an env file unchanged', () => { const directory = mkdtempSync(join(tmpdir(), 'api-keys-')); + onTestFinished(() => { + rmSync(directory, { recursive: true, force: true }); + }); const line = JSON.stringify([entry, everything]); writeFileSync(join(directory, 'keys.env'), `API_KEYS=${line}\n`); From c287a2c27f5328d082b46b0a68dadd74ae1d6528 Mon Sep 17 00:00:00 2001 From: Rami Date: Wed, 21 Oct 2026 22:59:39 +0100 Subject: [PATCH 2/4] test(server): the computation test over MCP stops each server it starts, so none leaves a ledger Its tests kept the server of onAlpha in one variable that afterEach stopped, and the test of the definition tools called onAlpha twice, so its first server ran on unstopped and left its auto-runtime-server- folder, the ledger with its 1.1 MB WAL, under the temporary directory on every run of the suite. onAlpha now stops the server it starts once the test finished. Co-Authored-By: Claude Fable 5.1 --- .../src/computation/computation-over-mcp.test.ts | 13 ++++--------- 1 file changed, 4 insertions(+), 9 deletions(-) diff --git a/packages/server/src/computation/computation-over-mcp.test.ts b/packages/server/src/computation/computation-over-mcp.test.ts index fca47ee44..d980ab494 100644 --- a/packages/server/src/computation/computation-over-mcp.test.ts +++ b/packages/server/src/computation/computation-over-mcp.test.ts @@ -1,9 +1,9 @@ import { listedTools, plainTextIn, problemIn, textOf, withMcpSession, type McpSession } from '@on.auto/api/testing'; import { campaignPace, campaignRows } from '@on.auto/computation/testing'; import { internalTermsIn } from '@on.auto/operations/testing'; -import { afterEach, describe, expect, it } from 'vitest'; +import { describe, expect, it, onTestFinished } from 'vitest'; -import { servingReasoning, type ReasoningServer } from '../testing/servers/reasoning-server.ts'; +import { servingReasoning } from '../testing/servers/reasoning-server.ts'; const dated = [ '---', @@ -23,14 +23,9 @@ const total = [ '}', ].join('\n'); -let server: ReasoningServer; - -afterEach(async () => { - await server.stop(); -}); - async function onAlpha(use: (session: McpSession) => Promise): Promise { - server = await servingReasoning([]); + const server = await servingReasoning([]); + onTestFinished(server.stop); await withMcpSession('current revision', { url: `${server.origin}/orgs/acme/mcp`, headers: {} }, (session) => session.callTool('create_playbook', { playbook: 'alpha', name: 'Alpha' }), ); From 922e77d6656f4dbe3efc77dc1b4678c2c130f028 Mon Sep 17 00:00:00 2001 From: Rami Date: Wed, 21 Oct 2026 23:10:04 +0100 Subject: [PATCH 3/4] test(global): a test run owns a temporary folder of its own, which its teardown checks and removes The helpers that make a folder remove it on onTestFinished or in a finally, which a worker that dies, a hook that times out or a child that is killed never reaches, so the suites left their folders under the machine's temporary directory. vitest.shared.ts now names a global setup, aFolderOfTheTestRun of @on.auto/operations, that makes auto-runtime-test-run-- in the temporary folder and sets TMPDIR to it before any worker starts, so tmpdir() answers it in every worker and in every process a test starts with the environment it inherits. Its teardown lists what the tests left there, ends any process the setup's runningIn finds still running in an entry, fails the run naming each entry and each process, and removes the folder whatever happened. Two runs at once have two folders. The ending of a process, SIGTERM and SIGKILL after stoppedWithinMs, moves from test-processes.ts to process-endings.ts, which imports nothing of Vitest, since a global setup runs in Vitest's own process. turbo.json counts both files among its global dependencies, as it counts vitest.shared.ts, since the tests of every package run under them. Co-Authored-By: Claude Fable 5.1 --- CONTRIBUTING.md | 1 + packages/operations/README.md | 2 + .../src/process-testing/process-endings.ts | 48 +++++++ .../process-testing/test-processes.test.ts | 3 +- .../src/process-testing/test-processes.ts | 43 +----- .../process-testing/test-run-folder.test.ts | 130 ++++++++++++++++++ .../src/process-testing/test-run-folder.ts | 56 ++++++++ packages/operations/src/testing/index.ts | 3 +- turbo.json | 2 + vitest.shared.ts | 5 + 10 files changed, 252 insertions(+), 41 deletions(-) create mode 100644 packages/operations/src/process-testing/process-endings.ts create mode 100644 packages/operations/src/process-testing/test-run-folder.test.ts create mode 100644 packages/operations/src/process-testing/test-run-folder.ts diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 60b026e5f..ba69744d4 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -43,6 +43,7 @@ The preview opens under `/docs/`. Read [Documentation contributions](docs/contri - **TypeScript must be erasable**, because Node strips the types at runtime. That means no `enum`, `namespace`, parameter properties or decorators. Relative imports include the `.ts` extension. - **Tests sit next to the code** as `*.test.ts` and test behaviour through the public interface, with injected fakes rather than mocks. - **A test waits for the event that shows the state it needs, never for the clock,** and the one-minute ceiling in `vitest.shared.ts` only guards against a hang. +- **A test removes what it makes.** Each run of a package's tests works in a temporary folder of its own, which `tmpdir()` answers in every worker, and its teardown fails the run, naming what a test left there, before it removes the folder. ## Commits and pull requests diff --git a/packages/operations/README.md b/packages/operations/README.md index 9b76ed70c..cd4cfec71 100644 --- a/packages/operations/README.md +++ b/packages/operations/README.md @@ -215,3 +215,5 @@ The digest is Node's, behind the one module of the repository for it: the digest `recordedContentBehaviour(entry)` is the behaviour suite of the content port, in `src/content-testing`: a content of no bytes to 64 MiB kept and streamed back, a text kept once with its first media type in its playbook alone, a content put again whole or with its digest, the refusals of a wrong digest, a wrong size and a stream that fails, wrong bytes put with the digest of a content held, a wrong put racing a right one, and staging, read through a second client, ended, refused and raced. An adapter runs it with a `ContentEntry` whose `aStore()` answers `{ content, elsewhere, unnamedLeft }`: the port, the port of a second client of the same store, and how many pieces of puts no kept content names the store holds, which is 0 after every test. This package runs it on `memoryRecordedContent`, the ledger on SQLite and PostgreSQL, and `@on.auto/object-storage` on its fake bucket and on a real S3-compatible server. The samples it is made of are exported with it: `generated(bytes, seed)` and `generatedStream`, bytes made the same for a seed, `answerOfAbout(bytes)`, a JSON answer of about that size, `readBack` and `readThrough`, which digest what a read streams, `deliveredBeforeFailing(stream)`, how many bytes a failing stream handed over, `brokenAfter(bytes)` and `unread()`, streams that fail or are never read, and `resultOf`, `refusalOf` and `present`, which run an effect for its value, its failure or its value present. `spawnForTheTest(command, args, options)` (`src/process-testing/test-processes.ts`) is how a test of the repository starts a process it runs itself. It starts it as `spawn` of `node:child_process` does and registers its end with `onTestFinished` of Vitest, the hook that runs once the test finished, whether it passed, failed or timed out: a process still there is sent SIGTERM, on which the server stops, and SIGKILL if it is still there `stoppedWithinMs`, 10 s, later, a bound the server's test of its longest stop holds that stop under, and well within the 60 s a hook may take. `endWithTheTest(pid)` ends in the same way, by its pid, a process the test did not start itself, such as a server started in the background, and signals no pid that `kill(pid, 0)` finds gone or another user's; it cannot tell a pid the system gave to another process of the same user since. `endWithTheTest(reader)` takes a `PidReader` in place of the pid, registered before the process is started: once the test finished, it reads the pid with `read()` again and again while `stillToCome()` holds, for at most `readWithinMs`, and ends the pid it read, if any. A worker of the test runner that dies runs no hook, so it ends nothing. + +`aFolderOfTheTestRun(runningIn)` (`src/process-testing/test-run-folder.ts`) is the global setup of the tests of every package, which `vitest.shared.ts` names with its default, `aFolderOfTheTestRun()`. Each run makes a folder of its own in the temporary folder, `auto-runtime-test-run--`, and sets `TMPDIR` to it before any worker starts, so `tmpdir()` of `node:os` answers it in every worker and in every process a test starts with the environment it inherits, and two runs at once have two folders. Its teardown, which a worker that dies, a hook that times out or a child that is killed does not keep from running, lists what the tests left there, ends each `RunningProcess` that `runningIn(entry)` finds still running in an entry, with SIGTERM and with SIGKILL if `isStillRunning()` still holds `stoppedWithinMs` later, sending a signal only while it holds, fails the run naming each entry and each process it ended, and removes the folder whatever happened. It imports nothing of Vitest, since a global setup runs in Vitest's own process, where the hooks of a test cannot. diff --git a/packages/operations/src/process-testing/process-endings.ts b/packages/operations/src/process-testing/process-endings.ts new file mode 100644 index 000000000..427a0c2d3 --- /dev/null +++ b/packages/operations/src/process-testing/process-endings.ts @@ -0,0 +1,48 @@ +import { setTimeout } from 'node:timers/promises'; + +export interface Ending { + readonly isGone: () => boolean; + readonly signal: (name: NodeJS.Signals) => unknown; +} + +export interface RunningProcess { + readonly name: string; + readonly pid: number; + readonly isStillRunning: () => boolean; +} + +export const stoppedWithinMs = 10_000; + +export const lookEveryMs = 50; + +export function signalled(pid: number, signal: NodeJS.Signals | 0): boolean { + try { + process.kill(pid, signal); + return true; + } catch { + return false; + } +} + +async function goneBy(ending: Ending, deadline: number): Promise { + if (ending.isGone() || performance.now() >= deadline) { + return ending.isGone(); + } + await setTimeout(lookEveryMs); + return goneBy(ending, deadline); +} + +export async function endedWithin(ending: Ending, withinMs: number): Promise { + if (ending.isGone()) { + return; + } + ending.signal('SIGTERM'); + if (!(await goneBy(ending, performance.now() + withinMs))) { + ending.signal('SIGKILL'); + await goneBy(ending, Number.POSITIVE_INFINITY); + } +} + +export function endedWhileItRuns({ pid, isStillRunning }: RunningProcess): Promise { + return endedWithin({ isGone: () => !isStillRunning(), signal: (name) => signalled(pid, name) }, stoppedWithinMs); +} diff --git a/packages/operations/src/process-testing/test-processes.test.ts b/packages/operations/src/process-testing/test-processes.test.ts index 0ef4c2895..d33168882 100644 --- a/packages/operations/src/process-testing/test-processes.test.ts +++ b/packages/operations/src/process-testing/test-processes.test.ts @@ -5,7 +5,8 @@ import { setTimeout } from 'node:timers/promises'; import { describe, expect, it, onTestFinished } from 'vitest'; -import { endWithTheTest, spawnForTheTest, stoppedWithinMs, type PidReader, type TestFinish } from './test-processes.ts'; +import { stoppedWithinMs } from './process-endings.ts'; +import { endWithTheTest, spawnForTheTest, type PidReader, type TestFinish } from './test-processes.ts'; const stopsWhenTold = [ "process.on('SIGTERM', () => { process.stdout.write('stopped'); process.exit(0); });", diff --git a/packages/operations/src/process-testing/test-processes.ts b/packages/operations/src/process-testing/test-processes.ts index e77d0938f..323dcf062 100644 --- a/packages/operations/src/process-testing/test-processes.ts +++ b/packages/operations/src/process-testing/test-processes.ts @@ -3,6 +3,8 @@ import { setTimeout } from 'node:timers/promises'; import { onTestFinished } from 'vitest'; +import { endedWithin, lookEveryMs, signalled, stoppedWithinMs, type Ending } from './process-endings.ts'; + export interface TestProcessOptions { readonly cwd?: string; readonly env?: Readonly>; @@ -23,15 +25,6 @@ export interface PidReader { readonly readWithinMs: number; } -interface Ending { - readonly isGone: () => boolean; - readonly signal: (name: NodeJS.Signals) => unknown; -} - -export const stoppedWithinMs = 10_000; - -const lookEveryMs = 50; - const theTestsFinish: TestFinish = { registered: (end) => { onTestFinished(end); @@ -39,34 +32,6 @@ const theTestsFinish: TestFinish = { stoppedWithinMs, }; -function signalled(pid: number, signal: NodeJS.Signals | 0): boolean { - try { - process.kill(pid, signal); - return true; - } catch { - return false; - } -} - -async function goneBy(ending: Ending, deadline: number): Promise { - if (ending.isGone() || performance.now() >= deadline) { - return ending.isGone(); - } - await setTimeout(lookEveryMs); - return goneBy(ending, deadline); -} - -async function ended(ending: Ending, withinMs: number): Promise { - if (ending.isGone()) { - return; - } - ending.signal('SIGTERM'); - if (!(await goneBy(ending, performance.now() + withinMs))) { - ending.signal('SIGKILL'); - await goneBy(ending, Number.POSITIVE_INFINITY); - } -} - function endingOfPid(pid: number): Ending { return { isGone: () => !signalled(pid, 0), signal: (name) => signalled(pid, name) }; } @@ -90,7 +55,7 @@ export function endWithTheTest(pidOrReader: number | PidReader, finish: TestFini finish.registered(async () => { const pid = await pidOf(pidOrReader); if (pid !== undefined) { - await ended(endingOfPid(pid), finish.stoppedWithinMs); + await endedWithin(endingOfPid(pid), finish.stoppedWithinMs); } }); } @@ -114,6 +79,6 @@ export function spawnForTheTest( isGone: () => child.pid === undefined || child.exitCode !== null || child.signalCode !== null, signal: (name) => child.kill(name), }; - finish.registered(() => ended(ending, finish.stoppedWithinMs)); + finish.registered(() => endedWithin(ending, finish.stoppedWithinMs)); return child; } diff --git a/packages/operations/src/process-testing/test-run-folder.test.ts b/packages/operations/src/process-testing/test-run-folder.test.ts new file mode 100644 index 000000000..b9476e905 --- /dev/null +++ b/packages/operations/src/process-testing/test-run-folder.test.ts @@ -0,0 +1,130 @@ +import { spawnSync } from 'node:child_process'; +import { once } from 'node:events'; +import { existsSync, mkdirSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { basename, dirname, join } from 'node:path'; + +import { describe, expect, it } from 'vitest'; + +import type { RunningProcess } from './process-endings.ts'; +import { spawnForTheTest } from './test-processes.ts'; +import theFolderOfTheTestRun, { aFolderOfTheTestRun, type RunningIn } from './test-run-folder.ts'; + +const run = { name: 'probe' }; + +function tmpdirOfAProcessItStarts(): string { + return spawnSync(process.execPath, ['--eval', "process.stdout.write(require('node:os').tmpdir())"], { + encoding: 'utf8', + }).stdout; +} + +function leftIn(folder: string, ...names: readonly string[]): void { + for (const name of names) { + mkdirSync(join(folder, name)); + } +} + +describe('the temporary folder of a test run', () => { + it('is made in the temporary folder it is set up in, named for the run, and is the temporary folder of the run and of what it starts until its teardown removes it', async () => { + const outside = tmpdir(); + const teardown = theFolderOfTheTestRun(run); + const folder = tmpdir(); + const ofAProcessItStarts = tmpdirOfAProcessItStarts(); + + await teardown(); + + expect([dirname(folder), basename(folder).startsWith('auto-runtime-test-run-probe-')]).toEqual([outside, true]); + expect([ofAProcessItStarts, existsSync(folder), tmpdir()]).toEqual([folder, false, outside]); + }); + + it('fails the run, naming in order what its tests left there, and is removed all the same', async () => { + const teardown = theFolderOfTheTestRun(run); + const folder = tmpdir(); + leftIn(folder, 'left-second'); + writeFileSync(join(folder, 'left-first'), ''); + + await expect(teardown()).rejects.toThrow( + new Error('The tests of the run did not remove what they left in its temporary folder: left-first, left-second.'), + ); + expect(existsSync(folder)).toBe(false); + }); + + it('is its own in each of two runs set up at once, so the teardown of one names nothing the other left', async () => { + const outside = tmpdir(); + const first = theFolderOfTheTestRun(run); + const ofTheFirst = tmpdir(); + process.env['TMPDIR'] = outside; + const second = theFolderOfTheTestRun(run); + leftIn(ofTheFirst, 'left-by-the-first'); + + await second(); + + await expect(first()).rejects.toThrow(/: left-by-the-first\.$/u); + }); +}); + +const stopsWhenTold = [ + "process.on('SIGTERM', () => { process.stdout.write('stopped'); process.exit(0); });", + 'setInterval(() => {}, 1000);', + "process.stdout.write('ready ');", +].join('\n'); + +interface Probe { + readonly ready: Promise; + readonly printed: () => string; + readonly closed: Promise; + readonly runningIn: RunningIn; + readonly lookedIn: () => readonly (readonly [string, boolean])[]; +} + +function aProbeRunningIn(home: string): Probe { + const child = spawnForTheTest(process.execPath, ['--eval', stopsWhenTold], { + stdio: ['ignore', 'pipe', 'inherit'], + }); + const printed = { text: '' }; + const ready = Promise.withResolvers(); + child.stdout?.setEncoding('utf8').on('data', (chunk: string) => { + printed.text += chunk; + ready.resolve(); + }); + const running: RunningProcess = { + name: 'a probe', + pid: Number(child.pid), + isStillRunning: () => child.exitCode === null && child.signalCode === null, + }; + const lookedIn: (readonly [string, boolean])[] = []; + return { + ready: ready.promise, + printed: () => printed.text, + closed: once(child, 'close'), + runningIn: (entry) => { + lookedIn.push([basename(entry), existsSync(entry)]); + return basename(entry) === home ? [running] : []; + }, + lookedIn: () => lookedIn, + }; +} + +describe('a process still running in what the tests of a run left in its folder', () => { + it('is ended as its teardown looks, before the folder is removed, and named with what it ran in', async () => { + const probe = aProbeRunningIn('a-home'); + const teardown = aFolderOfTheTestRun(probe.runningIn)(run); + leftIn(tmpdir(), 'a-home', 'a-folder'); + await probe.ready; + + await expect(teardown()).rejects.toThrow( + new Error( + 'The tests of the run did not remove what they left in its temporary folder: a-folder, a-home. The teardown ended a probe, which still ran in a-home.', + ), + ); + + expect(await probe.closed).toEqual([0, null]); + expect([probe.printed(), probe.lookedIn()]).toEqual([ + 'ready stopped', + [ + ['a-folder', true], + ['a-home', true], + ], + ]); + }); +}); diff --git a/packages/operations/src/process-testing/test-run-folder.ts b/packages/operations/src/process-testing/test-run-folder.ts new file mode 100644 index 000000000..963d8c8ab --- /dev/null +++ b/packages/operations/src/process-testing/test-run-folder.ts @@ -0,0 +1,56 @@ +import { mkdtempSync, readdirSync, rmSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; + +import { endedWhileItRuns, type RunningProcess } from './process-endings.ts'; + +export type RunningIn = (entry: string) => readonly RunningProcess[]; + +interface TestRun { + readonly name: string; +} + +type TestRunTeardown = () => Promise; + +interface RunningInAnEntry { + readonly entry: string; + readonly running: RunningProcess; +} + +const nothingRuns: RunningIn = () => []; + +function leftWords(left: readonly string[], ended: readonly RunningInAnEntry[]): string { + return [ + `The tests of the run did not remove what they left in its temporary folder: ${left.join(', ')}.`, + ...ended.map(({ entry, running }) => `The teardown ended ${running.name}, which still ran in ${entry}.`), + ].join(' '); +} + +async function checkedAndEnded(folder: string, runningIn: RunningIn): Promise { + const left = readdirSync(folder).toSorted(); + const ended = left.flatMap((entry) => + runningIn(join(folder, entry)).map((running): RunningInAnEntry => ({ entry, running })), + ); + await Promise.all(ended.map(({ running }) => endedWhileItRuns(running))); + if (left.length > 0) { + throw new Error(leftWords(left, ended)); + } +} + +export function aFolderOfTheTestRun(runningIn: RunningIn = nothingRuns): (run: TestRun) => TestRunTeardown { + return ({ name }) => { + const madeIn = tmpdir(); + const folder = mkdtempSync(join(madeIn, `auto-runtime-test-run-${name}-`)); + process.env['TMPDIR'] = folder; + return async () => { + try { + await checkedAndEnded(folder, runningIn); + } finally { + rmSync(folder, { recursive: true, force: true }); + process.env['TMPDIR'] = madeIn; + } + }; + }; +} + +export default aFolderOfTheTestRun(); diff --git a/packages/operations/src/testing/index.ts b/packages/operations/src/testing/index.ts index daec52a56..716cc5d0e 100644 --- a/packages/operations/src/testing/index.ts +++ b/packages/operations/src/testing/index.ts @@ -33,4 +33,5 @@ export { type Unread, } from '../content-testing/content-samples.ts'; export { internalTerms, internalTermsIn } from '../plain-language/internal-terms.ts'; -export { endWithTheTest, spawnForTheTest, stoppedWithinMs, type PidReader } from '../process-testing/test-processes.ts'; +export { stoppedWithinMs } from '../process-testing/process-endings.ts'; +export { endWithTheTest, spawnForTheTest, type PidReader } from '../process-testing/test-processes.ts'; diff --git a/turbo.json b/turbo.json index bad1476f9..2111f9fcc 100644 --- a/turbo.json +++ b/turbo.json @@ -8,6 +8,8 @@ "tsconfig.base.json", ".oxlintrc.json", "vitest.shared.ts", + "packages/operations/src/process-testing/test-run-folder.ts", + "packages/operations/src/process-testing/process-endings.ts", ".nvmrc", "pnpm-lock.yaml" ], diff --git a/vitest.shared.ts b/vitest.shared.ts index cacaaf0dd..939205141 100644 --- a/vitest.shared.ts +++ b/vitest.shared.ts @@ -1,7 +1,12 @@ +import { join } from 'node:path'; + import { defineConfig } from 'vitest/config'; +const theFolderOfTheTestRun = join(import.meta.dirname, 'packages/operations/src/process-testing/test-run-folder.ts'); + export const sharedConfig = defineConfig({ test: { + globalSetup: [theFolderOfTheTestRun], testTimeout: 60_000, hookTimeout: 60_000, experimental: { From 9206dc38c18c420721c85063dae47ccafdddd088 Mon Sep 17 00:00:00 2001 From: Rami Date: Wed, 21 Oct 2026 23:17:21 +0100 Subject: [PATCH 4/4] test(server): the suite's teardown ends a background runtime a test left running, and names it A runtime a relay starts with start --background --instance runs in a process group of its own, and runtimesStartedInTheBackgroundEndedWithTheTest ends it in onTestFinished, which a worker that dies or a hook that times out never reaches, so under load such a runtime ran on after the run. The server's vitest.config.ts now names src/testing/test-run/test-run-folder.ts in place of the shared global setup, through sharedConfigSetUpBy of vitest.shared.ts: aFolderOfTheTestRun over runtimesStillRunningInTheBackgroundIn, which reads the runtime.log of each home left in the run's folder for a start with no end, as the hook of a test does. A process is ended, SIGTERM and SIGKILL after stoppedWithinMs, only while ps shows it runs under the instance the log named, and the failure of the run names it with its home. The reading of the log moves to runtimes-in-the-background.ts, which imports nothing of Vitest, and the hook of a test ends what it finds with endedWhileItRuns of @on.auto/operations/testing. @on.auto/operations/test-run-folder is the entry a package's own global setup imports. The check that a process the log says ended, or one that runs as another, is left alone is the teardown's test now, beside the reading the hook and the teardown share, and the relay's test keeps to the relay. Co-Authored-By: Claude Fable 5.1 --- packages/operations/README.md | 2 +- packages/operations/package.json | 1 + packages/operations/src/testing/index.ts | 2 +- packages/server/README.md | 2 +- .../src/agent-relay/agent-relay.test.ts | 38 +--------- .../testing/processes/background-runtimes.ts | 64 +--------------- .../processes/runtimes-in-the-background.ts | 46 ++++++++++++ .../testing/test-run/test-run-folder.test.ts | 74 +++++++++++++++++++ .../src/testing/test-run/test-run-folder.ts | 5 ++ packages/server/vitest.config.ts | 7 +- vitest.shared.ts | 38 +++++----- 11 files changed, 160 insertions(+), 119 deletions(-) create mode 100644 packages/server/src/testing/processes/runtimes-in-the-background.ts create mode 100644 packages/server/src/testing/test-run/test-run-folder.test.ts create mode 100644 packages/server/src/testing/test-run/test-run-folder.ts diff --git a/packages/operations/README.md b/packages/operations/README.md index cd4cfec71..97a3f1deb 100644 --- a/packages/operations/README.md +++ b/packages/operations/README.md @@ -216,4 +216,4 @@ The digest is Node's, behind the one module of the repository for it: the digest `spawnForTheTest(command, args, options)` (`src/process-testing/test-processes.ts`) is how a test of the repository starts a process it runs itself. It starts it as `spawn` of `node:child_process` does and registers its end with `onTestFinished` of Vitest, the hook that runs once the test finished, whether it passed, failed or timed out: a process still there is sent SIGTERM, on which the server stops, and SIGKILL if it is still there `stoppedWithinMs`, 10 s, later, a bound the server's test of its longest stop holds that stop under, and well within the 60 s a hook may take. `endWithTheTest(pid)` ends in the same way, by its pid, a process the test did not start itself, such as a server started in the background, and signals no pid that `kill(pid, 0)` finds gone or another user's; it cannot tell a pid the system gave to another process of the same user since. `endWithTheTest(reader)` takes a `PidReader` in place of the pid, registered before the process is started: once the test finished, it reads the pid with `read()` again and again while `stillToCome()` holds, for at most `readWithinMs`, and ends the pid it read, if any. A worker of the test runner that dies runs no hook, so it ends nothing. -`aFolderOfTheTestRun(runningIn)` (`src/process-testing/test-run-folder.ts`) is the global setup of the tests of every package, which `vitest.shared.ts` names with its default, `aFolderOfTheTestRun()`. Each run makes a folder of its own in the temporary folder, `auto-runtime-test-run--`, and sets `TMPDIR` to it before any worker starts, so `tmpdir()` of `node:os` answers it in every worker and in every process a test starts with the environment it inherits, and two runs at once have two folders. Its teardown, which a worker that dies, a hook that times out or a child that is killed does not keep from running, lists what the tests left there, ends each `RunningProcess` that `runningIn(entry)` finds still running in an entry, with SIGTERM and with SIGKILL if `isStillRunning()` still holds `stoppedWithinMs` later, sending a signal only while it holds, fails the run naming each entry and each process it ended, and removes the folder whatever happened. It imports nothing of Vitest, since a global setup runs in Vitest's own process, where the hooks of a test cannot. +`aFolderOfTheTestRun(runningIn)` (`src/process-testing/test-run-folder.ts`) is the global setup of the tests of every package, which `vitest.shared.ts` names with its default, `aFolderOfTheTestRun()`. Each run makes a folder of its own in the temporary folder, `auto-runtime-test-run--`, and sets `TMPDIR` to it before any worker starts, so `tmpdir()` of `node:os` answers it in every worker and in every process a test starts with the environment it inherits, and two runs at once have two folders. Its teardown, which a worker that dies, a hook that times out or a child that is killed does not keep from running, lists what the tests left there, ends each `RunningProcess` that `runningIn(entry)` finds still running in an entry, with SIGTERM and with SIGKILL if `isStillRunning()` still holds `stoppedWithinMs` later, sending a signal only while it holds, fails the run naming each entry and each process it ended, and removes the folder whatever happened. A package whose own processes may outlive their test gives it a `runningIn` of its own, as the server does for the runtimes its tests start in the background, through `@on.auto/operations/test-run-folder`. That is an entry of its own because a global setup runs in Vitest's own process and not in a worker, so neither it nor anything it imports takes Vitest, with whose hooks `@on.auto/operations/testing` registers what it ends; `endedWhileItRuns(running)` of `@on.auto/operations/testing` is the same end for the hook of a test, over the same `RunningProcess`. diff --git a/packages/operations/package.json b/packages/operations/package.json index e15609621..41a1650e9 100644 --- a/packages/operations/package.json +++ b/packages/operations/package.json @@ -6,6 +6,7 @@ "type": "module", "exports": { ".": "./src/index.ts", + "./test-run-folder": "./src/process-testing/test-run-folder.ts", "./testing": "./src/testing/index.ts" }, "scripts": { diff --git a/packages/operations/src/testing/index.ts b/packages/operations/src/testing/index.ts index 716cc5d0e..4fdf3817c 100644 --- a/packages/operations/src/testing/index.ts +++ b/packages/operations/src/testing/index.ts @@ -33,5 +33,5 @@ export { type Unread, } from '../content-testing/content-samples.ts'; export { internalTerms, internalTermsIn } from '../plain-language/internal-terms.ts'; -export { stoppedWithinMs } from '../process-testing/process-endings.ts'; +export { endedWhileItRuns, stoppedWithinMs, type RunningProcess } from '../process-testing/process-endings.ts'; export { endWithTheTest, spawnForTheTest, type PidReader } from '../process-testing/test-processes.ts'; diff --git a/packages/server/README.md b/packages/server/README.md index 71d1d74aa..f647af354 100644 --- a/packages/server/README.md +++ b/packages/server/README.md @@ -22,6 +22,6 @@ A test that waits for what a workflow's step waits for before the step's timeout The composition of main, started in the process with a shutdown timeout longer than the test, stops with an idle keep-alive connection open after a workflow ran, and while a function a workflow called still runs, without waiting for that timeout (`src/lifecycle/composed-stop.test.ts`); so no test of a spawned process holds it to how long it takes: `src/main.test.ts` holds main to its exit code alone, `src/workflows/workflow-lifecycle.test.ts` to its exit code and what it says at its start, and `src/workflows/workflow-shutdown.test.ts` to its exit code, to leaving nothing running and to taking the call it was stopped in up again once started again. -The tests start the servers, the `auto-runtime` command and the development runner they run through `spawnForTheTest` of `@on.auto/operations/testing` (`src/testing/processes/spawned-server.ts`, `runtime-command.ts` and `development-process.ts`), which ends each once its test finished, whether it passed, failed or timed out: with SIGTERM, on which the server stops, and with SIGKILL if it is still there `stoppedWithinMs`, 10 s, later; `src/lifecycle/shutdown.test.ts` holds the server's longest stop, `longestStopMs`, under that bound. Two tests start their processes through the code they test instead: `src/development/children.test.ts` through the runner's `startChild`, which starts each child in a process group of its own, and ends each with SIGKILL once its test finished, and `src/background-start/started-in-the-background.test.ts` through `startInBackgroundOf`, whose recording entry ends itself once it has written what the test reads, and whose other entry, `src/testing/entries/stay-until-signalled.ts`, runs until the test signals it through the handle the start hands back, with `endWithTheTest` registered for its pid. The processes the helper starts are attached to the test's worker, in its process group, so a signal to that group, as Ctrl+C in a terminal sends, stops them too; a worker that dies alone runs no hook, and leaves them running until they next write to the output it held. A runtime started in the background, `start --background --instance `, runs in a process group of its own and writes to its log, so nothing of the test's ends it with the test. `src/agent-relay/agent-relay.test.ts` starts one alone, in the test that proves it runs on outside the process group of the agent that started it, and starts a second one the same way once the first was stopped; another has the relay start one as it starts, and a second once the test stopped the first and the relay's grace passed; two of its other tests start the runtime in a terminal, attached, and another starts none, since another program holds its port. `src/agent-relay/relay-replacement.test.ts` starts one in place of a runtime in a terminal that the test stops, and one through the relay of `src/testing/entries/relay-with-short-bounds.ts`, which the test freezes with SIGSTOP and the relay stops and replaces; its third test freezes a runtime in a terminal, which the relay leaves alone, and lets it go on with SIGCONT before it stops it with Ctrl+C. Each test that lets a relay start a runtime registers, before anything starts, `runtimesStartedInTheBackgroundEndedWithTheTest` of `src/testing/processes/background-runtimes.ts`. Once the test finished, it reads the lines the relay writes to the `runtime.log` of the test's home as it acts, `Auto Runtime was started in the background as process , instance ` and `Auto Runtime, process , ended`, and ends each process started and not ended: with SIGTERM, and with SIGKILL if it is still there `stoppedWithinMs` later, each sent only while its command line, as `ps` reads it, still carries that instance, and the process counts as gone once it no longer does. It asks the port nothing, so a runtime that answers slower than a probe's 2 s, or one whose relay was ended while it held the starting lock, is ended all the same, and a pid used again by another process is not signalled. +The tests start the servers, the `auto-runtime` command and the development runner they run through `spawnForTheTest` of `@on.auto/operations/testing` (`src/testing/processes/spawned-server.ts`, `runtime-command.ts` and `development-process.ts`), which ends each once its test finished, whether it passed, failed or timed out: with SIGTERM, on which the server stops, and with SIGKILL if it is still there `stoppedWithinMs`, 10 s, later; `src/lifecycle/shutdown.test.ts` holds the server's longest stop, `longestStopMs`, under that bound. Two tests start their processes through the code they test instead: `src/development/children.test.ts` through the runner's `startChild`, which starts each child in a process group of its own, and ends each with SIGKILL once its test finished, and `src/background-start/started-in-the-background.test.ts` through `startInBackgroundOf`, whose recording entry ends itself once it has written what the test reads, and whose other entry, `src/testing/entries/stay-until-signalled.ts`, runs until the test signals it through the handle the start hands back, with `endWithTheTest` registered for its pid. The processes the helper starts are attached to the test's worker, in its process group, so a signal to that group, as Ctrl+C in a terminal sends, stops them too; a worker that dies alone runs no hook, and leaves them running until they next write to the output it held. A runtime started in the background, `start --background --instance `, runs in a process group of its own and writes to its log, so nothing of the test's ends it with the test. `src/agent-relay/agent-relay.test.ts` starts one alone, in the test that proves it runs on outside the process group of the agent that started it, and starts a second one the same way once the first was stopped; another has the relay start one as it starts, and a second once the test stopped the first and the relay's grace passed; two of its other tests start the runtime in a terminal, attached, and another starts none, since another program holds its port. `src/agent-relay/relay-replacement.test.ts` starts one in place of a runtime in a terminal that the test stops, and one through the relay of `src/testing/entries/relay-with-short-bounds.ts`, which the test freezes with SIGSTOP and the relay stops and replaces; its third test freezes a runtime in a terminal, which the relay leaves alone, and lets it go on with SIGCONT before it stops it with Ctrl+C. Each test that lets a relay start a runtime registers, before anything starts, `runtimesStartedInTheBackgroundEndedWithTheTest` of `src/testing/processes/background-runtimes.ts`. Once the test finished, it reads the lines the relay writes to the `runtime.log` of the test's home as it acts, `Auto Runtime was started in the background as process , instance ` and `Auto Runtime, process , ended`, and ends each process started and not ended: with SIGTERM, and with SIGKILL if it is still there `stoppedWithinMs` later, each sent only while its command line, as `ps` reads it, still carries that instance, and the process counts as gone once it no longer does. It asks the port nothing, so a runtime that answers slower than a probe's 2 s, or one whose relay was ended while it held the starting lock, is ended all the same, and a pid used again by another process is not signalled. The suite's teardown does the same with what a test left in the temporary folder of the run, as when its worker died or its hook timed out: `vitest.config.ts` names `src/testing/test-run/test-run-folder.ts` in place of the shared global setup, `aFolderOfTheTestRun` of `@on.auto/operations/test-run-folder` over `runtimesStillRunningInTheBackgroundIn` of `src/testing/processes/runtimes-in-the-background.ts`, which reads the `runtime.log` of each home left there, so a runtime still running in one is ended once the run ends and named, with its home, in the failure of the run. The relay of `auto-runtime mcp` holds its bounds on Effect's test clock, in `src/agent-relay/relay-watch.test.ts`, `src/agent-relay/relay-losses.test.ts` and `src/runtime-recovery/runtime-stop.test.ts`; `src/agent-relay/relay-replacement.test.ts` runs a relay on real time with a silence of 1 s and a stop of 0.5 s in place of 15 s and 9.1 s, and holds what happens, never how long it took. The waits of its start, for a runtime it starts to answer, for an older one to stop and for one that restarts to come back, probe the port every 100 ms on Effect's clock and end with the relay: a start the relay makes while it relays ends at once when the agent leaves, aborts the probe in flight and frees the starting lock. `src/background-start/background-start.test.ts` holds them on the test clock, by the probes its fake answers and by the time they take, and `src/local-runtime/runtime-commands.test.ts` holds the waits of `stop` and `restart`, which run the same way. Some tests keep a bound on real time on purpose. `src/lifecycle/shutdown.test.ts` holds a request still running to being cut off no sooner than the shutdown timeout less 50 ms, a lower bound a slow machine only lengthens. `src/computation/computation-deadline.test.ts` holds a run that never ends by itself to being refused no sooner than its 10 s deadline less the slack of a timer, and a read of the definition and a run on another worker to answering before that deadline, a race the server wins by the pool's 10 s on the real clock, which the pool keeps and no clock of the server reaches; its saves check within 30 s through `checkingWithin`. `src/reasoning/large-answers-on-stores.test.ts` holds a reasoning run whose 25 calls each answer 1 MB to reading every answer back within `toolBounds.runMs`, the 600 s a run that calls tools may take, on the real clock on purpose, since that is the bound such a run meets in the server. `src/telemetry/start-telemetry.test.ts` holds the shutdown of telemetry whose collector cannot be reached to within 1.5 s, since that shutdown races its exports against a timer of 1 s of the platform, for which the exporters take no clock. `src/development/source-changes.test.ts` saves a file again every 500 ms until the watcher reports it, and `src/workflows/workflow-lifecycle.test.ts` waits until the timers of a stopped server's runs are due before it starts the next process, waits a slow machine only lengthens. diff --git a/packages/server/src/agent-relay/agent-relay.test.ts b/packages/server/src/agent-relay/agent-relay.test.ts index f8bbc57d7..de154ae6e 100644 --- a/packages/server/src/agent-relay/agent-relay.test.ts +++ b/packages/server/src/agent-relay/agent-relay.test.ts @@ -1,15 +1,12 @@ import { spawnSync } from 'node:child_process'; -import { existsSync, mkdirSync, writeFileSync } from 'node:fs'; +import { existsSync } from 'node:fs'; import { createServer } from 'node:http'; import { join } from 'node:path'; import { describe, expect, it, onTestFinished } from 'vitest'; import { restartMarkIn } from '../local-runtime/local-control.ts'; -import { - runtimesStartedInTheBackgroundEndedWithTheTest, - startedInTheBackgroundIn, -} from '../testing/processes/background-runtimes.ts'; +import { runtimesStartedInTheBackgroundEndedWithTheTest } from '../testing/processes/background-runtimes.ts'; import { aPortNothingListensOn } from '../testing/processes/free-port.ts'; import { homeOfTheTest, @@ -219,34 +216,3 @@ describe('an agent whose runtime port another program holds', () => { expect(await relay.exited).toBe(0); }); }); - -describe('the end of the runtimes a relay started in the home of a test', () => { - it('leaves alone a process the log of that home says has ended, and one whose command is no longer the runtime started', async () => { - const home = homeOfTheTest(homePrefix); - const another = spawnRuntimeCommand([], { home: homeOfTheTest(homePrefix), port: await aPortNothingListensOn() }); - await until(() => { - expect(readiesIn(another.stdout())).toBe(1); - }); - const instance = '0199a3c4-7d2e-7c1a-9b3f-2f1e0d9c8b7a'; - mkdirSync(join(home, '.auto-runtime'), { recursive: true }); - writeFileSync( - join(home, '.auto-runtime', 'runtime.log'), - [ - `Auto Runtime was started in the background as process ${String(another.pid)}, instance ${instance}`, - `Auto Runtime was started in the background as process ${process.pid}, instance ${instance}`, - `Auto Runtime, process ${process.pid}, ended`, - '', - ].join('\n'), - ); - onTestFinished(() => { - expect(() => process.kill(Number(another.pid), 0)).not.toThrow(); - }); - - runtimesStartedInTheBackgroundEndedWithTheTest(home); - - expect(startedInTheBackgroundIn(home)).toEqual([ - { pid: another.pid, instance }, - { pid: process.pid, instance }, - ]); - }); -}); diff --git a/packages/server/src/testing/processes/background-runtimes.ts b/packages/server/src/testing/processes/background-runtimes.ts index f376b9471..ffa5b81ca 100644 --- a/packages/server/src/testing/processes/background-runtimes.ts +++ b/packages/server/src/testing/processes/background-runtimes.ts @@ -1,74 +1,16 @@ -import { spawnSync } from 'node:child_process'; -import { readFileSync } from 'node:fs'; -import { setTimeout } from 'node:timers/promises'; - -import { stoppedWithinMs } from '@on.auto/operations/testing'; +import { endedWhileItRuns } from '@on.auto/operations/testing'; import { onTestFinished } from 'vitest'; -import { logFileIn } from '../../local-runtime/local-control.ts'; import { until } from '../waiting/until.ts'; +import { runtimesStillRunningInTheBackgroundIn, startedInTheBackgroundIn } from './runtimes-in-the-background.ts'; export interface StartedInTheBackground { readonly pid: () => Promise; } -function logIn(home: string): string { - try { - return readFileSync(logFileIn(home), 'utf8'); - } catch { - return ''; - } -} - -interface Started { - readonly pid: number; - readonly instance: string; -} - -export function startedInTheBackgroundIn(home: string): readonly Started[] { - return [ - ...logIn(home).matchAll(/Auto Runtime was started in the background as process (\d+), instance ([\da-f-]+)/gu), - ].map(([, pid, instance]: readonly string[]) => ({ pid: Number(pid), instance: String(instance) })); -} - -function endedIn(home: string): readonly number[] { - return [...logIn(home).matchAll(/Auto Runtime, process (\d+), ended/gu)].map(([, pid]: readonly string[]) => - Number(pid), - ); -} - -const lookEveryMs = 50; - -function stillTheOneStarted({ pid, instance }: Started): boolean { - return spawnSync('ps', ['-o', 'command=', '-p', String(pid)], { encoding: 'utf8' }).stdout.includes(instance); -} - -function signalledWhileItRuns(started: Started, signal: NodeJS.Signals): void { - if (stillTheOneStarted(started)) { - process.kill(started.pid, signal); - } -} - -async function goneBy(started: Started, deadline: number): Promise { - if (!stillTheOneStarted(started) || performance.now() >= deadline) { - return; - } - await setTimeout(lookEveryMs); - await goneBy(started, deadline); -} - -async function endedWhileItRuns(started: Started): Promise { - signalledWhileItRuns(started, 'SIGTERM'); - await goneBy(started, performance.now() + stoppedWithinMs); - signalledWhileItRuns(started, 'SIGKILL'); - await goneBy(started, performance.now() + stoppedWithinMs); -} - export function runtimesStartedInTheBackgroundEndedWithTheTest(home: string): StartedInTheBackground { onTestFinished(async () => { - const ended = endedIn(home); - const living = startedInTheBackgroundIn(home).filter(({ pid }) => !ended.includes(pid)); - await Promise.all(living.map((started) => endedWhileItRuns(started))); + await Promise.all(runtimesStillRunningInTheBackgroundIn(home).map((running) => endedWhileItRuns(running))); }); const read = { starts: 0 }; return { diff --git a/packages/server/src/testing/processes/runtimes-in-the-background.ts b/packages/server/src/testing/processes/runtimes-in-the-background.ts new file mode 100644 index 000000000..8a215d74c --- /dev/null +++ b/packages/server/src/testing/processes/runtimes-in-the-background.ts @@ -0,0 +1,46 @@ +import { spawnSync } from 'node:child_process'; +import { readFileSync } from 'node:fs'; + +import type { RunningProcess } from '@on.auto/operations/testing'; + +import { logFileIn } from '../../local-runtime/local-control.ts'; + +interface Started { + readonly pid: number; + readonly instance: string; +} + +function logIn(home: string): string { + try { + return readFileSync(logFileIn(home), 'utf8'); + } catch { + return ''; + } +} + +export function startedInTheBackgroundIn(home: string): readonly Started[] { + return [ + ...logIn(home).matchAll(/Auto Runtime was started in the background as process (\d+), instance ([\da-f-]+)/gu), + ].map(([, pid, instance]: readonly string[]) => ({ pid: Number(pid), instance: String(instance) })); +} + +function endedIn(home: string): readonly number[] { + return [...logIn(home).matchAll(/Auto Runtime, process (\d+), ended/gu)].map(([, pid]: readonly string[]) => + Number(pid), + ); +} + +function stillTheOneStarted({ pid, instance }: Started): boolean { + return spawnSync('ps', ['-o', 'command=', '-p', String(pid)], { encoding: 'utf8' }).stdout.includes(instance); +} + +export function runtimesStillRunningInTheBackgroundIn(home: string): readonly RunningProcess[] { + const ended = endedIn(home); + return startedInTheBackgroundIn(home) + .filter((started) => !ended.includes(started.pid) && stillTheOneStarted(started)) + .map((started) => ({ + name: `Auto Runtime, process ${started.pid}, instance ${started.instance}`, + pid: started.pid, + isStillRunning: () => stillTheOneStarted(started), + })); +} diff --git a/packages/server/src/testing/test-run/test-run-folder.test.ts b/packages/server/src/testing/test-run/test-run-folder.test.ts new file mode 100644 index 000000000..77b91d7c3 --- /dev/null +++ b/packages/server/src/testing/test-run/test-run-folder.test.ts @@ -0,0 +1,74 @@ +import { existsSync, mkdirSync, mkdtempSync, realpathSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { basename, join } from 'node:path'; + +import { endWithTheTest, spawnForTheTest } from '@on.auto/operations/testing'; +import { Function } from 'effect'; +import { describe, expect, it } from 'vitest'; + +import { loggedTo, startInBackgroundOf } from '../../background-start/background-process.ts'; +import { logFileIn, runtimeHomeIn } from '../../local-runtime/local-control.ts'; +import { until } from '../waiting/until.ts'; +import theFolderOfTheServersTestRun from './test-run-folder.ts'; + +const stayingEntry = join(import.meta.dirname, '../entries/stay-until-signalled.ts'); + +const run = { name: 'probe' }; + +const anInstanceThatEnded = '5c8e2a0f-3d14-4b6e-9a7c-1f2b3c4d5e6f'; + +const anInstanceNoProcessRunsAs = '9d0c1b2a-4e3f-4a5b-8c6d-7e8f9a0b1c2d'; + +const leftWords = 'The tests of the run did not remove what they left in its temporary folder:'; + +function aHomeLeftIn(folder: string): string { + return realpathSync(mkdtempSync(join(folder, 'auto-runtime-left-'))); +} + +describe("the teardown of the temporary folder of a run of the server's tests", () => { + it('ends a runtime a test left running in the background in a home it left there, and names the runtime with the home', async () => { + const teardown = theFolderOfTheServersTestRun(run); + const home = aHomeLeftIn(tmpdir()); + const started = startInBackgroundOf({ + execPath: process.execPath, + execArgv: [], + entry: stayingEntry, + home, + environment: { PATH: process.env['PATH'], NODE_V8_COVERAGE: process.env['NODE_V8_COVERAGE'] }, + report: Function.constVoid, + log: loggedTo(home, Function.constVoid), + })(); + const pid = Number(started.pid); + endWithTheTest(pid); + await until(() => existsSync(join(runtimeHomeIn(home), 'started.json')), Boolean); + + await expect(teardown()).rejects.toThrow( + new Error( + `${leftWords} ${basename(home)}. The teardown ended Auto Runtime, process ${pid}, instance ${started.instance}, which still ran in ${basename(home)}.`, + ), + ); + await started.ended; + + expect(started.endedAt()).toBeGreaterThanOrEqual(started.startedAt); + }); + + it('signals no process that a log names as ended, or that runs as another than the instance it names, and names what was left alone', async () => { + const another = spawnForTheTest(process.execPath, ['--eval', 'setInterval(() => {}, 1000);']); + const teardown = theFolderOfTheServersTestRun(run); + const home = aHomeLeftIn(tmpdir()); + mkdirSync(runtimeHomeIn(home)); + writeFileSync( + logFileIn(home), + [ + `Auto Runtime was started in the background as process ${process.pid}, instance ${anInstanceThatEnded}`, + `Auto Runtime, process ${process.pid}, ended`, + `Auto Runtime was started in the background as process ${Number(another.pid)}, instance ${anInstanceNoProcessRunsAs}`, + ].join('\n'), + ); + writeFileSync(join(tmpdir(), 'a-file-left'), ''); + + await expect(teardown()).rejects.toThrow(new Error(`${leftWords} a-file-left, ${basename(home)}.`)); + + expect([another.exitCode, another.signalCode]).toEqual([null, null]); + }); +}); diff --git a/packages/server/src/testing/test-run/test-run-folder.ts b/packages/server/src/testing/test-run/test-run-folder.ts new file mode 100644 index 000000000..85b8cbc28 --- /dev/null +++ b/packages/server/src/testing/test-run/test-run-folder.ts @@ -0,0 +1,5 @@ +import { aFolderOfTheTestRun } from '@on.auto/operations/test-run-folder'; + +import { runtimesStillRunningInTheBackgroundIn } from '../processes/runtimes-in-the-background.ts'; + +export default aFolderOfTheTestRun(runtimesStillRunningInTheBackgroundIn); diff --git a/packages/server/vitest.config.ts b/packages/server/vitest.config.ts index 7908619cd..e878eaf57 100644 --- a/packages/server/vitest.config.ts +++ b/packages/server/vitest.config.ts @@ -1,5 +1,8 @@ import { defineConfig, mergeConfig } from 'vitest/config'; -import { sharedConfig } from '../../vitest.shared.ts'; +import { sharedConfigSetUpBy } from '../../vitest.shared.ts'; -export default mergeConfig(sharedConfig, defineConfig({ test: { name: 'server' } })); +export default mergeConfig( + sharedConfigSetUpBy('src/testing/test-run/test-run-folder.ts'), + defineConfig({ test: { name: 'server' } }), +); diff --git a/vitest.shared.ts b/vitest.shared.ts index 939205141..730d7a920 100644 --- a/vitest.shared.ts +++ b/vitest.shared.ts @@ -4,21 +4,25 @@ import { defineConfig } from 'vitest/config'; const theFolderOfTheTestRun = join(import.meta.dirname, 'packages/operations/src/process-testing/test-run-folder.ts'); -export const sharedConfig = defineConfig({ - test: { - globalSetup: [theFolderOfTheTestRun], - testTimeout: 60_000, - hookTimeout: 60_000, - experimental: { - viteModuleRunner: false, +export function sharedConfigSetUpBy(globalSetup: string) { + return defineConfig({ + test: { + globalSetup, + testTimeout: 60_000, + hookTimeout: 60_000, + experimental: { + viteModuleRunner: false, + }, + coverage: { + provider: 'v8', + include: ['src/**/*.ts'], + exclude: ['src/**/*.test.ts'], + reporter: ['text', 'json-summary'], + autoAttachSubprocess: true, + thresholds: { 100: true, perFile: true }, + }, }, - coverage: { - provider: 'v8', - include: ['src/**/*.ts'], - exclude: ['src/**/*.test.ts'], - reporter: ['text', 'json-summary'], - autoAttachSubprocess: true, - thresholds: { 100: true, perFile: true }, - }, - }, -}); + }); +} + +export const sharedConfig = sharedConfigSetUpBy(theFolderOfTheTestRun);