From 711c07973a9da7359fe466697a906e19aa1cb101 Mon Sep 17 00:00:00 2001 From: David Cramer Date: Sat, 13 Jun 2026 09:55:03 -0700 Subject: [PATCH 1/2] test(qa): Add task-based example runner Move the checked-in example smoke into the dotagents QA skill and expose task-specific entry points for install/sync checks and optional Codex runtime proof. Keep smoke:examples as a compatibility alias while documenting the new qa:example workflow. --- package.json | 3 +- skills/dotagents-qa/SKILL.md | 32 ++--- skills/dotagents-qa/SOURCES.md | 2 +- skills/dotagents-qa/SPEC.md | 4 +- skills/dotagents-qa/references/claude.md | 2 +- skills/dotagents-qa/references/codex.md | 8 +- .../{core-smoke.md => core-agentic-qa.md} | 9 +- skills/dotagents-qa/references/cursor.md | 2 +- skills/dotagents-qa/references/opencode.md | 2 +- .../dotagents-qa/scripts/qa-example.mjs | 136 ++++++++++++------ 10 files changed, 129 insertions(+), 71 deletions(-) rename skills/dotagents-qa/references/{core-smoke.md => core-agentic-qa.md} (81%) rename scripts/smoke-examples.mjs => skills/dotagents-qa/scripts/qa-example.mjs (79%) diff --git a/package.json b/package.json index 47bbb24..c330364 100644 --- a/package.json +++ b/package.json @@ -11,7 +11,8 @@ "typecheck": "pnpm -r typecheck", "check": "pnpm lint && pnpm typecheck && pnpm test", "dev": "pnpm --filter @sentry/dotagents dev", - "smoke:examples": "pnpm build && node scripts/smoke-examples.mjs" + "smoke:examples": "pnpm qa:example", + "qa:example": "pnpm build && node skills/dotagents-qa/scripts/qa-example.mjs" }, "simple-git-hooks": { "pre-commit": "pnpm lint-staged" diff --git a/skills/dotagents-qa/SKILL.md b/skills/dotagents-qa/SKILL.md index 6e5fdcd..4e24dcc 100644 --- a/skills/dotagents-qa/SKILL.md +++ b/skills/dotagents-qa/SKILL.md @@ -34,9 +34,9 @@ Write down the QA target before running commands: broken state, user-scope state, or remote source. Read the targeted reference before running runtime-specific QA: -- Core install/sync example: [references/core-smoke.md](references/core-smoke.md) -- Codex custom agents: [references/codex.md](references/codex.md) -- Claude Code files/runtime caveats: [references/claude.md](references/claude.md) +- Core install/sync example: [references/core-agentic-qa.md](references/core-agentic-qa.md) +- Codex custom agents and runtime caveats: [references/codex.md](references/codex.md) +- Claude Code files and runtime caveats: [references/claude.md](references/claude.md) - Cursor files/runtime caveats: [references/cursor.md](references/cursor.md) - OpenCode files/runtime caveats: [references/opencode.md](references/opencode.md) @@ -119,7 +119,7 @@ su -s /bin/bash node -c ' pnpm install --frozen-lockfile pnpm build pnpm check - pnpm smoke:examples + pnpm qa:example ' ``` @@ -128,17 +128,17 @@ the check is already known to be unrelated. If `build` or `check` fails, treat that as a QA finding and stop before fixture work unless you are explicitly isolating the playbook mechanics. If skipped or bypassed, report why. -## 3. Prefer The Checked-In Smoke +## 3. Prefer The Checked-In Agentic QA -Use the checked-in example smoke for ordinary install/sync QA: +Use the checked-in example QA for ordinary install/sync QA: ```bash -pnpm smoke:examples +pnpm qa:example ``` -The smoke builds the local CLI, copies `examples/full/` to a temp project, and +The QA runner builds the local CLI, copies `examples/full/` to a temp project, and asserts: -- `install`, `list`, `doctor --fix`, and `doctor` +- `install`, `list`, `doctor --fix`, and `doctor` complete successfully - managed skills under `.agents/skills/` - Claude/Cursor skill symlink behavior - MCP files for Claude, Cursor, Codex, and OpenCode @@ -147,15 +147,15 @@ asserts: - generated subagent runtime files for Claude, Cursor, Codex, and OpenCode - `sync` repair after deleting representative generated files -Use `node scripts/smoke-examples.mjs --keep` when you need to inspect the temp -project; the script prints the retained path. +Use `node skills/dotagents-qa/scripts/qa-example.mjs all --keep` when you need +to inspect the temp project; the script prints the retained path. For paid Codex runtime proof of generated custom agents, run the runtime proof outside Docker only when the branch affects Codex custom agents or when reporting that Codex itself works: ```bash -node scripts/smoke-examples.mjs --codex-runtime --keep +node skills/dotagents-qa/scripts/qa-example.mjs codex-runtime --keep ``` That mode copies Codex auth/config into a temp `CODEX_HOME`, marks only the @@ -320,10 +320,10 @@ opencode --version ``` Codex subagents need real runtime proof before claiming Codex loaded them. Use -`node scripts/smoke-examples.mjs --codex-runtime --keep`; `codex debug -prompt-input` is not enough unless it visibly includes the generated agent name -or instructions. Project-scoped `.codex/agents/` load only when Codex trusts -the project. See [references/codex.md](references/codex.md). +`node skills/dotagents-qa/scripts/qa-example.mjs codex-runtime --keep`; +`codex debug prompt-input` is not enough unless it visibly includes the +generated agent name or instructions. Project-scoped `.codex/agents/` load only +when Codex trusts the project. See [references/codex.md](references/codex.md). Claude has no cheap dry-run skill list. If auth/network/model cost is acceptable, run a minimal non-interactive prompt from the temp project; diff --git a/skills/dotagents-qa/SOURCES.md b/skills/dotagents-qa/SOURCES.md index e2c548b..b1011d4 100644 --- a/skills/dotagents-qa/SOURCES.md +++ b/skills/dotagents-qa/SOURCES.md @@ -20,7 +20,7 @@ ## Decisions -- Keep the runtime skill as a guided QA playbook rather than a broad QA matrix or fixed smoke harness. +- Keep the runtime skill as a guided QA playbook rather than a broad QA matrix or fixed all-in-one harness. - Provide a repo-local Dockerfile for the sandbox/toolchain only; keep fixture design and assertions manual. - Keep the Dockerfile's prepared pnpm version aligned with the root `packageManager`. - Mount the host checkout read-only and do dependency install/build inside Docker to avoid host system-file writes and host binary assumptions. diff --git a/skills/dotagents-qa/SPEC.md b/skills/dotagents-qa/SPEC.md index 782c9d5..de18814 100644 --- a/skills/dotagents-qa/SPEC.md +++ b/skills/dotagents-qa/SPEC.md @@ -19,7 +19,7 @@ Out of scope: - release publishing - network-backed source testing for ordinary install-location changes - replacing focused Vitest regression coverage for logic bugs -- treating a fixed smoke script as sufficient QA for behavior-specific changes +- treating a fixed all-in-one QA task as sufficient for behavior-specific changes ## Runtime Contract @@ -36,6 +36,6 @@ Out of scope: ## Maintenance - Keep `SKILL.md` focused on guided, change-specific Docker QA. -- Keep examples editable and illustrative; do not turn the skill into a fixed test harness. +- Keep examples editable and illustrative; do not turn the skill into a fixed all-in-one harness. - Keep the Dockerfile pnpm version aligned with the root `packageManager`. - Update examples when dotagents changes config fields, generated file locations, or supported agents. diff --git a/skills/dotagents-qa/references/claude.md b/skills/dotagents-qa/references/claude.md index 72561dc..37f9796 100644 --- a/skills/dotagents-qa/references/claude.md +++ b/skills/dotagents-qa/references/claude.md @@ -4,7 +4,7 @@ Use this reference when changes affect Claude Code skill symlinks, `.claude/sett ## File-Level Checks -The core smoke asserts: +The core agentic QA asserts: - `.claude/skills` is a symlink to `.agents/skills` - `.mcp.json` exists for Claude MCP config diff --git a/skills/dotagents-qa/references/codex.md b/skills/dotagents-qa/references/codex.md index 7e7c294..311f069 100644 --- a/skills/dotagents-qa/references/codex.md +++ b/skills/dotagents-qa/references/codex.md @@ -4,7 +4,7 @@ Use this reference when changes affect Codex config generation, `.codex/agents/* ## File-Level Checks -The core smoke asserts: +The core agentic QA asserts: - `.codex/config.toml` exists for MCP config - `.codex/agents/code-reviewer.toml` exists @@ -19,10 +19,10 @@ These checks prove dotagents wrote the expected files. They do not prove Codex l Run the paid runtime proof when the branch affects Codex custom agents or when reporting that Codex itself works: ```bash -node scripts/smoke-examples.mjs --codex-runtime --keep +node skills/dotagents-qa/scripts/qa-example.mjs codex-runtime --keep ``` -The script: +The task: - copies `examples/full/` to a temp project - runs the built local dotagents CLI @@ -48,4 +48,4 @@ Project-scoped `.codex/` layers load only when Codex trusts the project. A one-o If Codex reports `unknown agent_type`, check project trust and the canonical path (`pwd -P`) before assuming the generated TOML schema is wrong. -Do not leave copied Codex auth in retained temp directories. The smoke script scrubs its temp `codex-home`; if you run manual experiments, remove copied auth/config before reporting. +Do not leave copied Codex auth in retained temp directories. The QA task scrubs its temp `codex-home`; if you run manual experiments, remove copied auth/config before reporting. diff --git a/skills/dotagents-qa/references/core-smoke.md b/skills/dotagents-qa/references/core-agentic-qa.md similarity index 81% rename from skills/dotagents-qa/references/core-smoke.md rename to skills/dotagents-qa/references/core-agentic-qa.md index 17be8b9..e58b32c 100644 --- a/skills/dotagents-qa/references/core-smoke.md +++ b/skills/dotagents-qa/references/core-agentic-qa.md @@ -1,4 +1,4 @@ -# Core Smoke +# Core Agentic QA Use this reference for ordinary dotagents install/sync QA before drilling into a specific runtime. @@ -7,12 +7,12 @@ Use this reference for ordinary dotagents install/sync QA before drilling into a Run: ```bash -pnpm smoke:examples +pnpm qa:example ``` This builds the local CLI, copies `examples/full/` to a temp project, and verifies: -- `install`, `list`, `doctor --fix`, and `doctor` +- `install`, `list`, `doctor --fix`, and `doctor` complete successfully - managed skills under `.agents/skills/` - Claude/Cursor skill symlink behavior - MCP files for Claude, Cursor, Codex, and OpenCode @@ -21,7 +21,8 @@ This builds the local CLI, copies `examples/full/` to a temp project, and verifi - generated subagent runtime files for Claude, Cursor, Codex, and OpenCode - `sync` repair after deleting representative generated files -Use `node scripts/smoke-examples.mjs --keep` to keep the temp project for inspection. The script prints the project path. +Use `node skills/dotagents-qa/scripts/qa-example.mjs all --keep` to keep the +temp project for inspection. The script prints the project path. ## What This Proves diff --git a/skills/dotagents-qa/references/cursor.md b/skills/dotagents-qa/references/cursor.md index 0503885..11ad2d5 100644 --- a/skills/dotagents-qa/references/cursor.md +++ b/skills/dotagents-qa/references/cursor.md @@ -4,7 +4,7 @@ Use this reference when changes affect Cursor skill sharing, `.cursor/mcp.json`, ## File-Level Checks -The core smoke asserts: +The core agentic QA asserts: - Cursor shares Claude-compatible skills through `.claude/skills` - `.cursor/mcp.json` exists for Cursor MCP config diff --git a/skills/dotagents-qa/references/opencode.md b/skills/dotagents-qa/references/opencode.md index 4465b07..d061308 100644 --- a/skills/dotagents-qa/references/opencode.md +++ b/skills/dotagents-qa/references/opencode.md @@ -4,7 +4,7 @@ Use this reference when changes affect OpenCode config generation, `opencode.jso ## File-Level Checks -The core smoke asserts: +The core agentic QA asserts: - `opencode.json` exists for OpenCode MCP config - `.opencode/agents/code-reviewer.md` exists diff --git a/scripts/smoke-examples.mjs b/skills/dotagents-qa/scripts/qa-example.mjs similarity index 79% rename from scripts/smoke-examples.mjs rename to skills/dotagents-qa/scripts/qa-example.mjs index 24bd009..c4d980e 100644 --- a/scripts/smoke-examples.mjs +++ b/skills/dotagents-qa/scripts/qa-example.mjs @@ -1,7 +1,6 @@ #!/usr/bin/env node -// Owns local dotagents example QA. The default path proves file wiring with an -// isolated HOME/state; --codex-runtime additionally proves Codex can spawn the -// generated project agent and always scrubs copied Codex auth/config. +// Task-oriented agentic QA for the checked-in dotagents example. Keep this +// script with the dotagents-qa skill so runtime proof stays beside its docs. import { execFileSync } from "node:child_process"; import { @@ -20,20 +19,43 @@ import { dirname, join, resolve } from "node:path"; import { fileURLToPath } from "node:url"; const __dirname = dirname(fileURLToPath(import.meta.url)); -const repoRoot = resolve(__dirname, ".."); +const repoRoot = resolve(__dirname, "../../.."); const cliPath = join(repoRoot, "packages", "dotagents", "dist", "cli", "index.js"); const exampleRoot = join(repoRoot, "examples", "full"); const sentinel = "DOTAGENTS_SUBAGENT_RUNTIME_PROOF_9b8e6f2c"; -const args = new Set(process.argv.slice(2)); -const keep = args.has("--keep"); -const runCodexRuntime = args.has("--codex-runtime"); + +const rawArgs = process.argv.slice(2); +const flags = new Set(rawArgs.filter((arg) => arg.startsWith("--"))); +const keep = flags.has("--keep"); +const task = taskName(rawArgs); + +const taskGroups = { + all: ["install-files", "sync-repair"], +}; + +const tasks = { + "install-files": runInstallFiles, + "sync-repair": runSyncRepair, + "codex-runtime": runCodexRuntimeProof, +}; + +if (flags.has("--help") || flags.has("-h")) { + printUsage(); + process.exit(0); +} if (!existsSync(cliPath)) { - console.error(`smoke-examples: missing built CLI at ${cliPath}`); + console.error(`qa-example: missing built CLI at ${cliPath}`); console.error("Run `pnpm build` first."); process.exit(1); } +if (!tasks[task] && !taskGroups[task]) { + console.error(`qa-example: unknown task "${task}"`); + printUsage(); + process.exit(1); +} + const tmp = mkdtempSync(join(tmpdir(), "dotagents-example-")); const projectDir = join(tmp, "project"); const homeDir = join(tmp, "home"); @@ -53,14 +75,76 @@ const fixtureEnv = { }; try { - console.log(`smoke-examples: project=${projectDir}`); + console.log(`qa-example: project=${projectDir}`); + for (const name of expandedTasks(task)) { + console.log(`qa-example: task=${name}`); + tasks[name](); + } + console.log("qa-example: ok"); +} catch (err) { + console.error("qa-example: failed"); + console.error(err instanceof Error ? err.message : String(err)); + console.error(`qa-example: project kept at ${projectDir}`); + process.exitCode = 1; + throw err; +} finally { + rmSync(codexHomeDir, { recursive: true, force: true }); + if (!keep && process.exitCode !== 1) { + rmSync(tmp, { recursive: true, force: true }); + } +} + +function taskName(args) { + if (flags.has("--codex-runtime")) {return "codex-runtime";} + return args.find((arg) => !arg.startsWith("--")) ?? "all"; +} + +function expandedTasks(name) { + return taskGroups[name] ?? [name]; +} + +function printUsage() { + console.error(`Usage: node skills/dotagents-qa/scripts/qa-example.mjs [--keep] + +Tasks: + all Run install-files and sync-repair + install-files Install the full example and assert generated files + sync-repair Delete representative generated files and assert sync repairs them + codex-runtime Paid proof that Codex can spawn the generated custom agent + +Compatibility: + --codex-runtime Alias for the codex-runtime task +`); +} + +function runInstallFiles() { + installAndAssert(); +} + +function runSyncRepair() { + installAndAssert(); + rmSync(join(projectDir, ".mcp.json"), { force: true }); + rmSync(join(projectDir, ".claude", "skills"), { force: true, recursive: true }); + rmSync(join(projectDir, ".claude", "agents", "code-reviewer.md"), { force: true }); + rmSync(join(projectDir, ".codex", "agents", "code-reviewer.toml"), { force: true }); + runCli(["sync"]); + assertFile(".mcp.json"); + assertSymlink(".claude/skills"); + assertFile(".claude/agents/code-reviewer.md"); + assertFile(".codex/agents/code-reviewer.toml"); +} + +function runCodexRuntimeProof() { + installAndAssert(); + proveCodexRuntime(); +} + +function installAndAssert() { runCli(["install"]); const list = runCli(["list"]); writeFileSync(join(tmp, "list.out"), list); runCli(["doctor", "--fix"]); runCli(["doctor"]); - assertIncludes(list, "review", "list output should include review"); - assertIncludes(list, "commit", "list output should include commit"); assertFile(".agents/skills/review/SKILL.md"); assertFile(".agents/skills/commit/SKILL.md"); @@ -76,33 +160,6 @@ try { assertFile(".claude/settings.json"); assertFile(".cursor/hooks.json"); assertSubagentOutputs(); - - rmSync(join(projectDir, ".mcp.json"), { force: true }); - rmSync(join(projectDir, ".claude", "skills"), { force: true, recursive: true }); - rmSync(join(projectDir, ".claude", "agents", "code-reviewer.md"), { force: true }); - rmSync(join(projectDir, ".codex", "agents", "code-reviewer.toml"), { force: true }); - runCli(["sync"]); - assertFile(".mcp.json"); - assertSymlink(".claude/skills"); - assertFile(".claude/agents/code-reviewer.md"); - assertFile(".codex/agents/code-reviewer.toml"); - - if (runCodexRuntime) { - proveCodexRuntime(); - } - - console.log("smoke-examples: ok"); -} catch (err) { - console.error("smoke-examples: failed"); - console.error(err instanceof Error ? err.message : String(err)); - console.error(`smoke-examples: project kept at ${projectDir}`); - process.exitCode = 1; - throw err; -} finally { - rmSync(codexHomeDir, { recursive: true, force: true }); - if (!keep && process.exitCode !== 1) { - rmSync(tmp, { recursive: true, force: true }); - } } function runCli(cliArgs) { @@ -127,7 +184,7 @@ function proveCodexRuntime() { const sourceAuth = join(sourceCodexHome, "auth.json"); const sourceConfig = join(sourceCodexHome, "config.toml"); if (!existsSync(sourceAuth)) { - throw new Error(`Codex runtime smoke requires auth.json at ${sourceAuth}`); + throw new Error(`Codex runtime QA requires auth.json at ${sourceAuth}`); } mkdirSync(codexHomeDir, { recursive: true }); @@ -231,7 +288,6 @@ function assertIncludes(value, expected, message) { } } -/** Verifies Codex spawned and waited on a child agent that returned the sentinel. */ function assertCodexRuntimeEvents(output) { assertIncludes(output, '"tool":"spawn_agent"', "Codex runtime JSONL should include a spawn_agent event"); assertIncludes(output, '"tool":"wait"', "Codex runtime JSONL should include a wait event"); From db6b6d359c3b3098d5527ccae43e40187907b74f Mon Sep 17 00:00:00 2001 From: David Cramer Date: Mon, 29 Jun 2026 15:40:57 -0700 Subject: [PATCH 2/2] fix(qa): Tighten example runner checks Recognize single-dash help flags and validate the checked-in example skills through the built list implementation after the CLI list command runs. --- skills/dotagents-qa/scripts/qa-example.mjs | 43 ++++++++++++++++------ 1 file changed, 32 insertions(+), 11 deletions(-) diff --git a/skills/dotagents-qa/scripts/qa-example.mjs b/skills/dotagents-qa/scripts/qa-example.mjs index c4d980e..37bda4d 100644 --- a/skills/dotagents-qa/scripts/qa-example.mjs +++ b/skills/dotagents-qa/scripts/qa-example.mjs @@ -16,7 +16,7 @@ import { } from "node:fs"; import { tmpdir } from "node:os"; import { dirname, join, resolve } from "node:path"; -import { fileURLToPath } from "node:url"; +import { fileURLToPath, pathToFileURL } from "node:url"; const __dirname = dirname(fileURLToPath(import.meta.url)); const repoRoot = resolve(__dirname, "../../.."); @@ -25,7 +25,7 @@ const exampleRoot = join(repoRoot, "examples", "full"); const sentinel = "DOTAGENTS_SUBAGENT_RUNTIME_PROOF_9b8e6f2c"; const rawArgs = process.argv.slice(2); -const flags = new Set(rawArgs.filter((arg) => arg.startsWith("--"))); +const flags = new Set(rawArgs.filter((arg) => arg.startsWith("-"))); const keep = flags.has("--keep"); const task = taskName(rawArgs); @@ -78,7 +78,7 @@ try { console.log(`qa-example: project=${projectDir}`); for (const name of expandedTasks(task)) { console.log(`qa-example: task=${name}`); - tasks[name](); + await tasks[name](); } console.log("qa-example: ok"); } catch (err) { @@ -96,7 +96,7 @@ try { function taskName(args) { if (flags.has("--codex-runtime")) {return "codex-runtime";} - return args.find((arg) => !arg.startsWith("--")) ?? "all"; + return args.find((arg) => !arg.startsWith("-")) ?? "all"; } function expandedTasks(name) { @@ -117,12 +117,12 @@ Compatibility: `); } -function runInstallFiles() { - installAndAssert(); +async function runInstallFiles() { + await installAndAssert(); } -function runSyncRepair() { - installAndAssert(); +async function runSyncRepair() { + await installAndAssert(); rmSync(join(projectDir, ".mcp.json"), { force: true }); rmSync(join(projectDir, ".claude", "skills"), { force: true, recursive: true }); rmSync(join(projectDir, ".claude", "agents", "code-reviewer.md"), { force: true }); @@ -134,15 +134,18 @@ function runSyncRepair() { assertFile(".codex/agents/code-reviewer.toml"); } -function runCodexRuntimeProof() { - installAndAssert(); +async function runCodexRuntimeProof() { + await installAndAssert(); proveCodexRuntime(); } -function installAndAssert() { +async function installAndAssert() { runCli(["install"]); const list = runCli(["list"]); writeFileSync(join(tmp, "list.out"), list); + const listStatuses = await listSkills(); + assertSkillStatus(listStatuses, "review"); + assertSkillStatus(listStatuses, "commit"); runCli(["doctor", "--fix"]); runCli(["doctor"]); @@ -171,6 +174,14 @@ function runCli(cliArgs) { }); } +async function listSkills() { + const [{ runList }, { resolveScope }] = await Promise.all([ + import(pathToFileURL(join(repoRoot, "packages", "dotagents", "dist", "cli", "commands", "list.js")).href), + import(pathToFileURL(join(repoRoot, "packages", "dotagents", "dist", "scope.js")).href), + ]); + return runList({ scope: resolveScope("project", projectDir) }); +} + function proveCodexRuntime() { if (!existsSync(join(projectDir, ".git"))) { execFileSync("git", ["init"], { @@ -282,6 +293,16 @@ function assertFileIncludes(relativePath, expected) { assertIncludes(readFileSync(join(projectDir, relativePath), "utf-8"), expected, `${relativePath} should include ${expected}`); } +function assertSkillStatus(statuses, name) { + const status = statuses.find((entry) => entry.name === name); + if (!status) { + throw new Error(`list should include ${name}`); + } + if (status.status !== "ok") { + throw new Error(`list should report ${name} as ok, got ${status.status}`); + } +} + function assertIncludes(value, expected, message) { if (!value.includes(expected)) { throw new Error(message);