From f9ff7c340ad71d4d70c8b0d71b6879e55c736841 Mon Sep 17 00:00:00 2001 From: Nathan Flurry Date: Wed, 8 Jul 2026 14:41:20 -0700 Subject: [PATCH] test(pi-cli): enable real ACP bash coverage --- docs-internal/registry-parity-worklist.md | 10 ++ .../helpers/registry-command-availability.ts | 58 +++++++++ packages/core/tests/pi-cli-headless.test.ts | 112 +++++++++--------- packages/core/tests/pi-headless.test.ts | 103 ++++++++-------- packages/core/vitest.config.ts | 1 - 5 files changed, 179 insertions(+), 105 deletions(-) create mode 100644 packages/core/tests/helpers/registry-command-availability.ts diff --git a/docs-internal/registry-parity-worklist.md b/docs-internal/registry-parity-worklist.md index e80daf37c1..221f905212 100644 --- a/docs-internal/registry-parity-worklist.md +++ b/docs-internal/registry-parity-worklist.md @@ -691,6 +691,16 @@ real e2e tests that prove Linux-parity behavior — not smoke tests. `2026-07-08T14-37-00-0700-item12-sidecar-build-after-manual-cc-restore.log`; `2026-07-08T14-38-00-0700-item12-pi-headless-final-after-cc-restore.log`. Rev: `mzuuypsm`. + - **pi-cli — DONE.** Enabled the existing real `createSession('pi-cli')` + headless suite in default core Vitest coverage and unskipped the unmodified + Pi CLI bash-tool flow. Pi and pi-cli now project registry command software + only when the local `.aospkg` artifacts are built, so write-tool coverage + still runs while bash-tool coverage is gated on real command availability. + Proof: + `2026-07-08T15-00-00-0700-item12-pi-cli-cc-cache-restored-target-files-final.log`; + `2026-07-08T15-00-00-0700-item12-pi-cli-sidecar-build-after-cc-restore-final.log`; + `2026-07-08T15-01-00-0700-item12-pi-cli-focused-final-after-cc-restore.log`. + Rev: `xqtkmsyn`. - **Objective:** write real e2e tests proving each behaves like its Linux counterpart (jq processes real JSON, sed edits streams, tar round-trips archives, gzip round-trips, etc.); agents exercise the real ACP diff --git a/packages/core/tests/helpers/registry-command-availability.ts b/packages/core/tests/helpers/registry-command-availability.ts new file mode 100644 index 0000000000..309df23375 --- /dev/null +++ b/packages/core/tests/helpers/registry-command-availability.ts @@ -0,0 +1,58 @@ +import { + closeSync, + existsSync, + openSync, + readdirSync, + readSync, + statSync, +} from "node:fs"; +import { dirname, join } from "node:path"; + +interface RegistryPackageRef { + packagePath: string; +} + +const AOSPKG_MAGIC = Buffer.from([0x89, 0x41, 0x4f, 0x53]); + +function isPackedAospkg(path: string): boolean { + try { + if (statSync(path).size <= 16) return false; + const fd = openSync(path, "r"); + try { + const head = Buffer.alloc(4); + readSync(fd, head, 0, 4, 0); + return head.equals(AOSPKG_MAGIC); + } finally { + closeSync(fd); + } + } catch { + return false; + } +} + +export function hasBuiltRegistryCommands( + packages: readonly RegistryPackageRef[], +): boolean { + return packages.every((pkg) => { + const path = pkg.packagePath; + if (path.endsWith(".aospkg")) { + if (!isPackedAospkg(path)) return false; + return hasCompleteCommandDir(join(dirname(path), "package")); + } + return ( + existsSync(join(path, "agentos-package.json")) && + hasCompleteCommandDir(path) + ); + }); +} + +function hasCompleteCommandDir(path: string): boolean { + if (!existsSync(path)) return true; + const binDir = join(path, "bin"); + if (!existsSync(binDir)) return false; + const entries = readdirSync(binDir); + return ( + entries.length > 0 && + entries.every((entry) => existsSync(join(binDir, entry))) + ); +} diff --git a/packages/core/tests/pi-cli-headless.test.ts b/packages/core/tests/pi-cli-headless.test.ts index 5f639a3498..f48c45c19c 100644 --- a/packages/core/tests/pi-cli-headless.test.ts +++ b/packages/core/tests/pi-cli-headless.test.ts @@ -1,8 +1,7 @@ import { resolve } from "node:path"; -import type { Fixture, ToolCall } from "@copilotkit/llmock"; -import { moduleAccessMounts } from "./helpers/node-modules-mount.js"; import common from "@agentos-software/common"; import piCli from "@agentos-software/pi-cli"; +import type { Fixture, ToolCall } from "@copilotkit/llmock"; import { describe, expect, test } from "vitest"; import { AgentOs } from "../src/agent-os.js"; import { @@ -10,8 +9,12 @@ import { startLlmock, stopLlmock, } from "./helpers/llmock-helper.js"; +import { moduleAccessMounts } from "./helpers/node-modules-mount.js"; +import { hasBuiltRegistryCommands } from "./helpers/registry-command-availability.js"; const MODULE_ACCESS_CWD = resolve(import.meta.dirname, ".."); +const registryCommandsAvailable = hasBuiltRegistryCommands(common); +const registryCommandTest = registryCommandsAvailable ? test : test.skip; function getRequestBody(req: unknown): Record { const direct = req as Record; @@ -49,7 +52,7 @@ async function createPiCliVm(mockUrl: string): Promise { return AgentOs.create({ loopbackExemptPorts: [Number(new URL(mockUrl).port)], mounts: moduleAccessMounts(MODULE_ACCESS_CWD), - software: [common, piCli], + software: [...(registryCommandsAvailable ? common : []), piCli], }); } @@ -152,61 +155,58 @@ describe("full createSession('pi-cli') inside the VM", () => { } }, 120_000); - // Blocked on shell `>` redirect output being visible to `vm.readFile()`. - // This is the unmodified upstream Pi CLI bash path (`createLocalBashOperations` - // spawning the shell directly), with no Agent OS operations override, so the - // failure is a runtime gap independent of the SDK adapter: the redirect runs - // inside the guest shell but the written bytes do not reconcile to the host - // read path yet. Tracked in ~/.agents/todo/agentos-runtime-fixes.md - // (shell-exec redirect visibility). - test.skip("runs the unmodified Pi CLI ACP flow end-to-end for bash tool calls", async () => { - const workspacePath = "/home/agentos/workspace/bash-output.txt"; - const fixtures = createToolFixtures( - { - name: "bash", - arguments: JSON.stringify({ - command: `printf 'bash-ok' > ${workspacePath}`, - timeout: 10, - }), - }, - "bash-ok", - "bash-output.txt was written successfully.", - ); - const { mock, url } = await startLlmock(fixtures); - const vm = await createPiCliVm(url); + registryCommandTest( + "runs the unmodified Pi CLI ACP flow end-to-end for bash tool calls", + async () => { + const workspacePath = "/home/agentos/workspace/bash-output.txt"; + const fixtures = createToolFixtures( + { + name: "bash", + arguments: JSON.stringify({ + command: `printf 'bash-ok' > ${workspacePath}`, + timeout: 10, + }), + }, + "bash-ok", + "bash-output.txt was written successfully.", + ); + const { mock, url } = await startLlmock(fixtures); + const vm = await createPiCliVm(url); - let sessionId: string | undefined; - try { - const homeDir = await createVmPiHome(vm, url); - const workspaceDir = await createVmWorkspace(vm); - sessionId = ( - await vm.createSession("pi-cli", { - cwd: workspaceDir, - env: { - HOME: homeDir, - ANTHROPIC_API_KEY: "mock-key", - ANTHROPIC_BASE_URL: url, - }, - }) - ).sessionId; + let sessionId: string | undefined; + try { + const homeDir = await createVmPiHome(vm, url); + const workspaceDir = await createVmWorkspace(vm); + sessionId = ( + await vm.createSession("pi-cli", { + cwd: workspaceDir, + env: { + HOME: homeDir, + ANTHROPIC_API_KEY: "mock-key", + ANTHROPIC_BASE_URL: url, + }, + }) + ).sessionId; - const { response, text } = await vm.prompt( - sessionId, - `Use bash to write bash-ok into ${workspacePath}.`, - ); + const { response, text } = await vm.prompt( + sessionId, + `Use bash to write bash-ok into ${workspacePath}.`, + ); - expect(response.error).toBeUndefined(); - expect(text).toContain("bash-output.txt was written successfully."); - expect(new TextDecoder().decode(await vm.readFile(workspacePath))).toBe( - "bash-ok", - ); - expect(mock.getRequests().length).toBeGreaterThanOrEqual(2); - } finally { - if (sessionId) { - vm.closeSession(sessionId); + expect(response.error).toBeUndefined(); + expect(text).toContain("bash-output.txt was written successfully."); + expect(new TextDecoder().decode(await vm.readFile(workspacePath))).toBe( + "bash-ok", + ); + expect(mock.getRequests().length).toBeGreaterThanOrEqual(2); + } finally { + if (sessionId) { + vm.closeSession(sessionId); + } + await vm.dispose(); + await stopLlmock(mock); } - await vm.dispose(); - await stopLlmock(mock); - } - }, 120_000); + }, + 120_000, + ); }); diff --git a/packages/core/tests/pi-headless.test.ts b/packages/core/tests/pi-headless.test.ts index 66246c810d..294dffe7e1 100644 --- a/packages/core/tests/pi-headless.test.ts +++ b/packages/core/tests/pi-headless.test.ts @@ -11,8 +11,11 @@ import { stopLlmock, } from "./helpers/llmock-helper.js"; import { moduleAccessMounts } from "./helpers/node-modules-mount.js"; +import { hasBuiltRegistryCommands } from "./helpers/registry-command-availability.js"; const MODULE_ACCESS_CWD = resolve(import.meta.dirname, ".."); +const registryCommandsAvailable = hasBuiltRegistryCommands(common); +const registryCommandTest = registryCommandsAvailable ? test : test.skip; function getRequestBody(req: unknown): Record { const direct = req as Record; @@ -51,7 +54,7 @@ async function createPiVm(mockUrl: string): Promise { loopbackExemptPorts: [Number(new URL(mockUrl).port)], mounts: moduleAccessMounts(MODULE_ACCESS_CWD), // Default software ships no agents; pass the pi agent package explicitly. - software: [common, pi], + software: [...(registryCommandsAvailable ? common : []), pi], }); } @@ -206,55 +209,59 @@ describe("full createSession('pi') inside the VM", () => { } }, 120_000); - test("runs the real Pi SDK ACP flow end-to-end for bash tool calls", async () => { - const fixtures = createToolFixtures( - { - name: "bash", - arguments: JSON.stringify({ - command: "printf 'bash-ok' > bash-output.txt", - timeout: 10, - }), - }, - "bash-ok", - "bash-output.txt was written successfully.", - ); - const { mock, url } = await startLlmock(fixtures); - const vm = await createPiVm(url); + registryCommandTest( + "runs the real Pi SDK ACP flow end-to-end for bash tool calls", + async () => { + const fixtures = createToolFixtures( + { + name: "bash", + arguments: JSON.stringify({ + command: "printf 'bash-ok' > bash-output.txt", + timeout: 10, + }), + }, + "bash-ok", + "bash-output.txt was written successfully.", + ); + const { mock, url } = await startLlmock(fixtures); + const vm = await createPiVm(url); - let sessionId: string | undefined; - try { - const homeDir = await createVmPiHome(vm, url); - const workspaceDir = await createVmWorkspace(vm); - sessionId = ( - await vm.createSession("pi", { - cwd: workspaceDir, - env: { - HOME: homeDir, - ANTHROPIC_API_KEY: "mock-key", - ANTHROPIC_BASE_URL: url, - }, - }) - ).sessionId; + let sessionId: string | undefined; + try { + const homeDir = await createVmPiHome(vm, url); + const workspaceDir = await createVmWorkspace(vm); + sessionId = ( + await vm.createSession("pi", { + cwd: workspaceDir, + env: { + HOME: homeDir, + ANTHROPIC_API_KEY: "mock-key", + ANTHROPIC_BASE_URL: url, + }, + }) + ).sessionId; - const { response, text } = await vm.prompt( - sessionId, - "Use bash to write bash-ok into bash-output.txt.", - ); + const { response, text } = await vm.prompt( + sessionId, + "Use bash to write bash-ok into bash-output.txt.", + ); - expect(response.error).toBeUndefined(); - expect(text).toContain("bash-output.txt was written successfully."); - expect( - new TextDecoder().decode( - await vm.readFile(`${workspaceDir}/bash-output.txt`), - ), - ).toBe("bash-ok"); - expect(mock.getRequests().length).toBeGreaterThanOrEqual(2); - } finally { - if (sessionId) { - vm.closeSession(sessionId); + expect(response.error).toBeUndefined(); + expect(text).toContain("bash-output.txt was written successfully."); + expect( + new TextDecoder().decode( + await vm.readFile(`${workspaceDir}/bash-output.txt`), + ), + ).toBe("bash-ok"); + expect(mock.getRequests().length).toBeGreaterThanOrEqual(2); + } finally { + if (sessionId) { + vm.closeSession(sessionId); + } + await vm.dispose(); + await stopLlmock(mock); } - await vm.dispose(); - await stopLlmock(mock); - } - }, 120_000); + }, + 120_000, + ); }); diff --git a/packages/core/vitest.config.ts b/packages/core/vitest.config.ts index f9881d112c..f79fb17ab8 100644 --- a/packages/core/vitest.config.ts +++ b/packages/core/vitest.config.ts @@ -27,7 +27,6 @@ const SLOW_E2E_FILES = [ "tests/child-process-detached.test.ts", "tests/readdir-recursive.test.ts", "tests/cron-integration.test.ts", - "tests/pi-cli-headless.test.ts", ]; // Pre-existing failures NOT caused by this branch (they were red before CI ever