From fc674854f80526211333d7327414ee9e139afb88 Mon Sep 17 00:00:00 2001 From: Shane Bishop <71288697+shanebishop1@users.noreply.github.com> Date: Thu, 23 Jul 2026 23:55:38 -0700 Subject: [PATCH 1/9] test: report disabled V1 E2E as skipped --- tests/e2e.hooks.test.ts | 48 +++++------------------------------------ 1 file changed, 5 insertions(+), 43 deletions(-) diff --git a/tests/e2e.hooks.test.ts b/tests/e2e.hooks.test.ts index b20c5c6..179babd 100644 --- a/tests/e2e.hooks.test.ts +++ b/tests/e2e.hooks.test.ts @@ -153,21 +153,13 @@ async function runOpenCode(prompt: string): Promise { } } -describe("E2E Hook Behavioral Tests", () => { - let skipTests = false - +describe.skipIf(!E2E_ENABLED)("E2E Hook Behavioral Tests", () => { beforeAll(async () => { - if (!E2E_ENABLED) { - skipTests = true - console.log("⚠️ Skipping E2E tests: set OPENCODE_E2E=1 to run") - return - } - // Check if OpenCode is available - skipTests = !(await isOpenCodeAvailable()) - if (skipTests) { - console.log("⚠️ Skipping E2E tests: OpenCode is not available or not working properly") - return + if (!(await isOpenCodeAvailable())) { + throw new Error( + "OPENCODE_E2E=1 was set, but the OpenCode CLI is unavailable. Install OpenCode or unset OPENCODE_E2E." + ) } // Enable the plugin in the test opencode config @@ -193,11 +185,6 @@ describe("E2E Hook Behavioral Tests", () => { }) it("Test 1: Inject during tool.execute.after - LLM responds", async () => { - if (skipTests) { - console.log("Skipping: OpenCode not available") - return - } - console.log("\n=== Test 1: Inject during tool.execute.after - LLM responds ===") const uniqueId = generateUniqueId() @@ -237,11 +224,6 @@ describe("E2E Hook Behavioral Tests", () => { }, 120000) it("Test 2: Inject during session.idle - LLM does NOT respond", async () => { - if (skipTests) { - console.log("Skipping: OpenCode not available") - return - } - console.log("\n=== Test 2: Inject during session.idle - LLM does NOT respond ===") const uniqueId = generateUniqueId() @@ -288,11 +270,6 @@ describe("E2E Hook Behavioral Tests", () => { }, 120000) it("Test 3: Toast doesn't affect LLM response", async () => { - if (skipTests) { - console.log("Skipping: OpenCode not available") - return - } - console.log("\n=== Test 3: Toast doesn't affect LLM response ===") const uniqueId = generateUniqueId() @@ -346,11 +323,6 @@ describe("E2E Hook Behavioral Tests", () => { }, 120000) it("Test 4: stdout template substitution works", async () => { - if (skipTests) { - console.log("Skipping: OpenCode not available") - return - } - console.log("\n=== Test 4: stdout template substitution works ===") const uniqueId = generateUniqueId() @@ -403,11 +375,6 @@ describe("E2E Hook Behavioral Tests", () => { }, 120000) it("Test 5: Hook fires after write tool and creates file", async () => { - if (skipTests) { - console.log("Skipping: OpenCode not available") - return - } - console.log("\n=== Test 5: Hook fires after write tool and creates file ===") const uniqueId = generateUniqueId() @@ -481,11 +448,6 @@ describe("E2E Hook Behavioral Tests", () => { }, 120000) it("Test 6: Agent frontmatter hooks do not leak to provider options", async () => { - if (skipTests) { - console.log("Skipping: OpenCode not available") - return - } - console.log("\n=== Test 6: Agent frontmatter hooks do not leak to provider options ===") const uniqueId = generateUniqueId() const marker = `FRONTMATTER_HOOK_${uniqueId}` From c588ba532ca296805d4c0e9ed698501d8674f756 Mon Sep 17 00:00:00 2001 From: Shane Bishop <71288697+shanebishop1@users.noreply.github.com> Date: Thu, 23 Jul 2026 23:59:30 -0700 Subject: [PATCH 2/9] ci: run V1 E2E against pinned OpenCode --- .github/workflows/v1-e2e.yml | 29 +++++++++++++++++++++++++++++ package.json | 1 + 2 files changed, 30 insertions(+) create mode 100644 .github/workflows/v1-e2e.yml diff --git a/.github/workflows/v1-e2e.yml b/.github/workflows/v1-e2e.yml new file mode 100644 index 0000000..a5af553 --- /dev/null +++ b/.github/workflows/v1-e2e.yml @@ -0,0 +1,29 @@ +name: V1 E2E + +on: + push: + pull_request: + +permissions: + contents: read + +jobs: + test: + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Setup Bun + uses: oven-sh/setup-bun@v2 + with: + bun-version: "1.3.9" + + - name: Install dependencies + run: bun install --frozen-lockfile + + - name: Install OpenCode V1 + run: bun install --global opencode-ai@1.18.3 + + - name: Run V1 E2E tests + run: npm run test:v1:e2e diff --git a/package.json b/package.json index b879efd..6f89fa6 100644 --- a/package.json +++ b/package.json @@ -24,6 +24,7 @@ "typecheck:bun": "bun --bun tsc --noEmit", "typecheck:bun-build": "bun build src/index.ts --target=bun", "typecheck:bun-run": "bun run src/index.ts", + "test:v1:e2e": "OPENCODE_E2E=1 bun test tests/e2e.hooks.test.ts", "dev": "tsc --watch", "prepublishOnly": "npm run build" }, From ba32b91e7901fe269129af06b3371b524e550c13 Mon Sep 17 00:00:00 2001 From: Shane Bishop <71288697+shanebishop1@users.noreply.github.com> Date: Fri, 24 Jul 2026 00:03:27 -0700 Subject: [PATCH 3/9] docs: define V1 E2E as headless coverage --- .github/workflows/v1-e2e.yml | 4 ++-- README.md | 4 ++++ package.json | 2 +- tests/{e2e.hooks.test.ts => e2e.v1-headless.test.ts} | 2 +- 4 files changed, 8 insertions(+), 4 deletions(-) rename tests/{e2e.hooks.test.ts => e2e.v1-headless.test.ts} (99%) diff --git a/.github/workflows/v1-e2e.yml b/.github/workflows/v1-e2e.yml index a5af553..d91580a 100644 --- a/.github/workflows/v1-e2e.yml +++ b/.github/workflows/v1-e2e.yml @@ -1,4 +1,4 @@ -name: V1 E2E +name: V1 Headless E2E on: push: @@ -25,5 +25,5 @@ jobs: - name: Install OpenCode V1 run: bun install --global opencode-ai@1.18.3 - - name: Run V1 E2E tests + - name: Run V1 headless E2E tests run: npm run test:v1:e2e diff --git a/README.md b/README.md index d60b03b..1d23fc0 100644 --- a/README.md +++ b/README.md @@ -18,6 +18,10 @@ hooks: ``` **This plugin was not built by the OpenCode team nor is it affiliated with them.** +## Testing + +`npm run test:v1:e2e` runs the opt-in headless real-host suite against the installed CLI. It validates command-hook behavior through `opencode run`, not interactive TUI rendering or interactions. + ## Table of Contents - [Features](#features) diff --git a/package.json b/package.json index 6f89fa6..62e67df 100644 --- a/package.json +++ b/package.json @@ -24,7 +24,7 @@ "typecheck:bun": "bun --bun tsc --noEmit", "typecheck:bun-build": "bun build src/index.ts --target=bun", "typecheck:bun-run": "bun run src/index.ts", - "test:v1:e2e": "OPENCODE_E2E=1 bun test tests/e2e.hooks.test.ts", + "test:v1:e2e": "OPENCODE_E2E=1 bun test tests/e2e.v1-headless.test.ts", "dev": "tsc --watch", "prepublishOnly": "npm run build" }, diff --git a/tests/e2e.hooks.test.ts b/tests/e2e.v1-headless.test.ts similarity index 99% rename from tests/e2e.hooks.test.ts rename to tests/e2e.v1-headless.test.ts index 179babd..d49a46c 100644 --- a/tests/e2e.hooks.test.ts +++ b/tests/e2e.v1-headless.test.ts @@ -153,7 +153,7 @@ async function runOpenCode(prompt: string): Promise { } } -describe.skipIf(!E2E_ENABLED)("E2E Hook Behavioral Tests", () => { +describe.skipIf(!E2E_ENABLED)("V1 headless real-host E2E", () => { beforeAll(async () => { // Check if OpenCode is available if (!(await isOpenCodeAvailable())) { From d6fb75a125501c75cfcf6ca6435a97aa848d34ef Mon Sep 17 00:00:00 2001 From: Shane Bishop <71288697+shanebishop1@users.noreply.github.com> Date: Fri, 24 Jul 2026 00:07:22 -0700 Subject: [PATCH 4/9] test: verify toast delivery at the host boundary --- tests/e2e.v1-headless.test.ts | 20 +++----------------- tests/tool.after-result.test.ts | 32 ++++++++++++++++++-------------- 2 files changed, 21 insertions(+), 31 deletions(-) diff --git a/tests/e2e.v1-headless.test.ts b/tests/e2e.v1-headless.test.ts index d49a46c..ea8470c 100644 --- a/tests/e2e.v1-headless.test.ts +++ b/tests/e2e.v1-headless.test.ts @@ -269,8 +269,8 @@ describe.skipIf(!E2E_ENABLED)("V1 headless real-host E2E", () => { expect(containsUniqueId).toBe(false) }, 120000) - it("Test 3: Toast doesn't affect LLM response", async () => { - console.log("\n=== Test 3: Toast doesn't affect LLM response ===") + it("Test 3: Toast marker does not leak into model output", async () => { + console.log("\n=== Test 3: Toast marker does not leak into model output ===") const uniqueId = generateUniqueId() console.log(`Unique ID: ${uniqueId}`) @@ -301,24 +301,10 @@ describe.skipIf(!E2E_ENABLED)("V1 headless real-host E2E", () => { console.log(`Response length: ${opencodeResponse.length}`) console.log(`Response preview: ${opencodeResponse.substring(0, 200)}...`) - const logContent = await waitForLogMatch( - content => content.includes(`TOAST_MARKER_${uniqueId}`), - 30000, - 500 - ) - console.log(`Log content length: ${logContent.length}`) - console.log(`Log contains [toast]: ${logContent.includes("[toast]")}`) - console.log(`Log contains toast marker: ${logContent.includes(`TOAST_MARKER_${uniqueId}`)}`) - console.log(`Log sample: ${logContent.substring(0, 500)}...`) - - // Assert: Toast marker should appear in logs but NOT in OpenCode response - const toastInLogs = logContent.includes(`TOAST_MARKER_${uniqueId}`) + // Headless CLI output cannot establish that a TUI toast rendered visibly. const toastInResponse = opencodeResponse.includes(`TOAST_MARKER_${uniqueId}`) - - console.log(`Toast marker in logs: ${toastInLogs}`) console.log(`Toast marker in OpenCode response: ${toastInResponse}`) - expect(toastInLogs).toBe(true) expect(toastInResponse).toBe(false) }, 120000) diff --git a/tests/tool.after-result.test.ts b/tests/tool.after-result.test.ts index 051ee23..9b07513 100644 --- a/tests/tool.after-result.test.ts +++ b/tests/tool.after-result.test.ts @@ -48,18 +48,19 @@ describe("tool after hooks", () => { rmSync(testDir, { recursive: true, force: true }); }); - it("executes tool.execute.after hooks and triggers inject + toast", async () => { + it("sends the fully interpolated after-hook toast payload to the host client", async () => { writeConfig({ tool: [ { id: "after-hook", when: { phase: "after", tool: ["bash"] }, - run: ["echo 'Hook executed'"], + run: ["sh -c 'printf hook-stdout; printf hook-stderr >&2; exit 23'"], inject: "Tool result: {stdout}", toast: { - title: "Hook", - message: "after complete", - variant: "success", + title: "Hook {id} for {tool}", + message: "stdout={stdout}; stderr={stderr}; exit={exitCode}", + variant: "warning", + duration: 4500, }, }, ], @@ -79,15 +80,18 @@ describe("tool after hooks", () => { expect((promptCalls[0].path as { id: string }).id).toBe("s1"); const promptParts = (promptCalls[0].body as { parts: Array<{ text: string }> }).parts; - expect(promptParts[0].text).toContain("Tool result: Hook executed"); - - expect(toastCalls).toHaveLength(1); - expect(toastCalls[0].body).toEqual({ - title: "Hook", - message: "after complete", - variant: "success", - duration: undefined, - }); + expect(promptParts[0].text).toContain("Tool result: hook-stdout"); + + expect(toastCalls).toEqual([ + { + body: { + title: "Hook after-hook for bash", + message: "stdout=hook-stdout; stderr=hook-stderr; exit=23", + variant: "warning", + duration: 4500, + }, + }, + ]); }); it("executes inject-only after hook without run", async () => { From edc18f796a21b435b92400f0b1677cc6e068ca50 Mon Sep 17 00:00:00 2001 From: Shane Bishop <71288697+shanebishop1@users.noreply.github.com> Date: Fri, 24 Jul 2026 00:12:42 -0700 Subject: [PATCH 5/9] test: isolate V1 E2E from user state --- README.md | 4 -- tests/e2e.v1-headless.test.ts | 79 ++++++++++++++++++++++++----------- 2 files changed, 55 insertions(+), 28 deletions(-) diff --git a/README.md b/README.md index 1d23fc0..d60b03b 100644 --- a/README.md +++ b/README.md @@ -18,10 +18,6 @@ hooks: ``` **This plugin was not built by the OpenCode team nor is it affiliated with them.** -## Testing - -`npm run test:v1:e2e` runs the opt-in headless real-host suite against the installed CLI. It validates command-hook behavior through `opencode run`, not interactive TUI rendering or interactions. - ## Table of Contents - [Features](#features) diff --git a/tests/e2e.v1-headless.test.ts b/tests/e2e.v1-headless.test.ts index ea8470c..e263418 100644 --- a/tests/e2e.v1-headless.test.ts +++ b/tests/e2e.v1-headless.test.ts @@ -1,19 +1,57 @@ import { describe, it, expect, beforeAll, afterAll } from "bun:test" -import { existsSync, readdirSync, readFileSync, writeFileSync, mkdirSync, statSync, unlinkSync, rmSync } from "fs" +import { existsSync, readdirSync, readFileSync, writeFileSync, mkdirSync, mkdtempSync, statSync, unlinkSync, rmSync } from "fs" import { join } from "path" -import { homedir } from "os" +import { tmpdir } from "os" import { $ } from "bun" -const TEST_CONFIG_DIR = join(process.cwd(), "tests", "fixtures", "e2e-config") -const TEST_OPENCODE_SUBDIR = join(TEST_CONFIG_DIR, ".opencode") -const TEST_OPENCODE_CONFIG = join(TEST_CONFIG_DIR, "opencode.jsonc") -const TEST_HOOKS_CONFIG = join(TEST_OPENCODE_SUBDIR, "command-hooks.jsonc") -const TEST_AGENT_DIR = join(TEST_OPENCODE_SUBDIR, "agent") -const LOG_DIR = join(homedir(), ".local", "share", "opencode", "log") +const REPOSITORY_ROOT = process.cwd() +let TEST_SANDBOX_DIR = "" +let TEST_CONFIG_DIR = "" +let TEST_OPENCODE_SUBDIR = "" +let TEST_OPENCODE_CONFIG = "" +let TEST_HOOKS_CONFIG = "" +let TEST_AGENT_DIR = "" +let TEST_HOME_DIR = "" +let TEST_XDG_CONFIG_HOME = "" +let TEST_XDG_DATA_HOME = "" +let TEST_XDG_CACHE_HOME = "" +let LOG_DIR = "" const LOG_WINDOW_MS = 15 * 60 * 1000 const LOG_FALLBACK_FILES = 3 const E2E_ENABLED = process.env.OPENCODE_E2E === "1" +function createTestSandbox(): void { + TEST_SANDBOX_DIR = mkdtempSync(join(tmpdir(), "opencode-command-hooks-e2e-")) + TEST_CONFIG_DIR = join(TEST_SANDBOX_DIR, "project") + TEST_OPENCODE_SUBDIR = join(TEST_CONFIG_DIR, ".opencode") + TEST_OPENCODE_CONFIG = join(TEST_CONFIG_DIR, "opencode.jsonc") + TEST_HOOKS_CONFIG = join(TEST_OPENCODE_SUBDIR, "command-hooks.jsonc") + TEST_AGENT_DIR = join(TEST_OPENCODE_SUBDIR, "agent") + TEST_HOME_DIR = join(TEST_SANDBOX_DIR, "home") + TEST_XDG_CONFIG_HOME = join(TEST_SANDBOX_DIR, "xdg-config") + TEST_XDG_DATA_HOME = join(TEST_SANDBOX_DIR, "xdg-data") + TEST_XDG_CACHE_HOME = join(TEST_SANDBOX_DIR, "xdg-cache") + LOG_DIR = join(TEST_XDG_DATA_HOME, "opencode", "log") + + mkdirSync(TEST_CONFIG_DIR, { recursive: true }) + mkdirSync(TEST_HOME_DIR, { recursive: true }) + mkdirSync(TEST_XDG_CONFIG_HOME, { recursive: true }) + mkdirSync(TEST_XDG_DATA_HOME, { recursive: true }) + mkdirSync(TEST_XDG_CACHE_HOME, { recursive: true }) + mkdirSync(LOG_DIR, { recursive: true }) +} + +function getOpenCodeEnvironment(): Record { + return { + ...process.env, + HOME: TEST_HOME_DIR, + XDG_CONFIG_HOME: TEST_XDG_CONFIG_HOME, + XDG_DATA_HOME: TEST_XDG_DATA_HOME, + XDG_CACHE_HOME: TEST_XDG_CACHE_HOME, + OPENCODE_CONFIG: TEST_OPENCODE_CONFIG, + } +} + /** * Check if OpenCode CLI is available and working properly * Tests both --version and a simple run command to ensure full functionality @@ -100,7 +138,7 @@ function writeTestOpencodeConfig(): void { mkdirSync(TEST_CONFIG_DIR, { recursive: true }) } const pluginConfig = { - plugin: [join(process.cwd(), "dist", "index.js")], + plugin: [join(REPOSITORY_ROOT, "dist", "index.js")], } writeFileSync(TEST_OPENCODE_CONFIG, JSON.stringify(pluginConfig, null, 2)) } @@ -139,7 +177,10 @@ async function waitForLogMatch( */ async function runOpenCode(prompt: string): Promise { try { - const result = await $`cd ${TEST_CONFIG_DIR} && OPENCODE_CONFIG=${TEST_OPENCODE_CONFIG} timeout 30 opencode -m opencode/big-pickle run ${prompt} 2>&1`.text() + const result = await $`timeout 30 opencode -m opencode/big-pickle run ${prompt} 2>&1` + .cwd(TEST_CONFIG_DIR) + .env(getOpenCodeEnvironment()) + .text() await new Promise(resolve => setTimeout(resolve, 2000)) // Ensure we always return a string @@ -161,27 +202,17 @@ describe.skipIf(!E2E_ENABLED)("V1 headless real-host E2E", () => { "OPENCODE_E2E=1 was set, but the OpenCode CLI is unavailable. Install OpenCode or unset OPENCODE_E2E." ) } + + createTestSandbox() // Enable the plugin in the test opencode config writeTestOpencodeConfig() }) afterAll(() => { - // Clean up the test config directory (optional - leave for debugging if needed) - // Uncomment the following code to clean up after tests: - /* - if (existsSync(TEST_CONFIG_DIR)) { - try { - const files = readdirSync(TEST_CONFIG_DIR) - for (const file of files) { - unlinkSync(join(TEST_CONFIG_DIR, file)) - } - rmdirSync(TEST_CONFIG_DIR) - } catch (e) { - // Ignore cleanup errors - } + if (TEST_SANDBOX_DIR) { + rmSync(TEST_SANDBOX_DIR, { recursive: true, force: true }) } - */ }) it("Test 1: Inject during tool.execute.after - LLM responds", async () => { From 81ff6fe0fd583f2808cff1e9bd86bf22e37e9797 Mon Sep 17 00:00:00 2001 From: Shane Bishop <71288697+shanebishop1@users.noreply.github.com> Date: Fri, 24 Jul 2026 00:17:28 -0700 Subject: [PATCH 6/9] test: fail V1 E2E on OpenCode process errors --- tests/e2e.v1-headless.test.ts | 115 +++++++++++++++++++++++----------- 1 file changed, 80 insertions(+), 35 deletions(-) diff --git a/tests/e2e.v1-headless.test.ts b/tests/e2e.v1-headless.test.ts index e263418..167078e 100644 --- a/tests/e2e.v1-headless.test.ts +++ b/tests/e2e.v1-headless.test.ts @@ -2,7 +2,6 @@ import { describe, it, expect, beforeAll, afterAll } from "bun:test" import { existsSync, readdirSync, readFileSync, writeFileSync, mkdirSync, mkdtempSync, statSync, unlinkSync, rmSync } from "fs" import { join } from "path" import { tmpdir } from "os" -import { $ } from "bun" const REPOSITORY_ROOT = process.cwd() let TEST_SANDBOX_DIR = "" @@ -18,6 +17,7 @@ let TEST_XDG_CACHE_HOME = "" let LOG_DIR = "" const LOG_WINDOW_MS = 15 * 60 * 1000 const LOG_FALLBACK_FILES = 3 +const OPENCODE_COMMAND_TIMEOUT_MS = 30_000 const E2E_ENABLED = process.env.OPENCODE_E2E === "1" function createTestSandbox(): void { @@ -52,19 +52,82 @@ function getOpenCodeEnvironment(): Record { } } +interface OpenCodeCommandResult { + stdout: string + stderr: string + exitCode: number +} + +function formatOpenCodeCommand(args: string[]): string { + return ["opencode", ...args].map(argument => JSON.stringify(argument)).join(" ") +} + +function formatOpenCodeCommandError( + message: string, + args: string[], + stdout: string, + stderr: string +): Error { + return new Error( + `${message}\nCommand: ${formatOpenCodeCommand(args)}\nstdout:\n${stdout}\nstderr:\n${stderr}` + ) +} + /** - * Check if OpenCode CLI is available and working properly - * Tests both --version and a simple run command to ensure full functionality + * Run an OpenCode command in the isolated test environment. Command failures + * are rejected so behavioral assertions never inspect failed-process output. */ -async function isOpenCodeAvailable(): Promise { +async function runOpenCodeCommand(args: string[]): Promise { + let subprocess: Bun.ReadableSubprocess + try { + subprocess = Bun.spawn(["opencode", ...args], { + cwd: TEST_CONFIG_DIR, + env: getOpenCodeEnvironment(), + stdout: "pipe", + stderr: "pipe", + }) + } catch (error) { + throw new Error( + `Failed to start OpenCode command: ${formatOpenCodeCommand(args)}\n${String(error)}` + ) + } + + let timedOut = false + const timeout = setTimeout(() => { + if (subprocess.exitCode === null) { + timedOut = true + subprocess.kill("SIGKILL") + } + }, OPENCODE_COMMAND_TIMEOUT_MS) + try { - const whichResult = await $`which opencode 2>&1`.text() - if (!whichResult || whichResult.includes("not found")) { - return false + const [exitCode, stdout, stderr] = await Promise.all([ + subprocess.exited, + new Response(subprocess.stdout).text(), + new Response(subprocess.stderr).text(), + ]) + + if (timedOut) { + throw formatOpenCodeCommandError( + `OpenCode command timed out after ${OPENCODE_COMMAND_TIMEOUT_MS / 1000} seconds.`, + args, + stdout, + stderr + ) + } + + if (exitCode !== 0) { + throw formatOpenCodeCommandError( + `OpenCode command exited with code ${exitCode}.`, + args, + stdout, + stderr + ) } - return true - } catch { - return false + + return { stdout, stderr, exitCode } + } finally { + clearTimeout(timeout) } } @@ -172,38 +235,20 @@ async function waitForLogMatch( } /** - * Run OpenCode with a prompt and capture the stdout response - * This function captures the actual stdout from opencode, not just runs it silently + * Run OpenCode with a prompt and capture its successful output. */ async function runOpenCode(prompt: string): Promise { - try { - const result = await $`timeout 30 opencode -m opencode/big-pickle run ${prompt} 2>&1` - .cwd(TEST_CONFIG_DIR) - .env(getOpenCodeEnvironment()) - .text() - await new Promise(resolve => setTimeout(resolve, 2000)) - - // Ensure we always return a string - const resultStr = typeof result === 'string' ? result : String(result || '') - return resultStr - } catch (e: unknown) { - // Handle error case - ensure we return a string - const error = e as { stdout?: unknown; stderr?: unknown; message?: string } - const errorContent = error.stdout || error.stderr || error.message || "" - return typeof errorContent === 'string' ? errorContent : String(errorContent || "") - } + const result = await runOpenCodeCommand(["-m", "opencode/big-pickle", "run", prompt]) + await new Promise(resolve => setTimeout(resolve, 2000)) + return `${result.stdout}${result.stderr}` } describe.skipIf(!E2E_ENABLED)("V1 headless real-host E2E", () => { beforeAll(async () => { - // Check if OpenCode is available - if (!(await isOpenCodeAvailable())) { - throw new Error( - "OPENCODE_E2E=1 was set, but the OpenCode CLI is unavailable. Install OpenCode or unset OPENCODE_E2E." - ) - } - createTestSandbox() + + // Verify the executable works in the same isolated environment as the tests. + await runOpenCodeCommand(["--version"]) // Enable the plugin in the test opencode config writeTestOpencodeConfig() From 1925db233fdead0e2dfd716f1e8c72c9afd05fa4 Mon Sep 17 00:00:00 2001 From: Shane Bishop <71288697+shanebishop1@users.noreply.github.com> Date: Fri, 24 Jul 2026 00:18:02 -0700 Subject: [PATCH 7/9] test: build V1 before real-host E2E --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 62e67df..142a23c 100644 --- a/package.json +++ b/package.json @@ -24,7 +24,7 @@ "typecheck:bun": "bun --bun tsc --noEmit", "typecheck:bun-build": "bun build src/index.ts --target=bun", "typecheck:bun-run": "bun run src/index.ts", - "test:v1:e2e": "OPENCODE_E2E=1 bun test tests/e2e.v1-headless.test.ts", + "test:v1:e2e": "npm run build && OPENCODE_E2E=1 bun test tests/e2e.v1-headless.test.ts", "dev": "tsc --watch", "prepublishOnly": "npm run build" }, From 8073194d335f54756f07924240902b8ccb1a10d3 Mon Sep 17 00:00:00 2001 From: Shane Bishop <71288697+shanebishop1@users.noreply.github.com> Date: Fri, 24 Jul 2026 00:40:54 -0700 Subject: [PATCH 8/9] test: use exact evidence in V1 real-host smoke --- .github/workflows/v1-e2e.yml | 2 + tests/e2e.v1-headless.test.ts | 341 +++++++--------------------------- 2 files changed, 68 insertions(+), 275 deletions(-) diff --git a/.github/workflows/v1-e2e.yml b/.github/workflows/v1-e2e.yml index d91580a..01274d8 100644 --- a/.github/workflows/v1-e2e.yml +++ b/.github/workflows/v1-e2e.yml @@ -27,3 +27,5 @@ jobs: - name: Run V1 headless E2E tests run: npm run test:v1:e2e + env: + OPENCODE_E2E_AUTH_JSON: ${{ secrets.OPENCODE_E2E_AUTH_JSON }} diff --git a/tests/e2e.v1-headless.test.ts b/tests/e2e.v1-headless.test.ts index 167078e..556046d 100644 --- a/tests/e2e.v1-headless.test.ts +++ b/tests/e2e.v1-headless.test.ts @@ -9,7 +9,6 @@ let TEST_CONFIG_DIR = "" let TEST_OPENCODE_SUBDIR = "" let TEST_OPENCODE_CONFIG = "" let TEST_HOOKS_CONFIG = "" -let TEST_AGENT_DIR = "" let TEST_HOME_DIR = "" let TEST_XDG_CONFIG_HOME = "" let TEST_XDG_DATA_HOME = "" @@ -19,14 +18,22 @@ const LOG_WINDOW_MS = 15 * 60 * 1000 const LOG_FALLBACK_FILES = 3 const OPENCODE_COMMAND_TIMEOUT_MS = 30_000 const E2E_ENABLED = process.env.OPENCODE_E2E === "1" +const E2E_MODEL = process.env.OPENCODE_E2E_MODEL ?? "cerebras/zai-glm-4.7" function createTestSandbox(): void { + const authJson = process.env.OPENCODE_E2E_AUTH_JSON + if (!authJson) { + throw new Error( + "OPENCODE_E2E_AUTH_JSON is required for the isolated real-host E2E suite." + ) + } + JSON.parse(authJson) + TEST_SANDBOX_DIR = mkdtempSync(join(tmpdir(), "opencode-command-hooks-e2e-")) TEST_CONFIG_DIR = join(TEST_SANDBOX_DIR, "project") TEST_OPENCODE_SUBDIR = join(TEST_CONFIG_DIR, ".opencode") TEST_OPENCODE_CONFIG = join(TEST_CONFIG_DIR, "opencode.jsonc") TEST_HOOKS_CONFIG = join(TEST_OPENCODE_SUBDIR, "command-hooks.jsonc") - TEST_AGENT_DIR = join(TEST_OPENCODE_SUBDIR, "agent") TEST_HOME_DIR = join(TEST_SANDBOX_DIR, "home") TEST_XDG_CONFIG_HOME = join(TEST_SANDBOX_DIR, "xdg-config") TEST_XDG_DATA_HOME = join(TEST_SANDBOX_DIR, "xdg-data") @@ -39,6 +46,9 @@ function createTestSandbox(): void { mkdirSync(TEST_XDG_DATA_HOME, { recursive: true }) mkdirSync(TEST_XDG_CACHE_HOME, { recursive: true }) mkdirSync(LOG_DIR, { recursive: true }) + writeFileSync(join(TEST_XDG_DATA_HOME, "opencode", "auth.json"), authJson, { + mode: 0o600, + }) } function getOpenCodeEnvironment(): Record { @@ -49,6 +59,7 @@ function getOpenCodeEnvironment(): Record { XDG_DATA_HOME: TEST_XDG_DATA_HOME, XDG_CACHE_HOME: TEST_XDG_CACHE_HOME, OPENCODE_CONFIG: TEST_OPENCODE_CONFIG, + PWD: TEST_CONFIG_DIR, } } @@ -206,15 +217,6 @@ function writeTestOpencodeConfig(): void { writeFileSync(TEST_OPENCODE_CONFIG, JSON.stringify(pluginConfig, null, 2)) } -function writeTestAgent(name: string, content: string): void { - mkdirSync(TEST_AGENT_DIR, { recursive: true }) - writeFileSync(join(TEST_AGENT_DIR, `${name}.md`), content) -} - -function removeTestAgent(name: string): void { - rmSync(join(TEST_AGENT_DIR, `${name}.md`), { force: true }) -} - /** * Poll for log content until a predicate is satisfied or times out. */ @@ -234,11 +236,36 @@ async function waitForLogMatch( return getRecentLogContent() } +/** + * Poll for one exact sandbox file effect from a hook. + */ +async function waitForFileContent( + filePath: string, + expectedContent: string, + timeoutMs = 10000, + intervalMs = 500 +): Promise { + const start = Date.now() + let content: string | undefined + while (Date.now() - start < timeoutMs) { + try { + content = readFileSync(filePath, "utf-8") + if (content === expectedContent) { + return content + } + } catch { + content = undefined + } + await new Promise(resolve => setTimeout(resolve, intervalMs)) + } + return content +} + /** * Run OpenCode with a prompt and capture its successful output. */ async function runOpenCode(prompt: string): Promise { - const result = await runOpenCodeCommand(["-m", "opencode/big-pickle", "run", prompt]) + const result = await runOpenCodeCommand(["-m", E2E_MODEL, "run", prompt]) await new Promise(resolve => setTimeout(resolve, 2000)) return `${result.stdout}${result.stderr}` } @@ -260,286 +287,50 @@ describe.skipIf(!E2E_ENABLED)("V1 headless real-host E2E", () => { } }) - it("Test 1: Inject during tool.execute.after - LLM responds", async () => { - console.log("\n=== Test 1: Inject during tool.execute.after - LLM responds ===") - - const uniqueId = generateUniqueId() - console.log(`Unique ID: ${uniqueId}`) - - // Configure a hook that injects a math question after any tool execution - const config = { - tool: [ - { - id: "test1-inject-math", - when: { phase: "after", tool: "*" }, - run: "echo test1_echo_output", - inject: `What is 2+2? Reply ONLY with the number 4, nothing else.`, - }, - ], - } - writeTestConfig(config) - console.log("Config written:", JSON.stringify(config, null, 2)) - - // Run OpenCode with a simple bash command - console.log("Running OpenCode...") - const opencodeResponse = await runOpenCode("use bash to echo hello") - - console.log("OpenCode response received") - console.log(`Response length: ${String(opencodeResponse).length}`) - console.log(`Response preview: ${String(opencodeResponse).substring(0, 200)}...`) - - // Check logs for inject marker - const logContent = getRecentLogContent() - console.log(`Log content length: ${logContent.length}`) - - // Assert: OpenCode response should contain "4" (the LLM should respond to the injected question) - const containsNumber4 = opencodeResponse.includes("4") || opencodeResponse.includes(" four") - console.log(`OpenCode response contains "4": ${containsNumber4}`) - - expect(containsNumber4).toBe(true) - }, 120000) - - it("Test 2: Inject during session.idle - LLM does NOT respond", async () => { - console.log("\n=== Test 2: Inject during session.idle - LLM does NOT respond ===") - + it("runs one write after-hook transaction with isolated observable evidence", async () => { const uniqueId = generateUniqueId() - console.log(`Unique ID: ${uniqueId}`) - - // Configure a session hook that injects a message on idle - const config = { - session: [ - { - id: "test2-session-idle", - when: { event: "session.idle" }, - inject: `IGNORE_THIS_IDLE_MESSAGE_${uniqueId}`, - }, - ], - } - writeTestConfig(config) - console.log("Config written:", JSON.stringify(config, null, 2)) - - // Run OpenCode with a simple bash command - console.log("Running OpenCode...") - const opencodeResponse = await runOpenCode("use bash to echo test") - - console.log("OpenCode response received") - console.log(`Response length: ${String(opencodeResponse).length}`) - console.log(`Response preview: ${String(opencodeResponse).substring(0, 200)}...`) - - // Check logs for inject marker - const logContent = getRecentLogContent() - console.log(`Log content length: ${logContent.length}`) - console.log(`Log contains [inject]: ${logContent.includes("[inject]")}`) - console.log(`Log contains unique ID: ${logContent.includes(`IGNORE_THIS_IDLE_MESSAGE_${uniqueId}`)}`) + const writtenFileName = `e2e-write-${uniqueId}.txt` + const writtenContent = `MODEL_WRITE_${uniqueId}` + const hookProofFileName = `e2e-write-hook-${uniqueId}.txt` + const hookProofContent = `WRITE_AFTER_HOOK_${uniqueId}` + const hookStdout = `WRITE_AFTER_STDOUT_${uniqueId}` + const injectedMarker = `WRITE_AFTER_INJECT_${uniqueId}:${hookStdout}` + const toastMarker = `WRITE_AFTER_TOAST_${uniqueId}` + const writtenFilePath = join(TEST_CONFIG_DIR, writtenFileName) + const hookProofFilePath = join(TEST_CONFIG_DIR, hookProofFileName) - // Assert: OpenCode response should NOT contain the ignore message or unique ID - // The idle injection should not alter the immediate model response - const responseStr = String(opencodeResponse) - const containsIgnoreMessage = responseStr.includes(`IGNORE_THIS_IDLE_MESSAGE_${uniqueId}`) - const containsUniqueId = responseStr.includes(uniqueId) - - console.log(`OpenCode response contains IGNORE message: ${containsIgnoreMessage}`) - console.log(`OpenCode response contains unique ID: ${containsUniqueId}`) - - expect(containsIgnoreMessage).toBe(false) - expect(containsUniqueId).toBe(false) - }, 120000) - - it("Test 3: Toast marker does not leak into model output", async () => { - console.log("\n=== Test 3: Toast marker does not leak into model output ===") - - const uniqueId = generateUniqueId() - console.log(`Unique ID: ${uniqueId}`) - - // Configure a hook that shows a toast before any tool execution const config = { tool: [ { - id: "test3-toast-before", - when: { phase: "before", tool: "*" }, - run: "echo test3_echo_output", + id: `e2e-write-after-${uniqueId}`, + when: { phase: "after", tool: "*" }, + run: `printf '%s' '${hookProofContent}' > '${hookProofFileName}'; printf '%s' '${hookStdout}'`, + inject: `WRITE_AFTER_INJECT_${uniqueId}:{stdout}`, toast: { - title: "Test 3 Toast", - message: `TOAST_MARKER_${uniqueId}`, + title: "E2E write after-hook", + message: toastMarker, variant: "info", }, }, ], } writeTestConfig(config) - console.log("Config written:", JSON.stringify(config, null, 2)) - - // Run OpenCode with a prompt asking to describe what it did - console.log("Running OpenCode...") - const opencodeResponse = await runOpenCode("use bash to echo hello and describe what you did") - - console.log("OpenCode response received") - console.log(`Response length: ${opencodeResponse.length}`) - console.log(`Response preview: ${opencodeResponse.substring(0, 200)}...`) - - // Headless CLI output cannot establish that a TUI toast rendered visibly. - const toastInResponse = opencodeResponse.includes(`TOAST_MARKER_${uniqueId}`) - console.log(`Toast marker in OpenCode response: ${toastInResponse}`) - - expect(toastInResponse).toBe(false) - }, 120000) - - it("Test 4: stdout template substitution works", async () => { - console.log("\n=== Test 4: stdout template substitution works ===") - - const uniqueId = generateUniqueId() - console.log(`Unique ID: ${uniqueId}`) - - // Configure a hook that runs a command and injects with {stdout} substitution - const config = { - tool: [ - { - id: "test4-stdout-subst", - when: { phase: "after", tool: "*" }, - run: `echo CAPTURED_${uniqueId}`, - inject: `Output was: {stdout}`, - }, - ], - } - writeTestConfig(config) - console.log("Config written:", JSON.stringify(config, null, 2)) - - // Run OpenCode with a simple bash command - console.log("Running OpenCode...") - const opencodeResponse = await runOpenCode("use bash to list files") - console.log("OpenCode response received") - console.log(`Response length: ${opencodeResponse.length}`) - console.log(`Response preview: ${opencodeResponse.substring(0, 200)}...`) - - const logContent = await waitForLogMatch( - content => content.includes(`Output was: CAPTURED_${uniqueId}`), - 15000, - 500 - ) - console.log(`Log content length: ${logContent.length}`) - console.log(`Log contains [inject]: ${logContent.includes("[inject]")}`) - console.log(`Log contains CAPTURED marker: ${logContent.includes(`CAPTURED_${uniqueId}`)}`) - - // Assert: Log should contain "[inject]" with "CAPTURED_{uniqueId}" - // proving that {stdout} was substituted with the actual command output - const hasInjectLog = logContent.includes("[inject]") - const hasCapturedMarker = logContent.includes(`CAPTURED_${uniqueId}`) - const hasStdoutSubstitution = logContent.includes(`Output was: CAPTURED_${uniqueId}`) - - console.log(`Log contains [inject]: ${hasInjectLog}`) - console.log(`Log contains CAPTURED marker: ${hasCapturedMarker}`) - console.log(`Log contains stdout substitution: ${hasStdoutSubstitution}`) - - expect(hasInjectLog).toBe(true) - expect(hasCapturedMarker).toBe(true) - expect(hasStdoutSubstitution).toBe(true) - }, 120000) - - it("Test 5: Hook fires after write tool and creates file", async () => { - console.log("\n=== Test 5: Hook fires after write tool and creates file ===") - - const uniqueId = generateUniqueId() - console.log(`Unique ID: ${uniqueId}`) - - // Configure a hook that runs after the write tool - const config = { - tool: [ - { - id: `test-write-hook-${uniqueId}`, - when: { phase: "after", tool: "write" }, - run: "touch goodbye.txt" - }, - ], - } - writeTestConfig(config) - console.log("Config written:", JSON.stringify(config, null, 2)) - - // Run OpenCode with a prompt that requires the write tool. Avoid bash here - // because this test specifically verifies hooks attached to write. - console.log("Running OpenCode...") - const opencodeResponse = await runOpenCode("Use the write tool, not bash, to create a file named hello.txt with exactly this content and no trailing newline: hello") - - console.log("OpenCode response received") - console.log(`Response length: ${opencodeResponse.length}`) - console.log(`Response preview: ${opencodeResponse.substring(0, 200)}...`) - - // Check file existence and content - const helloFilePath = join(TEST_CONFIG_DIR, "hello.txt") - const goodbyeFilePath = join(TEST_CONFIG_DIR, "goodbye.txt") - - // Wait a bit for file system operations to complete - await new Promise(resolve => setTimeout(resolve, 2000)) - try { - // Assert 1: hello.txt exists - const helloExists = existsSync(helloFilePath) - console.log(`hello.txt exists: ${helloExists}`) - expect(helloExists).toBe(true) - - // Assert 2: hello.txt contains exactly "hello" - if (helloExists) { - const helloContent = readFileSync(helloFilePath, "utf-8") - console.log(`hello.txt content: "${helloContent}"`) - expect(helloContent).toBe("hello") - } - - // Assert 3: goodbye.txt exists (proving the hook ran) - const goodbyeExists = existsSync(goodbyeFilePath) - console.log(`goodbye.txt exists: ${goodbyeExists}`) - expect(goodbyeExists).toBe(true) - - // Assert 4: Logs contain evidence the write tool was used - expect(opencodeResponse.toLowerCase()).toContain("write") - + const opencodeResponse = await runOpenCode( + `Use the write tool, not bash, to create ${writtenFileName} with exactly this content and no trailing newline: ${writtenContent}` + ) + expect(await waitForFileContent(writtenFilePath, writtenContent)).toBe(writtenContent) + expect(await waitForFileContent(hookProofFilePath, hookProofContent)).toBe(hookProofContent) + const logContent = await waitForLogMatch(content => content.includes(injectedMarker)) + expect(logContent).toContain(injectedMarker) + expect(opencodeResponse).not.toContain(toastMarker) } finally { - // Clean up test files - try { - if (existsSync(helloFilePath)) { - unlinkSync(helloFilePath) - console.log("Cleaned up hello.txt") - } - if (existsSync(goodbyeFilePath)) { - unlinkSync(goodbyeFilePath) - console.log("Cleaned up goodbye.txt") + for (const filePath of [writtenFilePath, hookProofFilePath]) { + if (existsSync(filePath)) { + unlinkSync(filePath) } - } catch (e) { - console.log("Cleanup warning:", e) } } }, 120000) - - it("Test 6: Agent frontmatter hooks do not leak to provider options", async () => { - console.log("\n=== Test 6: Agent frontmatter hooks do not leak to provider options ===") - const uniqueId = generateUniqueId() - const marker = `FRONTMATTER_HOOK_${uniqueId}` - - writeTestConfig({ tool: [], session: [] }) - writeTestAgent("author", `--- -description: E2E author test agent -mode: subagent -model: cerebras/zai-glm-4.7 -hooks: - after: - - run: "echo ${marker}" - inject: "Frontmatter output: {stdout}" ---- - -Say hi tersely. -`) - - try { - console.log("Running OpenCode...") - const opencodeResponse = await runOpenCode("get author subagent to say hi") - console.log("OpenCode response received") - console.log(`Response preview: ${opencodeResponse.substring(0, 200)}...`) - - const logContent = await waitForLogMatch((content) => content.includes(marker), 20000) - expect(opencodeResponse).not.toContain("property 'hooks' is unsupported") - expect(logContent).toContain(marker) - expect(logContent).toContain("[inject]") - } finally { - removeTestAgent("author") - } - }, 120000) }) From 7b6a91746a7630f5f8ea26cf36dd0fe982c69037 Mon Sep 17 00:00:00 2001 From: Shane Bishop <71288697+shanebishop1@users.noreply.github.com> Date: Fri, 24 Jul 2026 00:42:16 -0700 Subject: [PATCH 9/9] style: remove E2E trailing whitespace --- tests/e2e.v1-headless.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/e2e.v1-headless.test.ts b/tests/e2e.v1-headless.test.ts index 556046d..75ed004 100644 --- a/tests/e2e.v1-headless.test.ts +++ b/tests/e2e.v1-headless.test.ts @@ -276,7 +276,7 @@ describe.skipIf(!E2E_ENABLED)("V1 headless real-host E2E", () => { // Verify the executable works in the same isolated environment as the tests. await runOpenCodeCommand(["--version"]) - + // Enable the plugin in the test opencode config writeTestOpencodeConfig() })