build(codex-cli): make the codex-rs fork build reproducible

.gitignore

@@ -38,6 +38,11 @@ core.[0-9]*
 secrets/**/*
 
 # Registry build artifacts
+toolchain/c/build/
+toolchain/c/vendor/
+toolchain/c/libs/
+toolchain/c/sysroot/
+software/*/bin/
 registry/native/target/
 registry/native/vendor/
 registry/native/c/build/
@@ -46,6 +51,7 @@ registry/native/c/libs/
 registry/native/c/.cache/
 registry/native/c/sysroot/
 registry/software/*/wasm/
+software/*/wasm/
 registry/software/*/agentos-package.meta.json
 registry/.last-publish-hash
 registry/software/*/.last-publish-hash

docs-internal/registry-parity-worklist.md

@@ -272,14 +272,26 @@ so a reader sees the whole board at a glance.
 - **Note:** real upstream `curl` (#6) already covers downloads, so wget is not
   urgent — but it should be retried as the real tool, not left dropped.
 
-### 9. codex-cli — not buildable in-checkout (needs external fork)
-- **Broken:** requires the external `codex-rs` fork (`CODEX_REPO` absent); tests
-  `describe.skip`.
-- **Objective:** decide the build story — vendor/pin the fork or document the
-  required checkout — so `codex`/`codex-exec` build reproducibly here, then real
-  e2e tests run (real upstream SDK per CLAUDE.md, not an API stub).
-- **Proof:** codex builds in CI/dev; `software/codex-cli/test/` runs un-skipped.
-- **rev:** `build(codex-cli): make the codex-rs fork build reproducible`
+### 9. codex-cli — DONE
+- **Resolved:** the `codex`/`codex-exec` package now has an AgentOS-owned wrapper
+  for the external `codex-rs` fork build. `make -C toolchain codex-required`
+  requires `CODEX_REPO=/path/to/codex-rs/codex-rs`, uses this checkout's
+  `toolchain/c/vendor/wasi-sdk`, and installs the fork-built optimized wasm into
+  generated `software/codex/wasm/{codex-exec,codex}` for the package build. The
+  generated toolchain and wasm command directories are ignored and not committed.
+- **Test fix:** the real `codex-exec --session-turn` e2e now uses a streaming
+  Responses mock (SSE) and disables Codex shell snapshots inside the VM config,
+  avoiding the optional pre-turn shell-snapshot subprocess deadlock while still
+  driving the real codex-core agent and shell tool path.
+- **Proof:** `CODEX_REPO=/home/nathan/agent-e2e/codex-rs/codex-rs make -C
+  toolchain codex-required` builds and installs 29,924,651-byte command artifacts
+  in `2026-07-08T01-37-05-0700-item9-codex-build-rerun.txt`; `pnpm --dir
+  software/codex-cli build` stages 2 commands and assembles `package.aospkg` in
+  `2026-07-08T01-44-50-0700-item9-codex-cli-build.txt`;
+  `AGENTOS_E2E_FULL=1 pnpm --dir packages/core exec vitest run
+  tests/codex-fullturn.test.ts --reporter=verbose` passes 2 real VM tests in
+  `2026-07-08T01-53-55-0700-item9-core-codex-fullturn-pass.txt`.
+- **rev:** `svksnzon` — `build(codex-cli): make the codex-rs fork build reproducible`
 
 ### 10. vix — DONE (deleted)
 - **Resolved:** `vix` was a from-scratch, source-less drop-zone binary — exactly

packages/core/tests/codex-fullturn.test.ts

@@ -7,6 +7,13 @@ import {
 } from "./helpers/openai-responses-mock.js";
 import { REGISTRY_SOFTWARE } from "./helpers/registry-commands.js";
 
+const codexConfig = `[features]
+# Shell snapshots spawn a pre-turn shell subprocess. The real turn coverage
+# below does not need that optional context, and disabling it keeps the WASI VM
+# focused on the codex-core model/tool path under test.
+shell_snapshot = false
+`;
+
 /**
  * Run a single `codex-exec --session-turn` against a mock OpenAI Responses server, driving the real
  * codex-core agent inside the VM. `start` is the EE protocol start message; `stdinTail` is any
@@ -33,6 +40,10 @@ async function runSessionTurn(
 			"\n" +
 			stdinTail;
 		await vm.execArgv("mkdir", ["-p", "/root/.codex"]);
+		await vm.writeFile(
+			"/root/.codex/config.toml",
+			new TextEncoder().encode(codexConfig),
+		);
 		const r = await vm.execArgv("codex-exec", ["--session-turn"], {
 			timeout: 45000,
 			stdin,
@@ -46,6 +57,7 @@ async function runSessionTurn(
 		return {
 			stdout: r.stdout ?? "",
 			stderr: r.stderr ?? "",
+			exitCode: r.exitCode,
 			requests: mock.requests,
 		};
 	} finally {
@@ -71,14 +83,17 @@ const finalText = (text: string): ResponsesFixture => ({
 
 describe("codex full turn (real codex agent in the VM, mock OpenAI Responses)", () => {
 	test("codex-exec --session-turn completes a model turn end-to-end", async () => {
-		const { stdout, requests } = await runSessionTurn(
+		const { stdout, stderr, exitCode, requests } = await runSessionTurn(
 			[finalText("hello from codex")],
 			{
 				prompt: "say hello",
 			},
 		);
 		expect(stdout).toContain('"type":"start"');
-		expect(requests.length).toBeGreaterThan(0);
+		expect(
+			requests.length,
+			`codex-exec did not call mock Responses; exitCode=${exitCode}; stderr=${stderr}; stdout=${stdout}`,
+		).toBeGreaterThan(0);
 		// The engine must surface the assistant text as a text_delta — whether the
 		// model streamed deltas or returned a single final AgentMessage — not just
 		// reach `done`. (A prior `/(done|text_delta|error)/` regex passed on `done`
@@ -91,7 +106,7 @@ describe("codex full turn (real codex agent in the VM, mock OpenAI Responses)",
 	test("runs a shell tool call with on-request approval and reports tool_call updates", async () => {
 		const sawToolOutput = (body: Record<string, unknown>) =>
 			JSON.stringify(body).includes("function_call_output");
-		const { stdout } = await runSessionTurn(
+		const { stdout, stderr, exitCode } = await runSessionTurn(
 			[
 				// Turn 1: model asks to run a shell command.
 				{
@@ -131,7 +146,10 @@ describe("codex full turn (real codex agent in the VM, mock OpenAI Responses)",
 			// Approve the exec when the engine emits permission_request.
 			`${JSON.stringify({ decision: "allow" })}\n`,
 		);
-		expect(stdout).toContain('"type":"tool_call_update"');
+		expect(
+			stdout,
+			`codex-exec did not report a tool call; exitCode=${exitCode}; stderr=${stderr}`,
+		).toContain('"type":"tool_call_update"');
 		expect(stdout).toContain('"type":"done"');
 	}, 70000);
 

packages/core/tests/helpers/openai-responses-mock.ts

@@ -43,6 +43,52 @@ function writeJson(
 	res.end(payload);
 }
 
+function writeSse(
+	res: ServerResponse,
+	events: Record<string, unknown>[],
+): void {
+	const payload = events
+		.map((event) => {
+			const type = String(event.type);
+			return `event: ${type}\ndata: ${JSON.stringify(event)}\n\n`;
+		})
+		.join("");
+	res.statusCode = 200;
+	res.setHeader("content-type", "text/event-stream");
+	res.setHeader("cache-control", "no-cache");
+	res.setHeader("content-length", Buffer.byteLength(payload));
+	res.end(payload);
+}
+
+function responseToSseEvents(response: Record<string, unknown>) {
+	const id = typeof response.id === "string" ? response.id : "resp_mock";
+	const events: Record<string, unknown>[] = [
+		{ type: "response.created", response: { id } },
+	];
+	const output = Array.isArray(response.output) ? response.output : [];
+	for (const item of output) {
+		if (!item || typeof item !== "object") continue;
+		events.push({
+			type: "response.output_item.done",
+			item,
+		});
+	}
+	events.push({
+		type: "response.completed",
+		response: {
+			id,
+			usage: {
+				input_tokens: 0,
+				input_tokens_details: null,
+				output_tokens: 0,
+				output_tokens_details: null,
+				total_tokens: 0,
+			},
+		},
+	});
+	return events;
+}
+
 export async function startResponsesMock(
 	fixtures: ResponsesFixture[],
 ): Promise<RunningResponsesMock> {
@@ -69,7 +115,11 @@ export async function startResponsesMock(
 			if (fixture.delayMs) {
 				await new Promise((resolve) => setTimeout(resolve, fixture.delayMs));
 			}
-			writeJson(res, 200, fixture.response);
+			if (body.stream === true) {
+				writeSse(res, responseToSseEvents(fixture.response));
+			} else {
+				writeJson(res, 200, fixture.response);
+			}
 		} catch (error) {
 			writeJson(res, 500, {
 				error: "invalid_request",

packages/core/vitest.config.ts

@@ -30,6 +30,7 @@ const SLOW_E2E_FILES = [
 	"tests/readdir-recursive.test.ts",
 	"tests/cron-integration.test.ts",
 	"tests/pi-cli-headless.test.ts",
+	"tests/codex-fullturn.test.ts",
 ];
 
 // Pre-existing failures NOT caused by this branch (they were red before CI ever
@@ -54,11 +55,6 @@ const KNOWN_FAILING_E2E_FILES = [
 	//  - shell-flat-api: openShell/writeShell/onShellData yields empty output.
 	"tests/duckdb-package.test.ts",
 	"tests/shell-flat-api.test.ts",
-	// codex-fullturn: the pinned @agentos-software/codex package intentionally
-	// stubs the turn ("codex-exec --session-turn is disabled until the real Codex
-	// agent package is wired"). Pre-existing unwired-feature state, not a
-	// regression — re-enable once the real Codex agent package is wired.
-	"tests/codex-fullturn.test.ts",
 ];
 
 // Real-API, real-install matrix (agent × package manager). Hits a live LLM API

toolchain/Makefile

@@ -56,22 +56,25 @@ commands: wasm
 	$(MAKE) -C c programs install
 
 # Build the real wasm32-wasip1 `codex-exec` agent engine from the codex fork
-# (branch wasi-port-codex-core). The fork ships a committed, idempotent, self-prepping build script
-# (it prepares the rustup sysroot for -Z build-std and restores it on exit — no manual stash), so
-# this folds codex into the toolchain: `make -C toolchain codex` produces and installs
-# software/codex/wasm/{codex-exec,codex}. Override CODEX_REPO if the fork is not at the
-# default sibling path. See the fork's docs/wasi-build.md for the toolchain constraints, and the
-# remaining work to source the CRT from wasi-sdk so the prebuilt rust target need not be installed.
+# (branch wasi-port-codex-core) and install it where @agentos-software/codex-cli
+# stages commands from. The fork remains external; CODEX_REPO documents the
+# required checkout instead of silently falling back to the old stub crates.
 AGENTOS_ROOT := $(abspath $(CURDIR)/..)
 CODEX_REPO ?= $(abspath $(AGENTOS_ROOT)/../codex-rs/codex-rs)
-codex:
+CODEX_WASI_SDK_DIR := $(abspath $(CURDIR)/c/vendor/wasi-sdk)
+codex: c/vendor/wasi-sdk/bin/clang
 	@if [ -x "$(CODEX_REPO)/scripts/build-wasi-codex-exec.sh" ]; then \
-		AGENTOS_DIR="$(AGENTOS_ROOT)" "$(CODEX_REPO)/scripts/build-wasi-codex-exec.sh"; \
+		CODEX_REPO="$(CODEX_REPO)" \
+		WASI_SDK_DIR="$(CODEX_WASI_SDK_DIR)" \
+		"$(CURDIR)/scripts/build-codex-wasi.sh"; \
 	else \
 		echo "codex: fork build script not found at $(CODEX_REPO)/scripts/build-wasi-codex-exec.sh — skipping."; \
 		echo "       Set CODEX_REPO=/path/to/codex-rs/codex-rs (the wasi-port-codex-core checkout) to build it."; \
 	fi
 
+c/vendor/wasi-sdk/bin/clang:
+	$(MAKE) -C c wasi-sdk
+
 # Strict variant (fails if the fork is absent) for environments that require the codex artifact.
 .PHONY: codex-required
 codex-required:

toolchain/scripts/build-codex-wasi.sh

@@ -0,0 +1,68 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+usage() {
+	cat <<'USAGE'
+Usage:
+  CODEX_REPO=/path/to/codex-rs/codex-rs toolchain/scripts/build-codex-wasi.sh
+
+Builds the external Codex WASI fork and installs the real codex-exec artifact
+into software/codex/wasm/{codex-exec,codex}, where @agentos-software/codex-cli
+stages commands from.
+
+Env:
+  CODEX_REPO   Required. Checkout of the Codex fork with scripts/build-wasi-codex-exec.sh.
+  WASI_SDK_DIR Optional. Defaults to toolchain/c/vendor/wasi-sdk.
+  DEST_DIR     Optional. Defaults to software/codex/wasm.
+USAGE
+}
+
+if [ "${1:-}" = "--help" ]; then
+	usage
+	exit 0
+fi
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+TOOLCHAIN_DIR="$(cd "$SCRIPT_DIR/.." && pwd)"
+AGENTOS_ROOT="$(cd "$TOOLCHAIN_DIR/.." && pwd)"
+
+CODEX_REPO="${CODEX_REPO:-}"
+WASI_SDK_DIR="${WASI_SDK_DIR:-$TOOLCHAIN_DIR/c/vendor/wasi-sdk}"
+DEST_DIR="${DEST_DIR:-$AGENTOS_ROOT/software/codex/wasm}"
+
+if [ -z "$CODEX_REPO" ]; then
+	echo "ERROR: CODEX_REPO is required." >&2
+	echo "       Set it to the wasi-port-codex-core checkout, for example:" >&2
+	echo "       CODEX_REPO=/path/to/codex-rs/codex-rs make -C toolchain codex-required" >&2
+	exit 1
+fi
+
+BUILD_SCRIPT="$CODEX_REPO/scripts/build-wasi-codex-exec.sh"
+if [ ! -x "$BUILD_SCRIPT" ]; then
+	echo "ERROR: codex fork build script not found or not executable: $BUILD_SCRIPT" >&2
+	exit 1
+fi
+
+if [ ! -x "$WASI_SDK_DIR/bin/clang" ]; then
+	echo "ERROR: wasi-sdk clang not found at $WASI_SDK_DIR/bin/clang" >&2
+	echo "       Run: make -C toolchain/c wasi-sdk" >&2
+	exit 1
+fi
+
+echo "== codex repo: $CODEX_REPO =="
+echo "== wasi-sdk:   $WASI_SDK_DIR =="
+echo "== dest:       $DEST_DIR =="
+
+WASI_SDK_DIR="$(cd "$WASI_SDK_DIR" && pwd)" INSTALL=0 "$BUILD_SCRIPT"
+
+OUT="$CODEX_REPO/target/wasm32-wasip1/release/codex-exec.opt.wasm"
+if [ ! -f "$OUT" ]; then
+	echo "ERROR: expected build output missing: $OUT" >&2
+	exit 1
+fi
+
+mkdir -p "$DEST_DIR"
+cp "$OUT" "$DEST_DIR/codex-exec"
+cp "$OUT" "$DEST_DIR/codex"
+
+echo "== installed $(wc -c < "$OUT") bytes to $DEST_DIR/{codex-exec,codex} =="