Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
43 changes: 32 additions & 11 deletions src/CodexAcpServer.ts
Original file line number Diff line number Diff line change
@@ -1,5 +1,8 @@
import * as acp from "@agentclientprotocol/sdk";
import {RequestError, type SessionId, type SessionModeState} from "@agentclientprotocol/sdk";
import {readFile} from "node:fs/promises";
import {extname} from "node:path";
import {fileURLToPath, pathToFileURL} from "node:url";
import {CodexEventHandler} from "./CodexEventHandler";
import {CodexApprovalHandler} from "./CodexApprovalHandler";
import {CodexElicitationHandler} from "./CodexElicitationHandler";
Expand Down Expand Up @@ -80,6 +83,7 @@ import {
createAgentTextMessageChunk,
createAgentTextThoughtChunk,
createUserMessageChunk,
visibleUserMessageText,
} from "./ContentChunks";
import {
sameThreadGoalSnapshot,
Expand Down Expand Up @@ -1232,13 +1236,11 @@ export class CodexAcpServer {
}
}

private createUserMessageUpdates(item: ThreadItem & { type: "userMessage" }): UpdateSessionEvent[] {
private async createUserMessageUpdates(item: ThreadItem & { type: "userMessage" }): Promise<UpdateSessionEvent[]> {
const updates: UpdateSessionEvent[] = [];
const messageId = item.id;
for (const input of item.content) {
const blocks = this.userInputToContentBlocks(input);
for (const block of blocks) {
updates.push(createUserMessageChunk(block, messageId));
for (const block of await this.userInputToContentBlocks(input)) {
updates.push(createUserMessageChunk(block, item.id));
}
}
return updates;
Expand Down Expand Up @@ -1291,20 +1293,39 @@ export class CodexAcpServer {
};
}

private userInputToContentBlocks(input: UserInput): acp.ContentBlock[] {
private async userInputToContentBlocks(input: UserInput): Promise<acp.ContentBlock[]> {
switch (input.type) {
case "text":
return input.text.length > 0 ? [{ type: "text", text: input.text }] : [];
case "image":
case "text": {
const visibleText = visibleUserMessageText(input.text);
return visibleText.length > 0 ? [{ type: "text", text: visibleText }] : [];
}
case "image": {
const match = /^data:(image\/[^;]+);base64,(.+)$/.exec(input.url);
const mimeType = match?.[1];
const data = match?.[2];
if (mimeType && data) {
return [{ type: "image", mimeType, data }];
}
return [{ type: "text", text: this.formatUriAsLink("image", input.url) }];
}
case "localImage": {
const uri = input.path.startsWith("file://") ? input.path : `file://${input.path}`;
const path = input.path.startsWith("file://") ? fileURLToPath(input.path) : input.path;
const uri = pathToFileURL(path).href;
const extension = extname(path).slice(1).toLowerCase();
const data = extension ? await readFile(path).catch(() => null) : null;
if (data) {
const mimeType = `image/${extension === "jpg" ? "jpeg" : extension}`;
return [{ type: "image", data: data.toString("base64"), mimeType, uri }];
}
return [{ type: "text", text: this.formatUriAsLink(null, uri) }];
}
case "skill":
return [{ type: "text", text: `skill:${input.name} (${input.path})` }];
case "mention": {
const uri = input.path.startsWith("file://") ? input.path : pathToFileURL(input.path).href;
return [{ type: "resource_link", name: input.name, uri }];
}
}
return [];
}

private formatUriAsLink(name: string | null, uri: string): string {
Expand Down
14 changes: 14 additions & 0 deletions src/ContentChunks.ts
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,20 @@ import type {UpdateSessionEvent} from "./ACPSessionConnection";

type AcpMeta = Record<string, unknown>;

const FILES_MENTIONED_HEADER = "# Files mentioned by the user:\n";
const REQUEST_MARKER = "\n## My request for Codex:\n";

export function visibleUserMessageText(text: string): string {
const normalized = text.trimStart();
const requestIndex = normalized.indexOf(REQUEST_MARKER);

if (normalized.startsWith(FILES_MENTIONED_HEADER) && requestIndex !== -1) {
return normalized.slice(requestIndex + REQUEST_MARKER.length);
}

return text;
}

export function createCodexMessagePhaseMeta(phase: string | null | undefined): AcpMeta | undefined {
if (!phase) {
return undefined;
Expand Down
32 changes: 5 additions & 27 deletions src/ResponseItemHistoryFallback.ts
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@ import { stripShellPrefix } from "./CommandUtils";
import type { CommandAction, Thread, ThreadItem } from "./app-server/v2";
import { createCommandActionEvent } from "./CodexToolCallMapper";
import { createTerminalOutputMeta, type TerminalOutputMode } from "./TerminalOutputMode";
import { createAgentMessageChunk, createCodexMessagePhaseMeta } from "./ContentChunks";
import { createAgentMessageChunk, createCodexMessagePhaseMeta, createUserMessageChunk, visibleUserMessageText } from "./ContentChunks";

type JsonRecord = Record<string, unknown>;
type AcpToolCallEvent = Extract<UpdateSessionEvent, { sessionUpdate: "tool_call" }>;
Expand Down Expand Up @@ -262,18 +262,12 @@ function createEventMsgUpdates(record: JsonRecord): UpdateSessionEvent[] | null
}

function createUserMessageEventUpdates(payload: JsonRecord): UpdateSessionEvent[] {
const blocks: ContentBlock[] = [];
const message = stringValue(payload["message"]);
if (message !== null && message.length > 0) {
blocks.push({ type: "text", text: message });
const text = visibleUserMessageText(stringValue(payload["message"]) ?? "");
if (text.length > 0) {
return [createUserMessageChunk({ type: "text", text })];
}
blocks.push(...imageBlocks(payload["images"]));
blocks.push(...imageBlocks(payload["local_images"]));

return blocks.map((content) => ({
sessionUpdate: "user_message_chunk",
content,
}));
return [];
}

function createAgentReasoningEventUpdates(payload: JsonRecord): UpdateSessionEvent[] {
Expand All @@ -288,22 +282,6 @@ function createAgentReasoningEventUpdates(payload: JsonRecord): UpdateSessionEve
}];
}

function imageBlocks(images: unknown): ContentBlock[] {
if (!Array.isArray(images)) {
return [];
}

return images.flatMap((image): ContentBlock[] => {
if (typeof image === "string") {
return [{ type: "text", text: `[@image](${image})` }];
}

const record = asRecord(image);
const path = record ? stringValue(record["path"]) ?? stringValue(record["url"]) : null;
return path ? [{ type: "text", text: `[@image](${path})` }] : [];
});
}

function contentBlocksFromResponseContent(content: unknown): ContentBlock[] {
if (!Array.isArray(content)) {
return [];
Expand Down
54 changes: 53 additions & 1 deletion src/__tests__/CodexACPAgent/data/load-session-history.json
Original file line number Diff line number Diff line change
Expand Up @@ -151,6 +151,58 @@
}
]
}
{
"method": "sessionUpdate",
"args": [
{
"sessionId": "session-1",
"update": {
"sessionUpdate": "user_message_chunk",
"messageId": "item-user-1",
"content": {
"type": "image",
"mimeType": "image/png",
"data": "dGVzdCBpbWFnZQ=="
}
}
}
]
}
{
"method": "sessionUpdate",
"args": [
{
"sessionId": "session-1",
"update": {
"sessionUpdate": "user_message_chunk",
"messageId": "item-user-1",
"content": {
"type": "image",
"data": "dGVzdCBpbWFnZQ==",
"mimeType": "image/png",
"uri": "file:///tmp/codex-acp-load-session-image.png"
}
}
}
]
}
{
"method": "sessionUpdate",
"args": [
{
"sessionId": "session-1",
"update": {
"sessionUpdate": "user_message_chunk",
"messageId": "item-user-1",
"content": {
"type": "resource_link",
"name": "notes.txt",
"uri": "file:///test/project/notes.txt"
}
}
}
]
}
{
"method": "sessionUpdate",
"args": [
Expand Down Expand Up @@ -429,4 +481,4 @@
}
}
]
}
}
21 changes: 19 additions & 2 deletions src/__tests__/CodexACPAgent/load-session.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -3,11 +3,16 @@ import type * as acp from "@agentclientprotocol/sdk";
import { mkdtemp, rm, writeFile } from "node:fs/promises";
import { tmpdir } from "node:os";
import { join } from "node:path";
import { pathToFileURL } from "node:url";
import { createCodexMockTestFixture, createTestModel } from "../acp-test-utils";
import type { Model, Thread, ThreadGoal } from "../../app-server/v2";

describe("CodexACPAgent - loadSession", () => {
it("should replay history during loadSession", async () => {
const localImageDirectory = await mkdtemp(join(tmpdir(), "codex-acp-load-session-"));
const localImagePath = join(localImageDirectory, "image.png");
await writeFile(localImagePath, "test image");

const fixture = createCodexMockTestFixture();
const codexAcpAgent = fixture.getCodexAcpAgent();
const codexAcpClient = fixture.getCodexAcpClient();
Expand Down Expand Up @@ -82,8 +87,15 @@ describe("CodexACPAgent - loadSession", () => {
id: "item-user-1",
clientId: null,
content: [
{ type: "text", text: "Hi", text_elements: [] },
{
type: "text",
text: `\n# Files mentioned by the user:\n\n## image.png: ${localImagePath}\n\n## My request for Codex:\nHi`,
text_elements: [],
},
{ type: "image", url: "https://example.com/image.png" },
{ type: "image", url: "data:image/png;base64,dGVzdCBpbWFnZQ==" },
{ type: "localImage", path: localImagePath },
{ type: "mention", name: "notes.txt", path: "/test/project/notes.txt" },
],
},
{
Expand Down Expand Up @@ -223,9 +235,14 @@ describe("CodexACPAgent - loadSession", () => {
includeTurns: true,
});
expect(codexAppServerClient.threadGoalGet).toHaveBeenCalledWith({ threadId: thread.id });
await expect(fixture.getAcpConnectionDump([])).toMatchFileSnapshot(
const replay = fixture.getAcpConnectionDump([]).replaceAll(
pathToFileURL(localImagePath).href,
"file:///tmp/codex-acp-load-session-image.png",
);
await expect(`${replay}\n`).toMatchFileSnapshot(
"data/load-session-history.json"
);
await rm(localImageDirectory, { recursive: true });
});

it("should not recover session mcp servers during loadSession when request omits them", async () => {
Expand Down
26 changes: 26 additions & 0 deletions src/__tests__/CodexACPAgent/response-item-history-fallback.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -55,6 +55,24 @@ describe("ResponseItemHistoryFallback", () => {
expect(thoughtTexts(updates)).toEqual(["Need to inspect the directory."]);
});

it("leaves user attachments to thread history", () => {
const updates = parseResponseItemHistoryFallback(jsonl([
{
type: "event_msg",
payload: {
type: "user_message",
message: "\n# Files mentioned by the user:\n\n## screenshot.png: /tmp/screenshot.png\n\n## My request for Codex:\nInspect the screenshot",
images: [],
local_images: ["/tmp/screenshot.png"],
},
},
functionCall("call-missing", "ls"),
functionCallOutput("call-missing", "Chunk ID: missing\nProcess exited with code 0\nOutput:\nREADME.md\n"),
]), "terminal_output");

expect(userMessageTexts(updates)).toEqual(["Inspect the screenshot"]);
});

it("preserves assistant message phase metadata from response items", () => {
const updates = parseResponseItemHistoryFallback(jsonl([
{
Expand Down Expand Up @@ -154,6 +172,14 @@ function thoughtTexts(updates: UpdateSessionEvent[] | null): string[] {
.flatMap((update) => update.content.type === "text" ? [update.content.text] : []);
}

function userMessageTexts(updates: UpdateSessionEvent[] | null): string[] {
return (updates ?? [])
.filter((update): update is Extract<UpdateSessionEvent, { sessionUpdate: "user_message_chunk" }> => (
update.sessionUpdate === "user_message_chunk"
))
.flatMap((update) => update.content.type === "text" ? [update.content.text] : []);
}

function agentMessageMetas(updates: UpdateSessionEvent[] | null): unknown[] {
return (updates ?? [])
.filter((update): update is Extract<UpdateSessionEvent, { sessionUpdate: "agent_message_chunk" }> => (
Expand Down