Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
170 changes: 170 additions & 0 deletions test/harness/replayingCapiProxy.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -724,6 +724,111 @@ Always include PINEAPPLE_COCONUT_42.
});
}

async function replayToolResult(
toolName: string,
requestContent: string,
savedResults: Array<{ content?: string; response: string }>,
savedErrors: Array<{
content: string;
status: number;
message: string;
}> = [],
): Promise<string | null> {
const toolArguments = toolName === "view" ? '{"path":"file.txt"}' : "{}";
const createMessages = (
content: string | undefined,
response?: string,
): NormalizedData["conversations"][number]["messages"] => {
const messages: NormalizedData["conversations"][number]["messages"] = [
{ role: "system", content: "${system}" },
{ role: "user", content: "Use the tool" },
{
role: "assistant",
tool_calls: [
{
id: "toolcall_0",
type: "function",
function: {
name: toolName,
arguments: toolArguments,
},
},
],
},
...(content === undefined
? [{ role: "tool" as const, tool_call_id: "toolcall_0" }]
: [
{
role: "tool" as const,
tool_call_id: "toolcall_0",
content,
},
]),
];
if (response !== undefined) {
messages.push({ role: "assistant", content: response });
}
return messages;
};
const cachePath = path.join(tempDir, "cache.yaml");
const cacheContent = yaml.stringify({
models: ["test-model"],
errors: savedErrors.map((savedError) => ({
model: "test-model",
status: savedError.status,
message: savedError.message,
messages: createMessages(savedError.content),
})),
conversations: savedResults.map((savedResult) => ({
messages: createMessages(savedResult.content, savedResult.response),
})),
} satisfies NormalizedData);
await writeFile(cachePath, cacheContent);

const proxy = new ReplayingCapiProxy(
"http://localhost:9999",
cachePath,
workDir,
);
const proxyUrl = await proxy.start();

try {
const response = await makeRequest(proxyUrl, "/chat/completions", {
body: {
model: "test-model",
messages: [
{ role: "system", content: "System prompt" },
{ role: "user", content: "Use the tool" },
{
role: "assistant",
tool_calls: [
{
id: "request-tool-call",
type: "function",
function: {
name: toolName,
arguments: toolArguments,
},
},
],
},
{
role: "tool",
tool_call_id: "request-tool-call",
content: requestContent,
},
],
},
});

expect(response.status).toBe(200);
return (JSON.parse(response.body) as ChatCompletion).choices[0].message
.content;
} finally {
await proxy.stop();
}
}

test("returns cached response when request matches prefix", async () => {
const cachePath = path.join(tempDir, "cache.yaml");
const cacheContent = yaml.stringify({
Expand Down Expand Up @@ -820,6 +925,71 @@ Always include PINEAPPLE_COCONUT_42.
}
});

test("matches legacy numbered view results against raw cached content", async () => {
const result = await replayToolResult(
"view",
"42. alpha\r\n43. beta\r\n44.",
[{ content: "alpha\r\nbeta", response: "legacy view matched" }],
);

expect(result).toBe("legacy view matched");
});

test("matches a legacy numbered empty view result against omitted raw content", async () => {
const result = await replayToolResult("view", "1.", [
{ response: "empty view matched" },
]);

expect(result).toBe("empty view matched");
});

test("prefers an exact naturally numbered view result match", async () => {
const result = await replayToolResult("view", "1. alpha\n2. beta", [
{ content: "alpha\nbeta", response: "legacy fallback" },
{ content: "1. alpha\n2. beta", response: "exact match" },
]);

expect(result).toBe("exact match");
});

test("prefers an exact response over a legacy-fallback error", async () => {
const result = await replayToolResult(
"view",
"1. alpha\n2. beta",
[{ content: "1. alpha\n2. beta", response: "exact response" }],
[
{
content: "alpha\nbeta",
status: 429,
message: "legacy fallback error",
},
],
);

expect(result).toBe("exact response");
});

test("does not use legacy view compatibility for other tools", async () => {
const result = await replayToolResult("grep", "1. alpha\n2. beta", [
{ content: "alpha\nbeta", response: "incorrect fallback" },
{ content: "1. alpha\n2. beta", response: "exact grep match" },
]);

expect(result).toBe("exact grep match");
});

test("does not strip non-sequential view results", async () => {
const result = await replayToolResult("view", "1. alpha\n3. gamma", [
{ content: "alpha\ngamma", response: "incorrect fallback" },
{
content: "1. alpha\n3. gamma",
response: "exact non-sequential match",
},
]);

expect(result).toBe("exact non-sequential match");
});

test("matches shell tool results with shell ID completion markers", async () => {
const originalShellConfig =
process.platform === "win32" ? ShellConfig.powerShell : ShellConfig.bash;
Expand Down
Loading
Loading