diff --git a/bench/README.md b/bench/README.md index 4c739e4b..697584a8 100644 --- a/bench/README.md +++ b/bench/README.md @@ -228,8 +228,9 @@ and deletes every browser. - `agreement` (optional) compares the arms' answers within a pair. `{"path": "price", "better": "lower", "tolerance": 1}` counts pairs where one arm missed the cheapest fare. `{"path": "issues[]", "same": true}` is for one right answer that changes too often to pin in rules, and - counts both-correct pairs whose answers differ as sets. Leave it off intents with many right - answers. + counts both-correct pairs whose answers differ as sets. A path without `[]` compares the whole + value, so `{"path": "games", "same": true}` also counts a reordered array as differing. Leave it + off intents with many right answers. - `opt_in` (optional) is a reason the intent runs only when named with `--intent`, such as a checkout intent not yet rehearsed on the live site. diff --git a/bench/report.mjs b/bench/report.mjs index 86295291..02311259 100644 --- a/bench/report.mjs +++ b/bench/report.mjs @@ -53,9 +53,10 @@ export function armStats(rows) { }; } -// Strings compare case- and whitespace-insensitively, and objects regardless of key order. +// Strings compare case- and whitespace-insensitively and without the trademark symbols agents copy +// inconsistently from names, and objects regardless of key order. const canonical = (value) => JSON.stringify(value, (key, v) => { - if (typeof v === "string") return v.trim().toLowerCase().replace(/\s+/g, " "); + if (typeof v === "string") return v.replace(/[™®℠©]/g, "").trim().toLowerCase().replace(/\s+/g, " "); if (v && typeof v === "object" && !Array.isArray(v)) return Object.fromEntries(Object.entries(v).sort(([a], [b]) => a.localeCompare(b))); return v; }); diff --git a/bench/run.mjs b/bench/run.mjs index 0ba28e2a..89c3f521 100644 --- a/bench/run.mjs +++ b/bench/run.mjs @@ -540,13 +540,20 @@ async function setUpBrowser(plan, intentPlan, arm, label) { } // Reads where a prepare intent's agent left the browser from the page itself, the same way in both -// arms, so the grade does not depend on the tools under test. A page that cannot be read aborts the -// trial rather than grading it wrong. +// arms, so the grade does not depend on the tools under test. An agent can finish while its last +// action is still loading a page, so a read the navigation interrupts is tried again on the new page. +// A page that cannot be read aborts the trial rather than grading it wrong. async function readEndState(browser, intent) { - try { - return { output: await browsers.evaluate(browser, readPage, readPageArg(intent.check.fields)) }; - } catch (error) { - throw new Error(`could not read the end state: ${error.message}`); + for (let attempt = 1; ; attempt++) { + try { + return { output: await browsers.evaluate(browser, readPage, readPageArg(intent.check.fields)) }; + } catch (error) { + if (attempt < 3 && /Execution context was destroyed/.test(error.message)) { + await new Promise(resolve => setTimeout(resolve, 3000)); + continue; + } + throw new Error(`could not read the end state: ${error.message}`); + } } } diff --git a/bench/schemas/app-listing.json b/bench/schemas/app-listing.json new file mode 100644 index 00000000..b5c49803 --- /dev/null +++ b/bench/schemas/app-listing.json @@ -0,0 +1,24 @@ +{ + "type": "object", + "required": [ + "app", + "rating", + "review_count", + "cheapest_paid_monthly_price" + ], + "properties": { + "app": { + "type": "string", + "minLength": 1 + }, + "rating": { + "type": "number" + }, + "review_count": { + "type": "integer" + }, + "cheapest_paid_monthly_price": { + "type": "number" + } + } +} diff --git a/bench/schemas/app-listings.json b/bench/schemas/app-listings.json new file mode 100644 index 00000000..5e1fcea9 --- /dev/null +++ b/bench/schemas/app-listings.json @@ -0,0 +1,19 @@ +{ + "type": "object", + "required": ["apps"], + "properties": { + "apps": { + "type": "array", + "items": { + "type": "object", + "required": ["app", "rating", "review_count", "cheapest_paid_monthly_price"], + "properties": { + "app": { "type": "string", "minLength": 1 }, + "rating": { "type": "number" }, + "review_count": { "type": "integer" }, + "cheapest_paid_monthly_price": { "type": "number" } + } + } + } + } +} diff --git a/bench/schemas/game-prices.json b/bench/schemas/game-prices.json new file mode 100644 index 00000000..c3557faa --- /dev/null +++ b/bench/schemas/game-prices.json @@ -0,0 +1,18 @@ +{ + "type": "object", + "required": ["games"], + "properties": { + "games": { + "type": "array", + "items": { + "type": "object", + "required": ["game", "price", "review_summary"], + "properties": { + "game": { "type": "string", "minLength": 1 }, + "price": { "type": "number" }, + "review_summary": { "type": "string", "minLength": 1 } + } + } + } + } +} diff --git a/bench/tasks/doordash.com/doordash-pizza-near-address.json b/bench/tasks/doordash.com/doordash-pizza-near-address.json index 2ebccd6e..ef28d5a4 100644 --- a/bench/tasks/doordash.com/doordash-pizza-near-address.json +++ b/bench/tasks/doordash.com/doordash-pizza-near-address.json @@ -1,8 +1,8 @@ { "id": "doordash-pizza-near-address", "domain": "doordash.com", - "stage": "find", - "prompt": "On doordash.com, with the delivery address set to {{address}}, find three pizza restaurants that deliver there and the delivery fee the site shows for each. Reply with JSON: {\"places\": [{\"name\": string, \"detail\": the delivery fee as shown}]}.", + "stage": "prepare", + "prompt": "On doordash.com, set the delivery address to {{address}} with the site's address picker, then find three pizza restaurants that deliver there and the delivery fee the site shows for each. Leave the address set when you finish. Reply with JSON: {\"places\": [{\"name\": string, \"detail\": the delivery fee as shown}]}.", "fixture": { "address": "Times Square, New York, NY 10036" }, @@ -25,6 +25,14 @@ "matches": "\\$\\s*\\d|free" } ], + "check": { + "rules": [ + { + "path": "text", + "matches": "DoorDash[ \\t]*\\n[ \\t]*(?:\\d+[ \\t]+)?Times Sq(?:uare)?[ \\t]*(?:\\n|$)" + } + ] + }, "pairs": 5, "trial_timeout_sec": 600, "max_turns": 40 diff --git a/bench/tasks/shopify.com/shopify-app-store-compare.json b/bench/tasks/shopify.com/shopify-app-store-compare.json new file mode 100644 index 00000000..91aa933b --- /dev/null +++ b/bench/tasks/shopify.com/shopify-app-store-compare.json @@ -0,0 +1,22 @@ +{ + "id": "shopify-app-store-compare", + "domain": "shopify.com", + "stage": "compare", + "prompt": "On the Shopify App Store (apps.shopify.com), search for \"{{query}}\" and, for each of the first three apps in the results that are not ads, report the app's name, its average rating, how many reviews it has, and the monthly price in US dollars of its cheapest paid plan (0 if it has no paid plan). Do not install anything. Reply with JSON: {\"apps\": [{\"app\": string, \"rating\": number, \"review_count\": integer, \"cheapest_paid_monthly_price\": number}]}, in the order the results show them.", + "fixture": { + "query": "subscriptions" + }, + "start_url": "https://apps.shopify.com/", + "answer_schema": "schemas/app-listings.json", + "rules": [ + { "path": "apps", "count": [3, 3] }, + { "path": "apps[].app", "unique": true }, + { "path": "apps[].rating", "range": [0, 5] }, + { "path": "apps[].review_count", "range": [0, 10000000] }, + { "path": "apps[].cheapest_paid_monthly_price", "range": [0, 10000] } + ], + "agreement": { "path": "apps", "same": true }, + "pairs": 5, + "trial_timeout_sec": 600, + "max_turns": 40 +} diff --git a/bench/tasks/shopify.com/shopify-app-store-listing.json b/bench/tasks/shopify.com/shopify-app-store-listing.json new file mode 100644 index 00000000..8b0b0933 --- /dev/null +++ b/bench/tasks/shopify.com/shopify-app-store-listing.json @@ -0,0 +1,41 @@ +{ + "id": "shopify-app-store-listing", + "domain": "shopify.com", + "stage": "inspect", + "prompt": "On the Shopify App Store (apps.shopify.com), find the listing for {{app}} and report its average rating, how many reviews it has, and the monthly price in US dollars of its cheapest paid plan. Do not install the app. Reply with JSON: {\"app\": string, \"rating\": number, \"review_count\": integer, \"cheapest_paid_monthly_price\": number}.", + "fixture": { + "app": "Judge.me Product Reviews" + }, + "start_url": "https://apps.shopify.com/", + "answer_schema": "schemas/app-listing.json", + "rules": [ + { + "path": "app", + "matches": "judge" + }, + { + "path": "rating", + "range": [ + 1, + 5 + ] + }, + { + "path": "review_count", + "range": [ + 100, + 10000000 + ] + }, + { + "path": "cheapest_paid_monthly_price", + "range": [ + 1, + 1000 + ] + } + ], + "pairs": 5, + "trial_timeout_sec": 600, + "max_turns": 40 +} diff --git a/bench/tasks/steampowered.com/steam-compare-series.json b/bench/tasks/steampowered.com/steam-compare-series.json new file mode 100644 index 00000000..d37b7151 --- /dev/null +++ b/bench/tasks/steampowered.com/steam-compare-series.json @@ -0,0 +1,25 @@ +{ + "id": "steam-compare-series", + "domain": "steampowered.com", + "stage": "compare", + "prompt": "On store.steampowered.com, compare the {{series}} games {{game_v}}, {{game_vi}} and {{game_vii}}: for each, find its current price in the US store (the base game, not an edition or bundle) and the summary under English Reviews (not Recent Reviews) in its app page header, such as \"Very Positive\". Reply with JSON: {\"games\": [{\"game\": string, \"price\": number in US dollars, \"review_summary\": string}]}, one entry per game in the order given. Use exactly \"{{game_v}}\", \"{{game_vi}}\" and \"{{game_vii}}\" as the game values, not the full store titles or their trademark symbols.", + "fixture": { + "series": "Sid Meier's Civilization", + "game_v": "Civilization V", + "game_vi": "Civilization VI", + "game_vii": "Civilization VII" + }, + "start_url": "https://store.steampowered.com/", + "answer_schema": "schemas/game-prices.json", + "rules": [ + { "path": "games", "count": [3, 3] }, + { "path": "games[].game", "in": ["{{game_v}}", "{{game_vi}}", "{{game_vii}}"] }, + { "path": "games[].game", "unique": true }, + { "path": "games[].price", "range": [1, 100] }, + { "path": "games[].review_summary", "matches": "positive|mixed|negative" } + ], + "agreement": { "path": "games", "same": true }, + "pairs": 5, + "trial_timeout_sec": 600, + "max_turns": 40 +} diff --git a/bench/test/comparison-intents.test.mjs b/bench/test/comparison-intents.test.mjs new file mode 100644 index 00000000..5249a98d --- /dev/null +++ b/bench/test/comparison-intents.test.mjs @@ -0,0 +1,62 @@ +import assert from "node:assert/strict"; +import { dirname, join } from "node:path"; +import test from "node:test"; +import { fileURLToPath } from "node:url"; +import { gradeAnswer } from "../grade.mjs"; +import { loadIntents, loadSchema, trialPrompt } from "../lib/intents.mjs"; +import { answerCheck, pairStats } from "../report.mjs"; + +const benchDir = join(dirname(fileURLToPath(import.meta.url)), ".."); +const intents = loadIntents(benchDir); + +function checkAgreement(id, control, treatment, differed) { + const intent = intents.find(i => i.id === id); + const schema = loadSchema(benchDir, intent); + const rows = [control, treatment].map((answer, i) => { + const grade = gradeAnswer(intent, schema, JSON.stringify(answer)); + assert.equal(grade.outcome, "correct", grade.problems.join("; ")); + return { intent: id, pair: 1, arm: i ? "treatment" : "control", outcome: grade.outcome, duration_ms: 1000, grade }; + }); + assert.deepEqual(pairStats(rows, intent.agreement).agreement, { checked: 1, differed }); +} + +test("Shopify comparison agreement includes all listing fields and result order", () => { + const apps = ["First Subscriptions", "Second Subscriptions", "Third Subscriptions"].map(app => ({ + app, rating: 4.8, review_count: 123, cheapest_paid_monthly_price: 29 + })); + const check = (other, differed) => checkAgreement("shopify-app-store-compare", { apps }, { apps: other }, differed); + check(structuredClone(apps), 0); + for (const change of [{ rating: 1 }, { review_count: 0 }, { cheapest_paid_monthly_price: 0 }]) { + check([{ ...apps[0], ...change }, ...apps.slice(1)], 1); + } + check(apps.toReversed(), 1); + check([{ ...apps[0], app: "First℠ Subscriptions™" }, ...apps.slice(1)], 0); +}); + +test("Steam comparison agreement keeps prices and review summaries attached to their games in order", () => { + const games = ["Civilization V", "Civilization VI", "Civilization VII"].map((game, i) => ({ + game, price: (i + 1) * 10, review_summary: "Very Positive" + })); + const check = (other, differed) => checkAgreement("steam-compare-series", { games }, { games: other }, differed); + check(structuredClone(games), 0); + check(games.map((g, i) => ({ ...g, price: games[(i + 1) % games.length].price })), 1); + check([{ ...games[0], review_summary: "Mixed" }, ...games.slice(1)], 1); + check(games.toReversed(), 1); +}); + +test("Steam comparison requires canonical game labels instead of ambiguous title variants", () => { + const intent = intents.find(i => i.id === "steam-compare-series"); + const schema = loadSchema(benchDir, intent); + const games = ["Civilization V", "Civilization VI", "Civilization VII"].map(game => ({ + game, price: 29.99, review_summary: "Very Positive" + })); + const grade = value => gradeAnswer(intent, schema, JSON.stringify({ games: value })); + assert.equal(grade(games).outcome, "correct"); + for (const game of ["Sid Meier's Civilization® V", "Civilization IV", "Civilization VI"]) { + assert.equal(grade([{ ...games[0], game }, ...games.slice(1)]).outcome, "wrong", game); + } + const prompt = trialPrompt(intent, null); + assert.match(prompt, /Use exactly "Civilization V", "Civilization VI" and "Civilization VII" as the game values/); + assert.match(prompt, /English Reviews \(not Recent Reviews\)/); + assert.equal(answerCheck(intent), "agreement", "fixed game labels do not pin the prices or reviews"); +}); diff --git a/bench/test/run.test.mjs b/bench/test/run.test.mjs index dc9e86bc..f0ef713b 100644 --- a/bench/test/run.test.mjs +++ b/bench/test/run.test.mjs @@ -231,6 +231,13 @@ test("a checkout trial whose guard count or end state cannot be read aborts", { } }); +test("an end state read that a navigation interrupts is read again from the new page", { timeout: 120_000 }, () => { + const { run, rows } = workspace({ domain: { actions: { add: "adds to the cart" } }, intent: checkoutIntent }); + const result = run("navigating", { STUB_PAGE_NAVIGATING_ARMS: "treatment" }); + assert.equal(result.status, 0, result.stderr + result.stdout); + assert.deepEqual(rows("navigating").map(r => [r.arm, r.outcome]).sort(), [["control", "correct"], ["treatment", "correct"]]); +}); + test("an opt_in intent runs only when named with --intent", { timeout: 120_000 }, () => { const { run } = workspace({ intent: { ...lookupIntent, opt_in: "not rehearsed" } }); const unnamed = run("opt-in"); diff --git a/bench/test/stubs/kernel-api.mjs b/bench/test/stubs/kernel-api.mjs index 268012b1..e8fba826 100644 --- a/bench/test/stubs/kernel-api.mjs +++ b/bench/test/stubs/kernel-api.mjs @@ -7,7 +7,8 @@ // STUB_LIVE_CUSTOM_ARMS a custom tool is live without being registered, and in those tagged with an // arm listed in STUB_OFF_PAGE_ARMS registered tools are listed without a page. Reading cookies fails // in browsers tagged with an arm listed in STUB_COOKIES_FAIL_ARMS, and reading the page in those -// listed in STUB_PAGE_FAIL_ARMS. +// listed in STUB_PAGE_FAIL_ARMS; in those listed in STUB_PAGE_NAVIGATING_ARMS the first read fails as +// if the page were navigating. import { readFileSync, writeFileSync } from "node:fs"; import { createServer } from "node:http"; @@ -58,7 +59,12 @@ async function handle(request, url, body) { if (route.endsWith("/playwright/execute")) { const { code } = JSON.parse(body); if (code.includes("context.cookies") && listed("STUB_COOKIES_FAIL_ARMS", arm)) return [200, { success: false, error: "Target page, context or browser has been closed" }]; - if (code.includes("fields") && listed("STUB_PAGE_FAIL_ARMS", arm)) return [200, { success: false, error: "Execution context was destroyed" }]; + if (code.includes("fields") && listed("STUB_PAGE_FAIL_ARMS", arm)) return [200, { success: false, error: "Target page, context or browser has been closed" }]; + if (code.includes("fields") && listed("STUB_PAGE_NAVIGATING_ARMS", arm) && !browser.pageRead) { + browser.pageRead = true; + save(); + return [200, { success: false, error: "Execution context was destroyed, most likely because of a navigation" }]; + } if (code.includes("context.cookies")) { // The guard's cookie sits on the checkout host, not on the page the agent ended on. const blocked = listed("STUB_SUBMIT_ARMS", arm) ? [{ name: "bench_submit_blocked", value: "1", domain: "checkout.example.com" }] : []; diff --git a/bench/test/tasks.test.mjs b/bench/test/tasks.test.mjs index 9efb419e..f3d1ba17 100644 --- a/bench/test/tasks.test.mjs +++ b/bench/test/tasks.test.mjs @@ -4,6 +4,7 @@ import { tmpdir } from "node:os"; import { dirname, join } from "node:path"; import test from "node:test"; import { fileURLToPath } from "node:url"; +import { gradeAnswer } from "../grade.mjs"; import { ALLOW, fill, loadIntents, loadSchema, pastDates, resolveFixture, trialPrompt, validateDomain, validateIntent } from "../lib/intents.mjs"; import { checkSchema } from "../lib/schema.mjs"; import { composeBatch, evaluateDefinitions, matchesUrlPattern, normalize, registeredTools, resolveSource } from "../lib/tools.mjs"; @@ -356,3 +357,13 @@ test("the guard extension runs the guard on its hosts and is named by its conten assert.notEqual(writeGuardExtension(join(dir, "c"), { ...guard, buttons: "^buy$" }).name, first.name); assert.notEqual(writeGuardExtension(join(dir, "d"), { ...guard, selectors: "#buy" }).name, first.name); }); + +test("the DoorDash address check reads the header's address line, not a store or place named after the address", () => { + const intent = loadIntents(benchDir).find(i => i.id === "doordash-pizza-near-address"); + const answer = JSON.stringify({ places: [1, 2, 3].map(n => ({ name: `Pizza ${n}`, detail: "$0 delivery fee" })) }); + const grade = text => gradeAnswer(intent, loadSchema(benchDir, intent), answer, { output: { url: "https://www.doordash.com/", title: "DoorDash", text, fields: {}, missing: [], invalid: [] } }); + for (const text of ["DoorDash\n4 Times Sq\nSign In", "DoorDash\nTimes Square\nDelivery"]) assert.equal(grade(text).outcome, "correct", text); + for (const text of ["DoorDash\nYour Address\nJohn's of Times Square", "DoorDash\nJohn's of Times Square", "4 Times Sq, New York, NY 10036", "DoorDash\nTimes Square Kitchen\nDelivery", "DoorDash\nTimes Squares Square Dance Club\nDelivery", "DoorDash\nCrowne Plaza Manhattan Times Square\nDelivery"]) { + assert.equal(grade(text).outcome, "wrong", text); + } +});