From b1a88b466f69b8cd40a242649e7eeb6806f2f151 Mon Sep 17 00:00:00 2001
From: yummybomb <19238148+yummybomb@users.noreply.github.com>
Date: Tue, 20 Oct 2026 22:44:14 +0000
Subject: [PATCH 1/8] bench: grade the DoorDash address the task sets, and add
 a Shopify App Store intent

---
 bench/schemas/app-listing.json                | 24 +++++++++++
 .../doordash-pizza-near-address.json          | 12 +++++-
 .../shopify-app-store-listing.json            | 41 +++++++++++++++++++
 3 files changed, 75 insertions(+), 2 deletions(-)
 create mode 100644 bench/schemas/app-listing.json
 create mode 100644 bench/tasks/shopify.com/shopify-app-store-listing.json

diff --git a/bench/schemas/app-listing.json b/bench/schemas/app-listing.json
new file mode 100644
index 00000000..b5c49803
--- /dev/null
+++ b/bench/schemas/app-listing.json
@@ -0,0 +1,24 @@
+{
+  "type": "object",
+  "required": [
+    "app",
+    "rating",
+    "review_count",
+    "cheapest_paid_monthly_price"
+  ],
+  "properties": {
+    "app": {
+      "type": "string",
+      "minLength": 1
+    },
+    "rating": {
+      "type": "number"
+    },
+    "review_count": {
+      "type": "integer"
+    },
+    "cheapest_paid_monthly_price": {
+      "type": "number"
+    }
+  }
+}
diff --git a/bench/tasks/doordash.com/doordash-pizza-near-address.json b/bench/tasks/doordash.com/doordash-pizza-near-address.json
index 2ebccd6e..16db9b3a 100644
--- a/bench/tasks/doordash.com/doordash-pizza-near-address.json
+++ b/bench/tasks/doordash.com/doordash-pizza-near-address.json
@@ -1,8 +1,8 @@
 {
   "id": "doordash-pizza-near-address",
   "domain": "doordash.com",
-  "stage": "find",
-  "prompt": "On doordash.com, with the delivery address set to {{address}}, find three pizza restaurants that deliver there and the delivery fee the site shows for each. Reply with JSON: {\"places\": [{\"name\": string, \"detail\": the delivery fee as shown}]}.",
+  "stage": "prepare",
+  "prompt": "On doordash.com, set the delivery address to {{address}} with the site's address picker, then find three pizza restaurants that deliver there and the delivery fee the site shows for each. Leave the address set when you finish. Reply with JSON: {\"places\": [{\"name\": string, \"detail\": the delivery fee as shown}]}.",
   "fixture": {
     "address": "Times Square, New York, NY 10036"
   },
@@ -25,6 +25,14 @@
       "matches": "\\$\\s*\\d|free"
     }
   ],
+  "check": {
+    "rules": [
+      {
+        "path": "text",
+        "matches": "Times Sq"
+      }
+    ]
+  },
   "pairs": 5,
   "trial_timeout_sec": 600,
   "max_turns": 40
diff --git a/bench/tasks/shopify.com/shopify-app-store-listing.json b/bench/tasks/shopify.com/shopify-app-store-listing.json
new file mode 100644
index 00000000..8b0b0933
--- /dev/null
+++ b/bench/tasks/shopify.com/shopify-app-store-listing.json
@@ -0,0 +1,41 @@
+{
+  "id": "shopify-app-store-listing",
+  "domain": "shopify.com",
+  "stage": "inspect",
+  "prompt": "On the Shopify App Store (apps.shopify.com), find the listing for {{app}} and report its average rating, how many reviews it has, and the monthly price in US dollars of its cheapest paid plan. Do not install the app. Reply with JSON: {\"app\": string, \"rating\": number, \"review_count\": integer, \"cheapest_paid_monthly_price\": number}.",
+  "fixture": {
+    "app": "Judge.me Product Reviews"
+  },
+  "start_url": "https://apps.shopify.com/",
+  "answer_schema": "schemas/app-listing.json",
+  "rules": [
+    {
+      "path": "app",
+      "matches": "judge"
+    },
+    {
+      "path": "rating",
+      "range": [
+        1,
+        5
+      ]
+    },
+    {
+      "path": "review_count",
+      "range": [
+        100,
+        10000000
+      ]
+    },
+    {
+      "path": "cheapest_paid_monthly_price",
+      "range": [
+        1,
+        1000
+      ]
+    }
+  ],
+  "pairs": 5,
+  "trial_timeout_sec": 600,
+  "max_turns": 40
+}

From 5b2856ef5681b422c5faf3ac44b72547ec10d2f8 Mon Sep 17 00:00:00 2001
From: yummybomb <19238148+yummybomb@users.noreply.github.com>
Date: Wed, 21 Oct 2026 15:14:23 +0000
Subject: [PATCH 2/8] bench: read the end state again when a navigation
 interrupts the read

An agent can finish while its last action is still loading a page, and
the end-state read then fails with 'Execution context was destroyed',
which aborted the trial and reran its pair. The read is now tried again
on the new page, up to three times, in both arms.
---
 bench/run.mjs                   | 19 +++++++++++++------
 bench/test/run.test.mjs         |  7 +++++++
 bench/test/stubs/kernel-api.mjs |  8 +++++++-
 3 files changed, 27 insertions(+), 7 deletions(-)

diff --git a/bench/run.mjs b/bench/run.mjs
index 0ba28e2a..89c3f521 100644
--- a/bench/run.mjs
+++ b/bench/run.mjs
@@ -540,13 +540,20 @@ async function setUpBrowser(plan, intentPlan, arm, label) {
 }
 
 // Reads where a prepare intent's agent left the browser from the page itself, the same way in both
-// arms, so the grade does not depend on the tools under test. A page that cannot be read aborts the
-// trial rather than grading it wrong.
+// arms, so the grade does not depend on the tools under test. An agent can finish while its last
+// action is still loading a page, so a read the navigation interrupts is tried again on the new page.
+// A page that cannot be read aborts the trial rather than grading it wrong.
 async function readEndState(browser, intent) {
-  try {
-    return { output: await browsers.evaluate(browser, readPage, readPageArg(intent.check.fields)) };
-  } catch (error) {
-    throw new Error(`could not read the end state: ${error.message}`);
+  for (let attempt = 1; ; attempt++) {
+    try {
+      return { output: await browsers.evaluate(browser, readPage, readPageArg(intent.check.fields)) };
+    } catch (error) {
+      if (attempt < 3 && /Execution context was destroyed/.test(error.message)) {
+        await new Promise(resolve => setTimeout(resolve, 3000));
+        continue;
+      }
+      throw new Error(`could not read the end state: ${error.message}`);
+    }
   }
 }
 
diff --git a/bench/test/run.test.mjs b/bench/test/run.test.mjs
index dc9e86bc..f0ef713b 100644
--- a/bench/test/run.test.mjs
+++ b/bench/test/run.test.mjs
@@ -231,6 +231,13 @@ test("a checkout trial whose guard count or end state cannot be read aborts", {
   }
 });
 
+test("an end state read that a navigation interrupts is read again from the new page", { timeout: 120_000 }, () => {
+  const { run, rows } = workspace({ domain: { actions: { add: "adds to the cart" } }, intent: checkoutIntent });
+  const result = run("navigating", { STUB_PAGE_NAVIGATING_ARMS: "treatment" });
+  assert.equal(result.status, 0, result.stderr + result.stdout);
+  assert.deepEqual(rows("navigating").map(r => [r.arm, r.outcome]).sort(), [["control", "correct"], ["treatment", "correct"]]);
+});
+
 test("an opt_in intent runs only when named with --intent", { timeout: 120_000 }, () => {
   const { run } = workspace({ intent: { ...lookupIntent, opt_in: "not rehearsed" } });
   const unnamed = run("opt-in");
diff --git a/bench/test/stubs/kernel-api.mjs b/bench/test/stubs/kernel-api.mjs
index 268012b1..0bc11d8f 100644
--- a/bench/test/stubs/kernel-api.mjs
+++ b/bench/test/stubs/kernel-api.mjs
@@ -7,7 +7,8 @@
 // STUB_LIVE_CUSTOM_ARMS a custom tool is live without being registered, and in those tagged with an
 // arm listed in STUB_OFF_PAGE_ARMS registered tools are listed without a page. Reading cookies fails
 // in browsers tagged with an arm listed in STUB_COOKIES_FAIL_ARMS, and reading the page in those
-// listed in STUB_PAGE_FAIL_ARMS.
+// listed in STUB_PAGE_FAIL_ARMS; in those listed in STUB_PAGE_NAVIGATING_ARMS the first read fails as
+// if the page were navigating.
 import { readFileSync, writeFileSync } from "node:fs";
 import { createServer } from "node:http";
 
@@ -59,6 +60,11 @@ async function handle(request, url, body) {
     const { code } = JSON.parse(body);
     if (code.includes("context.cookies") && listed("STUB_COOKIES_FAIL_ARMS", arm)) return [200, { success: false, error: "Target page, context or browser has been closed" }];
     if (code.includes("fields") && listed("STUB_PAGE_FAIL_ARMS", arm)) return [200, { success: false, error: "Execution context was destroyed" }];
+    if (code.includes("fields") && listed("STUB_PAGE_NAVIGATING_ARMS", arm) && !browser.pageRead) {
+      browser.pageRead = true;
+      save();
+      return [200, { success: false, error: "Execution context was destroyed, most likely because of a navigation" }];
+    }
     if (code.includes("context.cookies")) {
       // The guard's cookie sits on the checkout host, not on the page the agent ended on.
       const blocked = listed("STUB_SUBMIT_ARMS", arm) ? [{ name: "bench_submit_blocked", value: "1", domain: "checkout.example.com" }] : [];

From b89602102c2291e4f0a01ab56130142614a215cc Mon Sep 17 00:00:00 2001
From: yummybomb <19238148+yummybomb@users.noreply.github.com>
Date: Wed, 21 Oct 2026 19:17:06 +0000
Subject: [PATCH 3/8] bench: add Steam and Shopify App Store comparison intents
 that need search

The existing Steam and App Store intents name one product the model can
open from memory, so they measure whether it knows the URL rather than
what the site's tools add. The new intents compare three products: three
Civilization games' prices and review summaries, and the first three
non-ad App Store results for a query with their ratings, review counts
and cheapest paid plans.
---
 bench/schemas/app-listings.json               | 19 +++++++++++++++
 bench/schemas/game-prices.json                | 18 +++++++++++++++
 .../shopify-app-store-compare.json            | 22 ++++++++++++++++++
 .../steam-compare-series.json                 | 23 +++++++++++++++++++
 4 files changed, 82 insertions(+)
 create mode 100644 bench/schemas/app-listings.json
 create mode 100644 bench/schemas/game-prices.json
 create mode 100644 bench/tasks/shopify.com/shopify-app-store-compare.json
 create mode 100644 bench/tasks/steampowered.com/steam-compare-series.json

diff --git a/bench/schemas/app-listings.json b/bench/schemas/app-listings.json
new file mode 100644
index 00000000..5e1fcea9
--- /dev/null
+++ b/bench/schemas/app-listings.json
@@ -0,0 +1,19 @@
+{
+  "type": "object",
+  "required": ["apps"],
+  "properties": {
+    "apps": {
+      "type": "array",
+      "items": {
+        "type": "object",
+        "required": ["app", "rating", "review_count", "cheapest_paid_monthly_price"],
+        "properties": {
+          "app": { "type": "string", "minLength": 1 },
+          "rating": { "type": "number" },
+          "review_count": { "type": "integer" },
+          "cheapest_paid_monthly_price": { "type": "number" }
+        }
+      }
+    }
+  }
+}
diff --git a/bench/schemas/game-prices.json b/bench/schemas/game-prices.json
new file mode 100644
index 00000000..c3557faa
--- /dev/null
+++ b/bench/schemas/game-prices.json
@@ -0,0 +1,18 @@
+{
+  "type": "object",
+  "required": ["games"],
+  "properties": {
+    "games": {
+      "type": "array",
+      "items": {
+        "type": "object",
+        "required": ["game", "price", "review_summary"],
+        "properties": {
+          "game": { "type": "string", "minLength": 1 },
+          "price": { "type": "number" },
+          "review_summary": { "type": "string", "minLength": 1 }
+        }
+      }
+    }
+  }
+}
diff --git a/bench/tasks/shopify.com/shopify-app-store-compare.json b/bench/tasks/shopify.com/shopify-app-store-compare.json
new file mode 100644
index 00000000..19c552b2
--- /dev/null
+++ b/bench/tasks/shopify.com/shopify-app-store-compare.json
@@ -0,0 +1,22 @@
+{
+  "id": "shopify-app-store-compare",
+  "domain": "shopify.com",
+  "stage": "compare",
+  "prompt": "On the Shopify App Store (apps.shopify.com), search for \"{{query}}\" and, for each of the first three apps in the results that are not ads, report the app's name, its average rating, how many reviews it has, and the monthly price in US dollars of its cheapest paid plan (0 if it has no paid plan). Do not install anything. Reply with JSON: {\"apps\": [{\"app\": string, \"rating\": number, \"review_count\": integer, \"cheapest_paid_monthly_price\": number}]}, in the order the results show them.",
+  "fixture": {
+    "query": "subscriptions"
+  },
+  "start_url": "https://apps.shopify.com/",
+  "answer_schema": "schemas/app-listings.json",
+  "rules": [
+    { "path": "apps", "count": [3, 3] },
+    { "path": "apps[].app", "unique": true },
+    { "path": "apps[].rating", "range": [0, 5] },
+    { "path": "apps[].review_count", "range": [0, 10000000] },
+    { "path": "apps[].cheapest_paid_monthly_price", "range": [0, 10000] }
+  ],
+  "agreement": { "path": "apps[].app", "same": true },
+  "pairs": 5,
+  "trial_timeout_sec": 600,
+  "max_turns": 40
+}
diff --git a/bench/tasks/steampowered.com/steam-compare-series.json b/bench/tasks/steampowered.com/steam-compare-series.json
new file mode 100644
index 00000000..246ef7d4
--- /dev/null
+++ b/bench/tasks/steampowered.com/steam-compare-series.json
@@ -0,0 +1,23 @@
+{
+  "id": "steam-compare-series",
+  "domain": "steampowered.com",
+  "stage": "compare",
+  "prompt": "On store.steampowered.com, compare the {{series}} games {{games}}: for each, find its current price in the US store (the base game, not an edition or bundle) and the user review summary Steam shows for it, such as \"Very Positive\". Reply with JSON: {\"games\": [{\"game\": string, \"price\": number in US dollars, \"review_summary\": string}]}, one entry per game in the order given.",
+  "fixture": {
+    "series": "Sid Meier's Civilization",
+    "games": "Civilization V, Civilization VI and Civilization VII"
+  },
+  "start_url": "https://store.steampowered.com/",
+  "answer_schema": "schemas/game-prices.json",
+  "rules": [
+    { "path": "games", "count": [3, 3] },
+    { "path": "games[].game", "matches": "civilization" },
+    { "path": "games[].game", "unique": true },
+    { "path": "games[].price", "range": [1, 100] },
+    { "path": "games[].review_summary", "matches": "positive|mixed|negative" }
+  ],
+  "agreement": { "path": "games[].price", "same": true },
+  "pairs": 5,
+  "trial_timeout_sec": 600,
+  "max_turns": 40
+}

From 59358ff0e02dbc0b6e93dcbcb6019106b87c5acd Mon Sep 17 00:00:00 2001
From: yummybomb <19238148+yummybomb@users.noreply.github.com>
Date: Wed, 21 Oct 2026 20:01:24 +0000
Subject: [PATCH 4/8] bench: grade the DoorDash address from the header, and
 abort an unreadable end state without retrying

---
 bench/tasks/doordash.com/doordash-pizza-near-address.json | 2 +-
 bench/test/stubs/kernel-api.mjs                           | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/bench/tasks/doordash.com/doordash-pizza-near-address.json b/bench/tasks/doordash.com/doordash-pizza-near-address.json
index 16db9b3a..65f41f06 100644
--- a/bench/tasks/doordash.com/doordash-pizza-near-address.json
+++ b/bench/tasks/doordash.com/doordash-pizza-near-address.json
@@ -29,7 +29,7 @@
     "rules": [
       {
         "path": "text",
-        "matches": "Times Sq"
+        "matches": "DoorDash[ \\t]*\\n[^\\n]*Times Sq"
       }
     ]
   },
diff --git a/bench/test/stubs/kernel-api.mjs b/bench/test/stubs/kernel-api.mjs
index 0bc11d8f..e8fba826 100644
--- a/bench/test/stubs/kernel-api.mjs
+++ b/bench/test/stubs/kernel-api.mjs
@@ -59,7 +59,7 @@ async function handle(request, url, body) {
   if (route.endsWith("/playwright/execute")) {
     const { code } = JSON.parse(body);
     if (code.includes("context.cookies") && listed("STUB_COOKIES_FAIL_ARMS", arm)) return [200, { success: false, error: "Target page, context or browser has been closed" }];
-    if (code.includes("fields") && listed("STUB_PAGE_FAIL_ARMS", arm)) return [200, { success: false, error: "Execution context was destroyed" }];
+    if (code.includes("fields") && listed("STUB_PAGE_FAIL_ARMS", arm)) return [200, { success: false, error: "Target page, context or browser has been closed" }];
     if (code.includes("fields") && listed("STUB_PAGE_NAVIGATING_ARMS", arm) && !browser.pageRead) {
       browser.pageRead = true;
       save();

From 575cf0f91d894cf0a97cf9e874a648d7a01052ee Mon Sep 17 00:00:00 2001
From: yummybomb <19238148+yummybomb@users.noreply.github.com>
Date: Wed, 21 Oct 2026 21:14:48 +0000
Subject: [PATCH 5/8] Compare complete ordered benchmark answers

---
 .../shopify-app-store-compare.json            |  2 +-
 .../steam-compare-series.json                 |  2 +-
 bench/test/comparison-intents.test.mjs        | 44 +++++++++++++++++++
 3 files changed, 46 insertions(+), 2 deletions(-)
 create mode 100644 bench/test/comparison-intents.test.mjs

diff --git a/bench/tasks/shopify.com/shopify-app-store-compare.json b/bench/tasks/shopify.com/shopify-app-store-compare.json
index 19c552b2..91aa933b 100644
--- a/bench/tasks/shopify.com/shopify-app-store-compare.json
+++ b/bench/tasks/shopify.com/shopify-app-store-compare.json
@@ -15,7 +15,7 @@
     { "path": "apps[].review_count", "range": [0, 10000000] },
     { "path": "apps[].cheapest_paid_monthly_price", "range": [0, 10000] }
   ],
-  "agreement": { "path": "apps[].app", "same": true },
+  "agreement": { "path": "apps", "same": true },
   "pairs": 5,
   "trial_timeout_sec": 600,
   "max_turns": 40
diff --git a/bench/tasks/steampowered.com/steam-compare-series.json b/bench/tasks/steampowered.com/steam-compare-series.json
index 246ef7d4..2e5ab086 100644
--- a/bench/tasks/steampowered.com/steam-compare-series.json
+++ b/bench/tasks/steampowered.com/steam-compare-series.json
@@ -16,7 +16,7 @@
     { "path": "games[].price", "range": [1, 100] },
     { "path": "games[].review_summary", "matches": "positive|mixed|negative" }
   ],
-  "agreement": { "path": "games[].price", "same": true },
+  "agreement": { "path": "games", "same": true },
   "pairs": 5,
   "trial_timeout_sec": 600,
   "max_turns": 40
diff --git a/bench/test/comparison-intents.test.mjs b/bench/test/comparison-intents.test.mjs
new file mode 100644
index 00000000..eea2fe0e
--- /dev/null
+++ b/bench/test/comparison-intents.test.mjs
@@ -0,0 +1,44 @@
+import assert from "node:assert/strict";
+import { dirname, join } from "node:path";
+import test from "node:test";
+import { fileURLToPath } from "node:url";
+import { gradeAnswer } from "../grade.mjs";
+import { loadIntents, loadSchema } from "../lib/intents.mjs";
+import { pairStats } from "../report.mjs";
+
+const benchDir = join(dirname(fileURLToPath(import.meta.url)), "..");
+const intents = loadIntents(benchDir);
+
+function checkAgreement(id, control, treatment, differed) {
+  const intent = intents.find(i => i.id === id);
+  const schema = loadSchema(benchDir, intent);
+  const rows = [control, treatment].map((answer, i) => {
+    const grade = gradeAnswer(intent, schema, JSON.stringify(answer));
+    assert.equal(grade.outcome, "correct", grade.problems.join("; "));
+    return { intent: id, pair: 1, arm: i ? "treatment" : "control", outcome: grade.outcome, duration_ms: 1000, grade };
+  });
+  assert.deepEqual(pairStats(rows, intent.agreement).agreement, { checked: 1, differed });
+}
+
+test("Shopify comparison agreement includes all listing fields and result order", () => {
+  const apps = ["First Subscriptions", "Second Subscriptions", "Third Subscriptions"].map(app => ({
+    app, rating: 4.8, review_count: 123, cheapest_paid_monthly_price: 29
+  }));
+  const check = (other, differed) => checkAgreement("shopify-app-store-compare", { apps }, { apps: other }, differed);
+  check(structuredClone(apps), 0);
+  for (const change of [{ rating: 1 }, { review_count: 0 }, { cheapest_paid_monthly_price: 0 }]) {
+    check([{ ...apps[0], ...change }, ...apps.slice(1)], 1);
+  }
+  check(apps.toReversed(), 1);
+});
+
+test("Steam comparison agreement keeps prices and review summaries attached to their games in order", () => {
+  const games = ["Civilization V", "Civilization VI", "Civilization VII"].map((game, i) => ({
+    game, price: (i + 1) * 10, review_summary: "Very Positive"
+  }));
+  const check = (other, differed) => checkAgreement("steam-compare-series", { games }, { games: other }, differed);
+  check(structuredClone(games), 0);
+  check(games.map((g, i) => ({ ...g, price: games[(i + 1) % games.length].price })), 1);
+  check([{ ...games[0], review_summary: "Mixed" }, ...games.slice(1)], 1);
+  check(games.toReversed(), 1);
+});

From af591836e1d16c1ee7c2ed948cee892ed7243b75 Mon Sep 17 00:00:00 2001
From: yummybomb <19238148+yummybomb@users.noreply.github.com>
Date: Wed, 21 Oct 2026 21:35:29 +0000
Subject: [PATCH 6/8] Disambiguate Steam comparison answers

---
 .../steam-compare-series.json                 |  8 ++++---
 bench/test/comparison-intents.test.mjs        | 21 +++++++++++++++++--
 2 files changed, 24 insertions(+), 5 deletions(-)

diff --git a/bench/tasks/steampowered.com/steam-compare-series.json b/bench/tasks/steampowered.com/steam-compare-series.json
index 2e5ab086..d37b7151 100644
--- a/bench/tasks/steampowered.com/steam-compare-series.json
+++ b/bench/tasks/steampowered.com/steam-compare-series.json
@@ -2,16 +2,18 @@
   "id": "steam-compare-series",
   "domain": "steampowered.com",
   "stage": "compare",
-  "prompt": "On store.steampowered.com, compare the {{series}} games {{games}}: for each, find its current price in the US store (the base game, not an edition or bundle) and the user review summary Steam shows for it, such as \"Very Positive\". Reply with JSON: {\"games\": [{\"game\": string, \"price\": number in US dollars, \"review_summary\": string}]}, one entry per game in the order given.",
+  "prompt": "On store.steampowered.com, compare the {{series}} games {{game_v}}, {{game_vi}} and {{game_vii}}: for each, find its current price in the US store (the base game, not an edition or bundle) and the summary under English Reviews (not Recent Reviews) in its app page header, such as \"Very Positive\". Reply with JSON: {\"games\": [{\"game\": string, \"price\": number in US dollars, \"review_summary\": string}]}, one entry per game in the order given. Use exactly \"{{game_v}}\", \"{{game_vi}}\" and \"{{game_vii}}\" as the game values, not the full store titles or their trademark symbols.",
   "fixture": {
     "series": "Sid Meier's Civilization",
-    "games": "Civilization V, Civilization VI and Civilization VII"
+    "game_v": "Civilization V",
+    "game_vi": "Civilization VI",
+    "game_vii": "Civilization VII"
   },
   "start_url": "https://store.steampowered.com/",
   "answer_schema": "schemas/game-prices.json",
   "rules": [
     { "path": "games", "count": [3, 3] },
-    { "path": "games[].game", "matches": "civilization" },
+    { "path": "games[].game", "in": ["{{game_v}}", "{{game_vi}}", "{{game_vii}}"] },
     { "path": "games[].game", "unique": true },
     { "path": "games[].price", "range": [1, 100] },
     { "path": "games[].review_summary", "matches": "positive|mixed|negative" }
diff --git a/bench/test/comparison-intents.test.mjs b/bench/test/comparison-intents.test.mjs
index eea2fe0e..0173967f 100644
--- a/bench/test/comparison-intents.test.mjs
+++ b/bench/test/comparison-intents.test.mjs
@@ -3,8 +3,8 @@ import { dirname, join } from "node:path";
 import test from "node:test";
 import { fileURLToPath } from "node:url";
 import { gradeAnswer } from "../grade.mjs";
-import { loadIntents, loadSchema } from "../lib/intents.mjs";
-import { pairStats } from "../report.mjs";
+import { loadIntents, loadSchema, trialPrompt } from "../lib/intents.mjs";
+import { answerCheck, pairStats } from "../report.mjs";
 
 const benchDir = join(dirname(fileURLToPath(import.meta.url)), "..");
 const intents = loadIntents(benchDir);
@@ -42,3 +42,20 @@ test("Steam comparison agreement keeps prices and review summaries attached to t
   check([{ ...games[0], review_summary: "Mixed" }, ...games.slice(1)], 1);
   check(games.toReversed(), 1);
 });
+
+test("Steam comparison requires canonical game labels instead of ambiguous title variants", () => {
+  const intent = intents.find(i => i.id === "steam-compare-series");
+  const schema = loadSchema(benchDir, intent);
+  const games = ["Civilization V", "Civilization VI", "Civilization VII"].map(game => ({
+    game, price: 29.99, review_summary: "Very Positive"
+  }));
+  const grade = value => gradeAnswer(intent, schema, JSON.stringify({ games: value }));
+  assert.equal(grade(games).outcome, "correct");
+  for (const game of ["Sid Meier's Civilization® V", "Civilization IV", "Civilization VI"]) {
+    assert.equal(grade([{ ...games[0], game }, ...games.slice(1)]).outcome, "wrong", game);
+  }
+  const prompt = trialPrompt(intent, null);
+  assert.match(prompt, /Use exactly "Civilization V", "Civilization VI" and "Civilization VII" as the game values/);
+  assert.match(prompt, /English Reviews \(not Recent Reviews\)/);
+  assert.equal(answerCheck(intent), "agreement", "fixed game labels do not pin the prices or reviews");
+});

From 8cae9a8ffaf7c12a6a1ac64ce9cf3aff52045767 Mon Sep 17 00:00:00 2001
From: yummybomb <19238148+yummybomb@users.noreply.github.com>
Date: Wed, 21 Oct 2026 21:49:57 +0000
Subject: [PATCH 7/8] Reject Times Square store names in the DoorDash address
 check

---
 bench/README.md                                       |  5 +++--
 .../doordash.com/doordash-pizza-near-address.json     |  2 +-
 bench/test/tasks.test.mjs                             | 11 +++++++++++
 3 files changed, 15 insertions(+), 3 deletions(-)

diff --git a/bench/README.md b/bench/README.md
index 4c739e4b..697584a8 100644
--- a/bench/README.md
+++ b/bench/README.md
@@ -228,8 +228,9 @@ and deletes every browser.
 - `agreement` (optional) compares the arms' answers within a pair. `{"path": "price", "better":
   "lower", "tolerance": 1}` counts pairs where one arm missed the cheapest fare. `{"path":
   "issues[]", "same": true}` is for one right answer that changes too often to pin in rules, and
-  counts both-correct pairs whose answers differ as sets. Leave it off intents with many right
-  answers.
+  counts both-correct pairs whose answers differ as sets. A path without `[]` compares the whole
+  value, so `{"path": "games", "same": true}` also counts a reordered array as differing. Leave it
+  off intents with many right answers.
 - `opt_in` (optional) is a reason the intent runs only when named with `--intent`, such as a
   checkout intent not yet rehearsed on the live site.
 
diff --git a/bench/tasks/doordash.com/doordash-pizza-near-address.json b/bench/tasks/doordash.com/doordash-pizza-near-address.json
index 65f41f06..23d6bb16 100644
--- a/bench/tasks/doordash.com/doordash-pizza-near-address.json
+++ b/bench/tasks/doordash.com/doordash-pizza-near-address.json
@@ -29,7 +29,7 @@
     "rules": [
       {
         "path": "text",
-        "matches": "DoorDash[ \\t]*\\n[^\\n]*Times Sq"
+        "matches": "DoorDash[ \\t]*\\n[^\\n]*Times Sq\\b"
       }
     ]
   },
diff --git a/bench/test/tasks.test.mjs b/bench/test/tasks.test.mjs
index 9efb419e..47a8a44b 100644
--- a/bench/test/tasks.test.mjs
+++ b/bench/test/tasks.test.mjs
@@ -4,6 +4,7 @@ import { tmpdir } from "node:os";
 import { dirname, join } from "node:path";
 import test from "node:test";
 import { fileURLToPath } from "node:url";
+import { gradeAnswer } from "../grade.mjs";
 import { ALLOW, fill, loadIntents, loadSchema, pastDates, resolveFixture, trialPrompt, validateDomain, validateIntent } from "../lib/intents.mjs";
 import { checkSchema } from "../lib/schema.mjs";
 import { composeBatch, evaluateDefinitions, matchesUrlPattern, normalize, registeredTools, resolveSource } from "../lib/tools.mjs";
@@ -356,3 +357,13 @@ test("the guard extension runs the guard on its hosts and is named by its conten
   assert.notEqual(writeGuardExtension(join(dir, "c"), { ...guard, buttons: "^buy$" }).name, first.name);
   assert.notEqual(writeGuardExtension(join(dir, "d"), { ...guard, selectors: "#buy" }).name, first.name);
 });
+
+test("the DoorDash address check reads the header's address line, not a store named after the address", () => {
+  const intent = loadIntents(benchDir).find(i => i.id === "doordash-pizza-near-address");
+  const answer = JSON.stringify({ places: [1, 2, 3].map(n => ({ name: `Pizza ${n}`, detail: "$0 delivery fee" })) });
+  const grade = text => gradeAnswer(intent, loadSchema(benchDir, intent), answer, { output: { url: "https://www.doordash.com/", title: "DoorDash", text, fields: {}, missing: [], invalid: [] } });
+  assert.equal(grade("DoorDash\n4 Times Sq\nSign In").outcome, "correct");
+  for (const text of ["DoorDash\nYour Address\nJohn's of Times Square", "DoorDash\nJohn's of Times Square", "4 Times Sq, New York, NY 10036"]) {
+    assert.equal(grade(text).outcome, "wrong", text);
+  }
+});

From adf8c0f31fb734afa644dd021c7635d76ace7c1d Mon Sep 17 00:00:00 2001
From: yummybomb <19238148+yummybomb@users.noreply.github.com>
Date: Thu, 22 Oct 2026 15:25:29 +0000
Subject: [PATCH 8/8] bench: accept a Times Square place in the DoorDash
 header, and compare names without trademark symbols

---
 bench/report.mjs                                          | 5 +++--
 bench/tasks/doordash.com/doordash-pizza-near-address.json | 2 +-
 bench/test/comparison-intents.test.mjs                    | 1 +
 bench/test/tasks.test.mjs                                 | 6 +++---
 4 files changed, 8 insertions(+), 6 deletions(-)

diff --git a/bench/report.mjs b/bench/report.mjs
index 86295291..02311259 100644
--- a/bench/report.mjs
+++ b/bench/report.mjs
@@ -53,9 +53,10 @@ export function armStats(rows) {
   };
 }
 
-// Strings compare case- and whitespace-insensitively, and objects regardless of key order.
+// Strings compare case- and whitespace-insensitively and without the trademark symbols agents copy
+// inconsistently from names, and objects regardless of key order.
 const canonical = (value) => JSON.stringify(value, (key, v) => {
-  if (typeof v === "string") return v.trim().toLowerCase().replace(/\s+/g, " ");
+  if (typeof v === "string") return v.replace(/[™®℠©]/g, "").trim().toLowerCase().replace(/\s+/g, " ");
   if (v && typeof v === "object" && !Array.isArray(v)) return Object.fromEntries(Object.entries(v).sort(([a], [b]) => a.localeCompare(b)));
   return v;
 });
diff --git a/bench/tasks/doordash.com/doordash-pizza-near-address.json b/bench/tasks/doordash.com/doordash-pizza-near-address.json
index 23d6bb16..ef28d5a4 100644
--- a/bench/tasks/doordash.com/doordash-pizza-near-address.json
+++ b/bench/tasks/doordash.com/doordash-pizza-near-address.json
@@ -29,7 +29,7 @@
     "rules": [
       {
         "path": "text",
-        "matches": "DoorDash[ \\t]*\\n[^\\n]*Times Sq\\b"
+        "matches": "DoorDash[ \\t]*\\n[ \\t]*(?:\\d+[ \\t]+)?Times Sq(?:uare)?[ \\t]*(?:\\n|$)"
       }
     ]
   },
diff --git a/bench/test/comparison-intents.test.mjs b/bench/test/comparison-intents.test.mjs
index 0173967f..5249a98d 100644
--- a/bench/test/comparison-intents.test.mjs
+++ b/bench/test/comparison-intents.test.mjs
@@ -30,6 +30,7 @@ test("Shopify comparison agreement includes all listing fields and result order"
     check([{ ...apps[0], ...change }, ...apps.slice(1)], 1);
   }
   check(apps.toReversed(), 1);
+  check([{ ...apps[0], app: "First℠ Subscriptions™" }, ...apps.slice(1)], 0);
 });
 
 test("Steam comparison agreement keeps prices and review summaries attached to their games in order", () => {
diff --git a/bench/test/tasks.test.mjs b/bench/test/tasks.test.mjs
index 47a8a44b..f3d1ba17 100644
--- a/bench/test/tasks.test.mjs
+++ b/bench/test/tasks.test.mjs
@@ -358,12 +358,12 @@ test("the guard extension runs the guard on its hosts and is named by its conten
   assert.notEqual(writeGuardExtension(join(dir, "d"), { ...guard, selectors: "#buy" }).name, first.name);
 });
 
-test("the DoorDash address check reads the header's address line, not a store named after the address", () => {
+test("the DoorDash address check reads the header's address line, not a store or place named after the address", () => {
   const intent = loadIntents(benchDir).find(i => i.id === "doordash-pizza-near-address");
   const answer = JSON.stringify({ places: [1, 2, 3].map(n => ({ name: `Pizza ${n}`, detail: "$0 delivery fee" })) });
   const grade = text => gradeAnswer(intent, loadSchema(benchDir, intent), answer, { output: { url: "https://www.doordash.com/", title: "DoorDash", text, fields: {}, missing: [], invalid: [] } });
-  assert.equal(grade("DoorDash\n4 Times Sq\nSign In").outcome, "correct");
-  for (const text of ["DoorDash\nYour Address\nJohn's of Times Square", "DoorDash\nJohn's of Times Square", "4 Times Sq, New York, NY 10036"]) {
+  for (const text of ["DoorDash\n4 Times Sq\nSign In", "DoorDash\nTimes Square\nDelivery"]) assert.equal(grade(text).outcome, "correct", text);
+  for (const text of ["DoorDash\nYour Address\nJohn's of Times Square", "DoorDash\nJohn's of Times Square", "4 Times Sq, New York, NY 10036", "DoorDash\nTimes Square Kitchen\nDelivery", "DoorDash\nTimes Squares Square Dance Club\nDelivery", "DoorDash\nCrowne Plaza Manhattan Times Square\nDelivery"]) {
     assert.equal(grade(text).outcome, "wrong", text);
   }
 });