diff --git a/packages/agent/src/index.ts b/packages/agent/src/index.ts index 220fff4..da4305b 100644 --- a/packages/agent/src/index.ts +++ b/packages/agent/src/index.ts @@ -5,7 +5,8 @@ export type { KernelBrowser } from "./translator/translator"; export { InternalComputerTranslator } from "./translator/translator"; export { CdpConnection } from "./translator/cdp"; export { BrowserExecutor } from "./translator/browser"; -export type { BrowserFindCandidate, BrowserRefState } from "./translator/browser"; +export type { BrowserFindCandidate } from "./translator/browser"; +export type { BrowserRefState } from "./translator/browser-ref-lifecycle"; export type { BatchExecutionResult, BatchReadResult } from "./translator/types"; export { createCuaComputerTools } from "./tools"; export type { diff --git a/packages/agent/src/translator/browser-frame-collection.ts b/packages/agent/src/translator/browser-frame-collection.ts new file mode 100644 index 0000000..a5d1bb7 --- /dev/null +++ b/packages/agent/src/translator/browser-frame-collection.ts @@ -0,0 +1,35 @@ +import { CdpProtocolError } from "./cdp"; + +export class FrameCollectionError extends Error { + constructor(message: string, cause: unknown) { + super(message, { cause }); + this.name = "FrameCollectionError"; + } +} + +export function isExpectedFrameCollectionError( + error: unknown, + method: "DOM.describeNode" | "Accessibility.getFullAXTree", +): error is CdpProtocolError { + if (!(error instanceof CdpProtocolError) || error.method !== method) return false; + const message = error.protocolMessage.trim(); + if (method === "DOM.describeNode") { + return /^(?:Could not find node with given id|No node with given id found)\.?$/i.test(message); + } + return /^(?:Frame with the given id was not found|No frame for given id found|Session with given id not found|Target session terminated)\.?$/i.test( + message, + ); +} + +export function frameCollectionError( + backendNodeId: number, + frameId: string | undefined, + stage: string, + cause: unknown, +): FrameCollectionError { + const detail = cause instanceof Error ? cause.message : String(cause); + return new FrameCollectionError( + `Failed to collect iframe ${frameId ?? "with unknown frame id"} at backend node ${backendNodeId} during ${stage}: ${detail}`, + cause, + ); +} diff --git a/packages/agent/src/translator/browser-observation.ts b/packages/agent/src/translator/browser-observation.ts new file mode 100644 index 0000000..7ac9bc2 --- /dev/null +++ b/packages/agent/src/translator/browser-observation.ts @@ -0,0 +1,243 @@ +/** Minimal CDP accessibility node used by browser observations. */ +export interface AXNode { + readonly nodeId: string; + readonly ignored?: boolean; + readonly role?: { readonly value?: string }; + readonly name?: { readonly value?: string }; + readonly value?: { readonly value?: unknown }; + readonly properties?: readonly { readonly name: string; readonly value?: { readonly value?: unknown } }[]; + readonly backendDOMNodeId?: number; + readonly parentId?: string; + readonly childIds?: readonly string[]; +} + +/** Role/name cohort positions used when minting and healing refs. */ +export interface NthIndex { + readonly index: ReadonlyMap; + readonly cohorts: ReadonlyMap; +} + +/** Immutable frame and generation metadata required to mint a ref. */ +export interface RenderContext { + readonly targetId: string; + readonly frameKey: string; + readonly sessionId: string; + readonly generation: number; + readonly nthIndex: NthIndex; + readonly cursorIds?: ReadonlySet; +} + +/** One normalized accessibility line before its ref is minted. */ +export interface ObservationLine { + readonly text: string; + readonly refNode?: AXNode; + readonly ctx: RenderContext; +} + +/** Accessibility tree collected for one frame. */ +export interface FrameStitch { + readonly byId: ReadonlyMap; + readonly roots: readonly string[]; + readonly ctx: RenderContext; +} + +/** A child frame that could not be collected because it detached or became inaccessible. */ +export interface IncompleteFrame { + readonly backendNodeId: number; + readonly frameId?: string; + readonly stage: "describe" | "resolve" | "accessibility"; + readonly reason: string; +} + +/** Accessibility node paired with its frame context. */ +export interface ObservedNode { + readonly node: AXNode; + readonly ctx: RenderContext; +} + +/** Stable structured browser state collected before presentation filtering. */ +export interface BrowserObservation { + readonly targetId: string; + readonly tree: FrameStitch; + readonly stitches: ReadonlyMap; + readonly incompleteFrames: readonly IncompleteFrame[]; + /** Target topology revision used only for observation/cache fencing, not ref validity. */ + readonly revision: number; + readonly generations: ReadonlyMap; +} + +/** Render-ready projection of one structured browser observation. */ +export interface BrowserPresentation { + readonly observation: BrowserObservation; + readonly cacheKey: string; + readonly lines: readonly ObservationLine[]; + readonly shape: string; +} + +/** Signals that browser state changed while an observation was collected. */ +export class ObservationChangedError extends Error { + constructor(message = "Browser observation changed during collection") { + super(message); + this.name = "ObservationChangedError"; + } +} + +/** Signals that a scoped ref could not be verified in an incompletely collected frame. */ +export class IncompleteObservationError extends Error { + constructor(message: string) { + super(message); + this.name = "IncompleteObservationError"; + } +} + +export const REF_PLACEHOLDER = "\u0000"; + +export const INTERACTIVE_ROLES: ReadonlySet = new Set([ + "button", + "link", + "textbox", + "searchbox", + "checkbox", + "radio", + "combobox", + "listbox", + "option", + "menuitem", + "menuitemcheckbox", + "menuitemradio", + "slider", + "spinbutton", + "switch", + "tab", + "treeitem", +]); + +export const FRAME_ROLES: ReadonlySet = new Set(["Iframe", "IframePresentational"]); + +const SKIPPED_ROLES: ReadonlySet = new Set(["none", "generic", "InlineTextBox", "LineBreak", "StaticText"]); + +/** Non-interactive roles that get refs when named, so scroll_to / ref-scoped snapshots can target them. */ +const CONTENT_ROLES: ReadonlySet = new Set([ + "heading", + "cell", + "gridcell", + "columnheader", + "rowheader", + "row", + "listitem", + "article", + "region", + "main", + "navigation", + "banner", + "contentinfo", + "complementary", + "tabpanel", + "figure", + "image", +]); + +/** Index each ref-healing candidate by its position among nodes with the same role and name, in tree order. */ +export function buildNthIndex(nodes: readonly AXNode[]): NthIndex { + const cohorts = new Map(); + const index = new Map(); + for (const node of nodes) { + if (node.ignored || node.backendDOMNodeId === undefined) continue; + const key = cohortKey(node.role?.value ?? "", node.name?.value ?? ""); + const nth = cohorts.get(key) ?? 0; + cohorts.set(key, nth + 1); + index.set(node.nodeId, nth); + } + return { index, cohorts }; +} + +/** Build the stable key for a role/name ref-healing cohort. */ +export function cohortKey(role: string, name: string): string { + return `${role}\u0000${name}`; +} + +/** Iterate an observation without eagerly allocating a wrapper for every AX node. */ +export function* observedNodes(observation: BrowserObservation): IterableIterator { + for (const node of observation.tree.byId.values()) yield { node, ctx: observation.tree.ctx }; + for (const stitch of observation.stitches.values()) { + for (const node of stitch.byId.values()) yield { node, ctx: stitch.ctx }; + } +} + +/** Merge a run of two or more consecutive StaticText siblings (text split by inline markup) into one node. */ +export function staticTextRun( + tree: ReadonlyMap, + childIds: readonly string[], + start: number, +): { node: AXNode; end: number } | undefined { + let end = start; + const parts: string[] = []; + while (end < childIds.length) { + const node = tree.get(childIds[end]!); + if (!node || node.ignored || node.role?.value !== "StaticText") break; + const text = node.name?.value ?? ""; + if (text) parts.push(text); + end += 1; + } + if (end - start < 2) return undefined; + const first = tree.get(childIds[start]!)!; + return { node: { ...first, name: { value: parts.join(" ") }, childIds: [] }, end: end - 1 }; +} + +export function renderObservationNode( + node: AXNode, + depth: number, + parentName: string, + ctx: RenderContext, + interactiveOnly: boolean, +): { text: string; refNode?: AXNode } | undefined { + const role = node.role?.value ?? ""; + const name = node.name?.value ?? ""; + const interactive = INTERACTIVE_ROLES.has(role); + const pointer = node.backendDOMNodeId !== undefined && (ctx.cursorIds?.has(node.backendDOMNodeId) ?? false); + if (interactiveOnly && !interactive && !pointer) return undefined; + if (role === "StaticText" && name === parentName) return undefined; + if (!interactiveOnly && !name && !interactive && !pointer && SKIPPED_ROLES.has(role)) return undefined; + let line = `${" ".repeat(Math.min(depth, 20))}${role || "node"}${name ? ` ${JSON.stringify(name)}` : ""}`; + let refNode: AXNode | undefined; + const refWorthy = interactive || pointer || FRAME_ROLES.has(role) || (name !== "" && CONTENT_ROLES.has(role)); + if (node.backendDOMNodeId !== undefined && refWorthy) { + line += ` [${REF_PLACEHOLDER}]`; + refNode = node; + } + const states = collectStates(node); + if (pointer && !interactive) states.push("cursor:pointer"); + if (states.length > 0) line += ` [${states.join(", ")}]`; + return { text: line, refNode }; +} + +function collectStates(node: AXNode): string[] { + const states: string[] = []; + for (const property of node.properties ?? []) { + const value = property.value?.value; + switch (property.name) { + case "checked": + case "pressed": + case "expanded": + // False is meaningful here: it distinguishes an unchecked checkbox or + // collapsed disclosure from an element without the state at all. + if (value === true || value === "true") states.push(property.name); + else if (value === false || value === "false") states.push(`${property.name}=false`); + else if (value === "mixed") states.push(`${property.name}=mixed`); + break; + case "disabled": + case "selected": + case "required": + if (value === true || value === "true") states.push(property.name); + break; + case "level": + if (typeof value === "number") states.push(`level=${value}`); + break; + } + } + const value = node.value?.value; + if (value !== undefined && value !== "" && String(value) !== (node.name?.value ?? "")) { + states.push(`value=${JSON.stringify(String(value))}`); + } + return states; +} diff --git a/packages/agent/src/translator/browser-ref-lifecycle.ts b/packages/agent/src/translator/browser-ref-lifecycle.ts new file mode 100644 index 0000000..d85a730 --- /dev/null +++ b/packages/agent/src/translator/browser-ref-lifecycle.ts @@ -0,0 +1,263 @@ +import { ObservationChangedError } from "./browser-observation"; + +const STALE_REF_HINT = "Call snapshot (or find) to get fresh element references."; + +export interface RefEntry { + backendNodeId: number; + targetId: string; + /** Generation key: the owning page target id for main-frame refs, the frame id for iframe refs. */ + frameId: string; + /** Session to route DOM/Input calls through: the frame's own session for OOPIFs, the page session otherwise. */ + sessionId: string; + generation: number; + role: string; + name: string; + nth: number; + /** Size of the (role, name) cohort in the tree the ref was minted from. */ + cohort: number; +} + +interface TargetGenerationState { + readonly targetId: string; + generation: number; + observationRevision: number; +} + +interface FrameGenerationState { + readonly targetId: string; + readonly frameId: string; + generation: number; + captures: number; + refs: number; +} + +export interface GenerationCapture { + readonly targetId: string; + readonly frameKey: string; + readonly generation: number; + readonly targetGeneration: number; + readonly observationRevision: number; + readonly targetState: TargetGenerationState; + readonly frameState?: FrameGenerationState; +} + +/** + * Serializable ref state, so refs minted in one process (e.g. a `cua + * snapshot` invocation) can be resolved in a later one against the same + * browser. Session ids are process-local and deliberately not exported; + * imported refs rebind lazily. Backend node ids stay valid for the life of + * the document, and the usual generation/self-heal machinery covers pages + * that changed in between. + */ +export interface BrowserRefState { + refCounter: number; + activeTargetId?: string; + generations: Array<[string, number]>; + refs: Array<[string, Omit]>; +} + +/** + * Owns generation identity and ref retention. Target documents and child + * frames have distinct records even though CDP represents both ids as strings. + * Frame records live only while an observation is collecting or a ref needs + * them, so transient and rotating frames cannot grow retained state. + */ +export class RefGenerationLifecycle { + private readonly targets = new Map(); + private readonly frames = new Map(); + + constructor(private readonly refs: Map) {} + + captureTarget(targetId: string): GenerationCapture { + const targetState = this.ensureTarget(targetId); + return { + targetId, + frameKey: targetId, + generation: targetState.generation, + targetGeneration: targetState.generation, + observationRevision: targetState.observationRevision, + targetState, + }; + } + + captureFrame(targetId: string, frameId: string): GenerationCapture { + const targetState = this.ensureTarget(targetId); + let frameState = this.frames.get(frameId); + if (frameState && frameState.targetId !== targetId) { + throw new Error(`frame ${frameId} changed owner from ${frameState.targetId} to ${targetId}`); + } + frameState ??= { targetId, frameId, generation: 0, captures: 0, refs: 0 }; + this.frames.set(frameId, frameState); + frameState.captures += 1; + return { + targetId, + frameKey: frameId, + generation: frameState.generation, + targetGeneration: targetState.generation, + observationRevision: targetState.observationRevision, + targetState, + frameState, + }; + } + + isCurrent(capture: GenerationCapture): boolean { + if ( + this.targets.get(capture.targetId) !== capture.targetState || + capture.targetState.generation !== capture.targetGeneration || + capture.targetState.observationRevision !== capture.observationRevision + ) { + return false; + } + return ( + !capture.frameState || + (this.frames.get(capture.frameKey) === capture.frameState && capture.frameState.generation === capture.generation) + ); + } + + release(captures: readonly GenerationCapture[]): void { + for (const capture of captures) { + if (!capture.frameState) continue; + capture.frameState.captures = Math.max(0, capture.frameState.captures - 1); + this.deleteUnusedFrame(capture.frameState); + } + } + + retainRef(ref: string, entry: RefEntry): void { + this.deleteRef(ref); + if (entry.frameId === entry.targetId) { + const target = this.targets.get(entry.targetId); + if (!target || target.generation !== entry.generation) throw new ObservationChangedError(); + } else { + const frame = this.frames.get(entry.frameId); + if (!frame || frame.targetId !== entry.targetId || frame.generation !== entry.generation) { + throw new ObservationChangedError(); + } + frame.refs += 1; + } + this.refs.set(ref, entry); + } + + deleteRef(ref: string): void { + const entry = this.refs.get(ref); + if (!entry) return; + this.refs.delete(ref); + if (entry.frameId === entry.targetId) return; + const frame = this.frames.get(entry.frameId); + if (!frame || frame.targetId !== entry.targetId) return; + frame.refs = Math.max(0, frame.refs - 1); + this.deleteUnusedFrame(frame); + } + + isRefCurrent(entry: RefEntry): boolean { + if (entry.frameId === entry.targetId) return this.targets.get(entry.targetId)?.generation === entry.generation; + const frame = this.frames.get(entry.frameId); + return frame?.targetId === entry.targetId && frame.generation === entry.generation; + } + + invalidateTarget(targetId: string): void { + const target = this.ensureTarget(targetId); + target.generation += 1; + target.observationRevision += 1; + this.deleteRefs((entry) => entry.targetId === targetId); + for (const frame of [...this.frames.values()]) { + if (frame.targetId !== targetId) continue; + frame.generation += 1; + this.deleteUnusedFrame(frame); + } + } + + invalidateFrame(targetId: string, frameId: string): void { + const target = this.targets.get(targetId); + if (target) target.observationRevision += 1; + const frame = this.frames.get(frameId); + if (!frame || frame.targetId !== targetId) return; + frame.generation += 1; + this.deleteRefs((entry) => entry.targetId === targetId && entry.frameId === frameId); + this.deleteUnusedFrame(frame); + } + + removeFrame(targetId: string, frameId: string): void { + const target = this.targets.get(targetId); + if (target) target.observationRevision += 1; + const frame = this.frames.get(frameId); + if (!frame || frame.targetId !== targetId) return; + this.deleteRefs((entry) => entry.targetId === targetId && entry.frameId === frameId); + this.frames.delete(frameId); + } + + dropTarget(targetId: string): void { + this.deleteRefs((entry) => entry.targetId === targetId); + this.targets.delete(targetId); + for (const [frameId, frame] of this.frames) { + if (frame.targetId === targetId) this.frames.delete(frameId); + } + } + + exportGenerations(): Array<[string, number]> { + return [ + ...[...this.targets.values()].map((state): [string, number] => [state.targetId, state.generation]), + ...[...this.frames.values()].map((state): [string, number] => [state.frameId, state.generation]), + ]; + } + + importState( + generations: readonly (readonly [string, number])[], + entries: readonly (readonly [string, Omit])[], + activeTargetId?: string, + ): void { + const importedGenerations = new Map(generations); + const targetIds = new Set(entries.map(([, entry]) => entry.targetId)); + if (activeTargetId) targetIds.add(activeTargetId); + for (const targetId of targetIds) { + this.targets.set(targetId, { + targetId, + generation: importedGenerations.get(targetId) ?? 0, + observationRevision: 0, + }); + } + for (const [ref, imported] of entries) { + this.deleteRef(ref); + const entry: RefEntry = { ...imported, sessionId: "" }; + if (entry.frameId !== entry.targetId) { + let frame = this.frames.get(entry.frameId); + if (!frame) { + frame = { + targetId: entry.targetId, + frameId: entry.frameId, + generation: importedGenerations.get(entry.frameId) ?? entry.generation, + captures: 0, + refs: 0, + }; + this.frames.set(entry.frameId, frame); + } + frame.refs += 1; + } + this.refs.set(ref, entry); + } + } + + private ensureTarget(targetId: string): TargetGenerationState { + let target = this.targets.get(targetId); + if (!target) { + target = { targetId, generation: 0, observationRevision: 0 }; + this.targets.set(targetId, target); + } + return target; + } + + private deleteRefs(predicate: (entry: RefEntry) => boolean): void { + for (const [ref, entry] of this.refs) { + if (predicate(entry)) this.deleteRef(ref); + } + } + + private deleteUnusedFrame(frame: FrameGenerationState): void { + if (frame.captures === 0 && frame.refs === 0 && this.frames.get(frame.frameId) === frame) { + this.frames.delete(frame.frameId); + } + } +} + +export function staleRefError(ref: string, cause?: unknown): Error { + return new Error(`ref ${ref} is stale or not on the current page. ${STALE_REF_HINT}`, cause === undefined ? undefined : { cause }); +} diff --git a/packages/agent/src/translator/browser.ts b/packages/agent/src/translator/browser.ts index 9cb21a1..4b9122c 100644 --- a/packages/agent/src/translator/browser.ts +++ b/packages/agent/src/translator/browser.ts @@ -13,6 +13,33 @@ import { type CuaBrowserAction, } from "@onkernel/cua-ai"; import { CdpConnection, type CdpEventMessage } from "./cdp"; +import { FrameCollectionError, frameCollectionError, isExpectedFrameCollectionError } from "./browser-frame-collection"; +import { + FRAME_ROLES, + INTERACTIVE_ROLES, + REF_PLACEHOLDER, + IncompleteObservationError, + ObservationChangedError, + buildNthIndex, + cohortKey, + observedNodes, + renderObservationNode, + staticTextRun, + type AXNode, + type BrowserObservation, + type BrowserPresentation, + type FrameStitch, + type IncompleteFrame, + type ObservationLine, + type RenderContext, +} from "./browser-observation"; +import { + RefGenerationLifecycle, + staleRefError, + type BrowserRefState, + type GenerationCapture, + type RefEntry, +} from "./browser-ref-lifecycle"; import type { BatchReadResult } from "./types"; const SNAPSHOT_CHAR_LIMIT = 50_000; @@ -21,62 +48,11 @@ const FIND_MATCH_LIMIT = 20; const REF_LIMIT_PER_TARGET = 1000; const SCROLL_NOTCH_PX = 120; -const STALE_REF_HINT = "Call snapshot (or find) to get fresh element references."; -const REF_PLACEHOLDER = "\u0000"; const UNCHANGED_SNAPSHOT = "Page unchanged since the last snapshot; previous element refs are still valid."; -interface AXNode { - nodeId: string; - ignored?: boolean; - role?: { value?: string }; - name?: { value?: string }; - value?: { value?: unknown }; - properties?: Array<{ name: string; value?: { value?: unknown } }>; - backendDOMNodeId?: number; - parentId?: string; - childIds?: string[]; -} - -interface RefEntry { - backendNodeId: number; - targetId: string; - /** Generation key: the owning page target id for main-frame refs, the frame id for iframe refs. */ - frameId: string; - /** Session to route DOM/Input calls through: the frame's own session for OOPIFs, the page session otherwise. */ - sessionId: string; - generation: number; - role: string; - name: string; - nth: number; - /** Size of the (role, name) cohort in the tree the ref was minted from. */ - cohort: number; -} - -interface NthIndex { - index: Map; - cohorts: Map; -} - -interface RenderContext { - targetId: string; - frameKey: string; - sessionId: string; - generation: number; - interactiveOnly: boolean; - nthIndex: NthIndex; - cursorIds?: ReadonlySet; -} - -interface RenderedLine { - text: string; - refNode?: AXNode; - ctx: RenderContext; -} - -interface FrameStitch { - byId: Map; - roots: string[]; - ctx: RenderContext; +interface CollectedObservation { + readonly observation: BrowserObservation; + readonly captures: readonly GenerationCapture[]; } export interface BrowserFindCandidate { @@ -86,21 +62,6 @@ export interface BrowserFindCandidate { score: number; } -/** - * Serializable ref state, so refs minted in one process (e.g. a `cua - * snapshot` invocation) can be resolved in a later one against the same - * browser. Session ids are process-local and deliberately not exported; - * imported refs rebind lazily. Backend node ids stay valid for the life of - * the document, and the usual generation/self-heal machinery covers pages - * that changed in between. - */ -export interface BrowserRefState { - refCounter: number; - activeTargetId?: string; - generations: Array<[string, number]>; - refs: Array<[string, Omit]>; -} - /** * Executes browser-plane canonical actions over CDP. * @@ -131,9 +92,10 @@ export interface BrowserRefState { */ export class BrowserExecutor { private readonly refs = new Map(); - private readonly generations = new Map(); + private readonly lifecycle = new RefGenerationLifecycle(this.refs); private readonly targetsBySession = new Map(); private readonly frameSessions = new Map(); + private readonly frameOwners = new Map(); private readonly frameTargets = new Set(); private readonly lastSnapshots = new Map(); private readonly selfNavigations = new Set(); @@ -152,20 +114,33 @@ export class BrowserExecutor { case "Page.frameNavigated": { const frame = event.params.frame as { id?: string; parentId?: string } | undefined; if (!event.sessionId || !frame) return; - const targetId = this.targetsBySession.get(event.sessionId); - if (!targetId) return; - if (this.frameTargets.has(targetId)) { - // Refs from a frame target's tree (its root and any same-process - // subframes inlined in it) are all minted against the target's key, - // so any navigation observed in its session stales them. - this.invalidateFrame(targetId); + const sessionTargetId = this.targetsBySession.get(event.sessionId); + if (!sessionTargetId) return; + if (this.frameTargets.has(sessionTargetId)) { + // The OOPIF tree and any same-process descendants fetched through + // its session share the OOPIF generation key. + const owner = this.frameOwners.get(sessionTargetId); + if (owner) this.lifecycle.invalidateFrame(owner, sessionTargetId); return; } if (frame.parentId) { - if (frame.id) this.invalidateFrame(frame.id); + if (frame.id) this.lifecycle.invalidateFrame(sessionTargetId, frame.id); return; } - if (!this.selfNavigations.delete(targetId)) this.invalidateRefs(targetId); + if (!this.selfNavigations.delete(sessionTargetId)) this.lifecycle.invalidateTarget(sessionTargetId); + return; + } + case "Page.frameDetached": { + if (!event.sessionId) return; + const { frameId } = event.params as { frameId?: string; reason?: "remove" | "swap" }; + const sessionTargetId = this.targetsBySession.get(event.sessionId); + if (!frameId || !sessionTargetId) return; + if (this.frameTargets.has(sessionTargetId)) { + const owner = this.frameOwners.get(sessionTargetId); + if (owner) this.lifecycle.invalidateFrame(owner, sessionTargetId); + } else { + this.lifecycle.removeFrame(sessionTargetId, frameId); + } return; } case "Page.navigatedWithinDocument": { @@ -180,7 +155,10 @@ export class BrowserExecutor { case "Target.attachedToTarget": { const { sessionId, targetInfo } = event.params as { sessionId?: string; targetInfo?: { targetId?: string; type?: string } }; if (!sessionId || !targetInfo?.targetId || targetInfo.type !== "iframe") return; + const parentTarget = event.sessionId ? this.targetsBySession.get(event.sessionId) : undefined; + const owner = parentTarget ? (this.frameOwners.get(parentTarget) ?? parentTarget) : undefined; this.frameSessions.set(targetInfo.targetId, sessionId); + if (owner) this.frameOwners.set(targetInfo.targetId, owner); this.frameTargets.add(targetInfo.targetId); this.targetsBySession.set(sessionId, targetInfo.targetId); void this.cdp.send("Page.enable", {}, sessionId).catch(() => {}); @@ -205,7 +183,16 @@ export class BrowserExecutor { if (typeof sessionId !== "string") return; const targetId = this.targetsBySession.get(sessionId); this.targetsBySession.delete(sessionId); - if (targetId) this.dropTarget(targetId); + if (!targetId) return; + if (this.frameTargets.has(targetId)) { + const owner = this.frameOwners.get(targetId); + if (owner) this.lifecycle.removeFrame(owner, targetId); + this.frameSessions.delete(targetId); + this.frameOwners.delete(targetId); + this.frameTargets.delete(targetId); + } else { + this.dropTarget(targetId); + } return; } } @@ -221,7 +208,7 @@ export class BrowserExecutor { return { refCounter: this.refCounter, ...(this.activeTargetId ? { activeTargetId: this.activeTargetId } : {}), - generations: [...this.generations], + generations: this.lifecycle.exportGenerations(), refs: [...this.refs].map(([ref, { sessionId: _sessionId, ...entry }]) => [ref, entry]), }; } @@ -230,8 +217,7 @@ export class BrowserExecutor { importRefState(state: BrowserRefState): void { this.refCounter = Math.max(this.refCounter, state.refCounter); this.activeTargetId = state.activeTargetId ?? this.activeTargetId; - for (const [frameId, generation] of state.generations) this.generations.set(frameId, generation); - for (const [ref, entry] of state.refs) this.refs.set(ref, { ...entry, sessionId: "" }); + this.lifecycle.importState(state.generations, state.refs, state.activeTargetId); } async execute(action: CuaBrowserAction): Promise { @@ -306,72 +292,144 @@ export class BrowserExecutor { } private async snapshot(action: CuaActionBrowserSnapshot): Promise { - const targetId = await this.resolveTarget(action.tab_id); + return this.withObservation(action.tab_id, true, (observation) => + this.renderObservation(this.presentObservation(observation, action)), + ); + } + + private async withObservation( + tabId: string | undefined, + includeCursor: boolean, + consume: (observation: BrowserObservation) => T, + ): Promise { + for (let attempt = 0; ; attempt += 1) { + let collected: CollectedObservation | undefined; + try { + collected = await this.collectObservation(tabId, includeCursor); + return consume(collected.observation); + } catch (error) { + const retryable = error instanceof ObservationChangedError || error instanceof IncompleteObservationError; + if (!retryable || attempt === 2) throw error; + } finally { + if (collected) this.lifecycle.release(collected.captures); + } + } + } + + private async collectObservation(tabId: string | undefined, includeCursor: boolean): Promise { + const targetId = await this.resolveTarget(tabId); const pageSession = await this.attach(targetId); - const refEntry = action.ref ? this.resolveRef(action.ref, targetId) : undefined; - const frameKey = refEntry?.frameId ?? targetId; - const { nodes, sessionId } = await this.frameAxTree(frameKey, targetId, pageSession); - const byId = new Map(nodes.map((node) => [node.nodeId, node])); - let rootIds = nodes.filter((node) => !node.parentId).map((node) => node.nodeId); + const before = (await this.cdp.pageTargets()).find((target) => target.targetId === targetId); + if (!before) throw new ObservationChangedError("Browser target disappeared during observation"); + + const captures: GenerationCapture[] = [this.lifecycle.captureTarget(targetId)]; + try { + const rootCapture = captures[0]!; + const { nodes, sessionId } = await this.frameAxTree(targetId, targetId, pageSession); + const rootCtx: RenderContext = { + targetId, + frameKey: targetId, + sessionId, + generation: rootCapture.generation, + nthIndex: buildNthIndex(nodes), + ...(includeCursor ? { cursorIds: await this.cursorPointerIds(pageSession) } : {}), + }; + const tree = this.frameStitch(nodes, rootCtx); + const { stitches, incompleteFrames } = await this.stitchFrames(nodes, targetId, pageSession, captures); + const after = (await this.cdp.pageTargets()).find((target) => target.targetId === targetId); + if (!after || before.url !== after.url || before.title !== after.title) { + throw new ObservationChangedError("Browser target metadata changed during observation"); + } + if (captures.some((capture) => !this.lifecycle.isCurrent(capture))) throw new ObservationChangedError(); + return { + observation: { + targetId, + tree, + stitches, + incompleteFrames, + revision: rootCapture.observationRevision, + generations: new Map(captures.map((capture) => [capture.frameKey, capture.generation])), + }, + captures, + }; + } catch (error) { + this.lifecycle.release(captures); + throw error; + } + } + + private presentObservation(observation: BrowserObservation, action: CuaActionBrowserSnapshot): BrowserPresentation { + const refEntry = action.ref ? this.resolveRef(action.ref, observation.targetId) : undefined; + let tree = observation.tree; + let rootIds = tree.roots; if (action.ref && refEntry) { + if (refEntry.frameId === observation.targetId) { + tree = observation.tree; + } else { + const frameTree = [...observation.stitches.values()].find((stitch) => stitch.ctx.frameKey === refEntry.frameId); + if (!frameTree) { + const details = observation.incompleteFrames + .map((frame) => `${frame.frameId ?? `backend node ${frame.backendNodeId}`} (${frame.reason})`) + .join(", "); + throw new IncompleteObservationError( + `Could not verify ref ${action.ref}: owning frame ${refEntry.frameId} was not collected${details ? `; incomplete frames: ${details}` : ""}`, + ); + } + tree = frameTree; + } + const treeNodes = [...tree.byId.values()]; const rootNode = - nodes.find((node) => node.backendDOMNodeId === refEntry.backendNodeId) ?? this.healEntry(action.ref, refEntry, nodes); + treeNodes.find((node) => node.backendDOMNodeId === refEntry.backendNodeId) ?? this.healEntry(action.ref, refEntry, treeNodes); rootIds = [rootNode.nodeId]; } - const interactiveOnly = action.filter === "interactive"; - const ctx: RenderContext = { - targetId, - frameKey, - sessionId, - generation: this.generation(frameKey), - interactiveOnly, - nthIndex: buildNthIndex(nodes), - cursorIds: frameKey === targetId ? await this.cursorPointerIds(pageSession) : undefined, - }; - const stitches = frameKey === targetId ? await this.stitchFrames(nodes, targetId, pageSession, interactiveOnly) : new Map(); - const lines: RenderedLine[] = []; + const lines: ObservationLine[] = []; const maxDepth = action.depth ?? DEFAULT_SNAPSHOT_DEPTH; - const walk = (tree: Map, treeCtx: RenderContext, nodeId: string, depth: number, parentName: string): void => { - const node = tree.get(nodeId); + const walk = (current: FrameStitch, nodeId: string, depth: number, parentName: string): void => { + const node = current.byId.get(nodeId); if (!node) return; + const ctx = current.ctx; let childDepth = depth; if (!node.ignored) { - const rendered = this.renderNode(node, depth, parentName, treeCtx); + const rendered = renderObservationNode(node, depth, parentName, ctx, interactiveOnly); if (rendered) { - lines.push({ ...rendered, ctx: treeCtx }); + lines.push({ ...rendered, ctx }); childDepth = depth + 1; } } if (childDepth > maxDepth) return; - const stitch = treeCtx === ctx && node.backendDOMNodeId !== undefined ? stitches.get(node.backendDOMNodeId) : undefined; + const stitch = current === observation.tree && node.backendDOMNodeId !== undefined ? observation.stitches.get(node.backendDOMNodeId) : undefined; if (stitch) { - for (const frameRootId of stitch.roots) walk(stitch.byId, stitch.ctx, frameRootId, childDepth, ""); + for (const frameRootId of stitch.roots) walk(stitch, frameRootId, childDepth, ""); return; } - const name = node.name?.value ?? ""; - const childName = name || parentName; + const childName = node.name?.value || parentName; const childIds = node.childIds ?? []; for (let i = 0; i < childIds.length; i += 1) { - const run = staticTextRun(tree, childIds, i); + const run = staticTextRun(current.byId, childIds, i); if (run) { - const rendered = this.renderNode(run.node, childDepth, childName, treeCtx); - if (rendered) lines.push({ ...rendered, ctx: treeCtx }); + const rendered = renderObservationNode(run.node, childDepth, childName, ctx, interactiveOnly); + if (rendered) lines.push({ ...rendered, ctx }); i = run.end; - continue; - } - walk(tree, treeCtx, childIds[i]!, childDepth, childName); + } else walk(current, childIds[i]!, childDepth, childName); } }; - for (const rootId of rootIds) walk(byId, ctx, rootId, 0, ""); - + for (const rootId of rootIds) walk(tree, rootId, 0, ""); const shape = lines.map((line) => line.text).join("\n"); - const frameGenerations = [...stitches.values()].map((stitch) => `${stitch.ctx.frameKey}:${stitch.ctx.generation}`); - const key = [action.ref ?? "", action.depth ?? "", action.filter ?? "", `${frameKey}:${ctx.generation}`, ...frameGenerations].join("|"); - const cached = this.lastSnapshots.get(targetId); - this.lastSnapshots.set(targetId, { key, shape }); - if (cached && cached.key === key && cached.shape === shape) return UNCHANGED_SNAPSHOT; + const generationKey = [...observation.generations].map(([frameKey, generation]) => `${frameKey}:${generation}`).join("|"); + return { + observation, + cacheKey: [action.ref ?? "", action.depth ?? "", action.filter ?? "", `revision:${observation.revision}`, generationKey].join("|"), + lines, + shape, + }; + } + private renderObservation(presentation: BrowserPresentation): string { + const { observation, cacheKey, lines, shape } = presentation; + const cached = this.lastSnapshots.get(observation.targetId); + this.lastSnapshots.set(observation.targetId, { key: cacheKey, shape }); + if (cached?.key === cacheKey && cached.shape === shape) return UNCHANGED_SNAPSHOT; let text = ""; for (const line of lines) { if (text.length > SNAPSHOT_CHAR_LIMIT) break; @@ -381,31 +439,10 @@ export class BrowserExecutor { if (text.length > SNAPSHOT_CHAR_LIMIT) { text = `${text.slice(0, SNAPSHOT_CHAR_LIMIT)}\n… truncated at ${SNAPSHOT_CHAR_LIMIT} characters. Re-request with a smaller depth, filter: "interactive", or a ref to narrow the subtree.`; } - this.pruneRefs(targetId); + this.pruneRefs(observation.targetId); return text || "(empty accessibility tree)"; } - private renderNode(node: AXNode, depth: number, parentName: string, ctx: RenderContext): { text: string; refNode?: AXNode } | undefined { - const role = node.role?.value ?? ""; - const name = node.name?.value ?? ""; - const interactive = INTERACTIVE_ROLES.has(role); - const pointer = node.backendDOMNodeId !== undefined && (ctx.cursorIds?.has(node.backendDOMNodeId) ?? false); - if (ctx.interactiveOnly && !interactive && !pointer) return undefined; - if (role === "StaticText" && name === parentName) return undefined; - if (!ctx.interactiveOnly && !name && !interactive && !pointer && SKIPPED_ROLES.has(role)) return undefined; - let line = `${" ".repeat(Math.min(depth, 20))}${role || "node"}${name ? ` ${JSON.stringify(name)}` : ""}`; - let refNode: AXNode | undefined; - const refWorthy = interactive || pointer || FRAME_ROLES.has(role) || (name !== "" && CONTENT_ROLES.has(role)); - if (node.backendDOMNodeId !== undefined && refWorthy) { - line += ` [${REF_PLACEHOLDER}]`; - refNode = node; - } - const states = collectStates(node); - if (pointer && !interactive) states.push("cursor:pointer"); - if (states.length > 0) line += ` [${states.join(", ")}]`; - return { text: line, refNode }; - } - /** Fetch a frame's AX tree: OOPIFs through their own session, same-process frames through the page session with a frameId. */ private async frameAxTree(frameKey: string, targetId: string, pageSession: string): Promise<{ nodes: AXNode[]; sessionId: string }> { const frameSession = this.frameSessions.get(frameKey); @@ -418,42 +455,84 @@ export class BrowserExecutor { return { nodes, sessionId: pageSession }; } + private frameStitch(nodes: AXNode[], ctx: RenderContext): FrameStitch { + return { + byId: new Map(nodes.map((node) => [node.nodeId, node])), + roots: nodes.filter((node) => !node.parentId).map((node) => node.nodeId), + ctx, + }; + } + /** Resolve each iframe node's child frame and fetch its AX tree for stitching. One nesting level only. */ private async stitchFrames( - nodes: AXNode[], + nodes: readonly AXNode[], targetId: string, pageSession: string, - interactiveOnly: boolean, - ): Promise> { + captures: GenerationCapture[], + ): Promise<{ stitches: ReadonlyMap; incompleteFrames: readonly IncompleteFrame[] }> { const stitches = new Map(); + const incompleteFrames: IncompleteFrame[] = []; for (const node of nodes) { if (node.ignored || !FRAME_ROLES.has(node.role?.value ?? "") || node.backendDOMNodeId === undefined) continue; + const backendNodeId = node.backendDOMNodeId; + let dom: { frameId?: string; contentDocument?: { frameId?: string } }; try { - const { node: dom } = await this.cdp.send<{ node: { frameId?: string; contentDocument?: { frameId?: string } } }>( + const described = await this.cdp.send<{ node?: typeof dom }>( "DOM.describeNode", - { backendNodeId: node.backendDOMNodeId, depth: 1 }, + { backendNodeId, depth: 1 }, pageSession, ); - const frameId = dom.contentDocument?.frameId ?? dom.frameId; - if (!frameId || frameId === targetId) continue; + if (!described.node) throw new Error("response did not include a node"); + dom = described.node; + } catch (error) { + if (!isExpectedFrameCollectionError(error, "DOM.describeNode")) { + throw frameCollectionError(backendNodeId, undefined, "DOM.describeNode", error); + } + incompleteFrames.push({ backendNodeId, stage: "describe", reason: error.message }); + continue; + } + const frameId = dom.contentDocument?.frameId ?? dom.frameId; + if (!frameId || frameId === targetId) { + incompleteFrames.push({ backendNodeId, stage: "resolve", reason: "iframe did not expose a child frame id" }); + continue; + } + + if (this.frameSessions.has(frameId)) this.frameOwners.set(frameId, targetId); + let capture: GenerationCapture; + try { + capture = this.lifecycle.captureFrame(targetId, frameId); + } catch (error) { + throw frameCollectionError(backendNodeId, frameId, "capturing the frame generation", error); + } + let retained = false; + try { const { nodes: frameNodes, sessionId } = await this.frameAxTree(frameId, targetId, pageSession); - stitches.set(node.backendDOMNodeId, { - byId: new Map(frameNodes.map((frameNode) => [frameNode.nodeId, frameNode])), - roots: frameNodes.filter((frameNode) => !frameNode.parentId).map((frameNode) => frameNode.nodeId), - ctx: { + let stitch: FrameStitch; + try { + stitch = this.frameStitch(frameNodes, { targetId, frameKey: frameId, sessionId, - generation: this.generation(frameId), - interactiveOnly, + generation: capture.generation, nthIndex: buildNthIndex(frameNodes), - }, - }); - } catch { - // Cross-origin or already-detached frames can refuse the fetch; the iframe renders without children. + }); + } catch (error) { + throw frameCollectionError(backendNodeId, frameId, "building the accessibility index", error); + } + stitches.set(backendNodeId, stitch); + captures.push(capture); + retained = true; + } catch (error) { + if (error instanceof FrameCollectionError) throw error; + if (!isExpectedFrameCollectionError(error, "Accessibility.getFullAXTree")) { + throw frameCollectionError(backendNodeId, frameId, "Accessibility.getFullAXTree", error); + } + incompleteFrames.push({ backendNodeId, frameId, stage: "accessibility", reason: error.message }); + } finally { + if (!retained) this.lifecycle.release([capture]); } } - return stitches; + return { stitches, incompleteFrames }; } /** Resolve backend node ids for elements whose own computed cursor is "pointer", without touching the DOM. */ @@ -499,46 +578,32 @@ export class BrowserExecutor { * matches, best first. Structured counterpart of the `browser_find` action. */ async findCandidates(query: string, tabId?: string, roles?: ReadonlySet): Promise { - const targetId = await this.resolveTarget(tabId); - const session = await this.attach(targetId); - const { nodes } = await this.cdp.send<{ nodes: AXNode[] }>("Accessibility.getFullAXTree", {}, session); - const pools: Array<{ nodes: AXNode[]; ctx: RenderContext }> = [ - { - nodes, - ctx: { - targetId, - frameKey: targetId, - sessionId: session, - generation: this.generation(targetId), - interactiveOnly: false, - nthIndex: buildNthIndex(nodes), - }, - }, - ]; - // Search stitched frames too so find sees everything snapshot renders. - for (const stitch of (await this.stitchFrames(nodes, targetId, session, false)).values()) { - pools.push({ nodes: [...stitch.byId.values()], ctx: stitch.ctx }); - } - const queryTokens = tokenize(query); - const scored = pools - .flatMap(({ nodes: poolNodes, ctx }) => - poolNodes - .filter( - (node) => !node.ignored && node.backendDOMNodeId !== undefined && (node.name?.value || INTERACTIVE_ROLES.has(node.role?.value ?? "")), - ) - .map((node) => ({ node, ctx, score: overlapScore(queryTokens, tokenize(`${node.role?.value ?? ""} ${node.name?.value ?? ""}`)) })), - ) - .filter((entry) => entry.score > 0 && (!roles || roles.has(entry.node.role?.value ?? ""))) - .sort((a, b) => b.score - a.score) - .slice(0, FIND_MATCH_LIMIT); - const candidates = scored.map(({ node, ctx, score }) => ({ - ref: this.mintRef(node, ctx), - role: node.role?.value ?? "", - name: node.name?.value ?? "", - score, - })); - this.pruneRefs(targetId); - return candidates; + return this.withObservation(tabId, false, (observation) => { + const queryTokens = tokenize(query); + const scored = [...observedNodes(observation)] + .filter( + ({ node }) => + !node.ignored && + node.backendDOMNodeId !== undefined && + (node.name?.value || INTERACTIVE_ROLES.has(node.role?.value ?? "")), + ) + .map(({ node, ctx }) => ({ + node, + ctx, + score: overlapScore(queryTokens, tokenize(`${node.role?.value ?? ""} ${node.name?.value ?? ""}`)), + })) + .filter((entry) => entry.score > 0 && (!roles || roles.has(entry.node.role?.value ?? ""))) + .sort((a, b) => b.score - a.score) + .slice(0, FIND_MATCH_LIMIT); + const candidates = scored.map(({ node, ctx, score }) => ({ + ref: this.mintRef(node, ctx), + role: node.role?.value ?? "", + name: node.name?.value ?? "", + score, + })); + this.pruneRefs(observation.targetId); + return candidates; + }); } private async click(action: CuaActionBrowserClick): Promise { @@ -651,7 +716,7 @@ export class BrowserExecutor { const entry = history.entries[history.currentIndex + (direction === "back" ? -1 : 1)]; if (!entry) throw new Error(`cannot go ${direction}: no history entry`); await this.selfNavigate(targetId, () => this.cdp.send("Page.navigateToHistoryEntry", { entryId: entry.id }, session)); - this.invalidateRefs(targetId); + this.lifecycle.invalidateTarget(targetId); return `Navigated ${direction}.\n${await this.tabContext(targetId)}`; } const url = normalizeGotoUrl(action.url); @@ -663,7 +728,7 @@ export class BrowserExecutor { this.selfNavigations.delete(targetId); throw new Error(`navigation to ${url} failed: ${errorText}`); } - this.invalidateRefs(targetId); + this.lifecycle.invalidateTarget(targetId); return `Navigated to ${url}.\n${await this.tabContext(targetId)}`; } @@ -806,7 +871,7 @@ export class BrowserExecutor { const name = node.name?.value ?? ""; this.refCounter += 1; const ref = `e${this.refCounter}`; - this.refs.set(ref, { + this.lifecycle.retainRef(ref, { backendNodeId: node.backendDOMNodeId!, targetId: ctx.targetId, frameId: ctx.frameKey, @@ -837,47 +902,21 @@ export class BrowserExecutor { const entry = this.refs.get(ref); // Entries are deleted eagerly on invalidation; the generation check only // guards refs resolved while a navigation event is still in flight. - if (!entry || entry.targetId !== targetId || entry.generation !== this.generation(entry.frameId)) { - throw staleRefError(ref); - } + if (!entry || entry.targetId !== targetId || !this.lifecycle.isRefCurrent(entry)) throw staleRefError(ref); return entry; } - private generation(targetId: string): number { - return this.generations.get(targetId) ?? 0; - } - - private invalidateRefs(targetId: string): void { - this.generations.set(targetId, this.generation(targetId) + 1); - for (const [ref, entry] of this.refs) { - if (entry.targetId === targetId) this.refs.delete(ref); - } - } - - private invalidateFrame(frameKey: string): void { - let tracked = this.generations.has(frameKey) || this.frameSessions.has(frameKey); - for (const [ref, entry] of this.refs) { - if (entry.frameId === frameKey) { - this.refs.delete(ref); - tracked = true; - } - } - // Frames we never referenced don't get a generation entry, or pages with - // rotating ad iframes would grow the map without bound. - if (tracked) this.generations.set(frameKey, this.generation(frameKey) + 1); - } - private dropTarget(targetId: string): void { - this.generations.delete(targetId); + this.lifecycle.dropTarget(targetId); this.selfNavigations.delete(targetId); this.lastSnapshots.delete(targetId); - this.frameSessions.delete(targetId); - this.frameTargets.delete(targetId); - for (const [ref, entry] of this.refs) { - if (entry.targetId === targetId || entry.frameId === targetId) { - if (entry.frameId !== targetId) this.generations.delete(entry.frameId); - this.refs.delete(ref); - } + for (const [frameId, owner] of this.frameOwners) { + if (owner !== targetId) continue; + const sessionId = this.frameSessions.get(frameId); + if (sessionId) this.targetsBySession.delete(sessionId); + this.frameSessions.delete(frameId); + this.frameOwners.delete(frameId); + this.frameTargets.delete(frameId); } } @@ -887,7 +926,7 @@ export class BrowserExecutor { for (const [ref, entry] of this.refs) { if (entry.targetId === targetId) owned.push(ref); } - for (const ref of owned.slice(0, Math.max(0, owned.length - REF_LIMIT_PER_TARGET))) this.refs.delete(ref); + for (const ref of owned.slice(0, Math.max(0, owned.length - REF_LIMIT_PER_TARGET))) this.lifecycle.deleteRef(ref); } private drainDialogNotes(): string | undefined { @@ -932,75 +971,6 @@ function tabOf(action: { tab_id?: string }): string | undefined { return action.tab_id; } -function staleRefError(ref: string, cause?: unknown): Error { - return new Error(`ref ${ref} is stale or not on the current page. ${STALE_REF_HINT}`, cause === undefined ? undefined : { cause }); -} - -/** Index each ref-eligible node by its position among nodes with the same role and name, in tree order. */ -function buildNthIndex(nodes: AXNode[]): NthIndex { - const cohorts = new Map(); - const index = new Map(); - for (const node of nodes) { - if (node.ignored || node.backendDOMNodeId === undefined) continue; - const key = cohortKey(node.role?.value ?? "", node.name?.value ?? ""); - const nth = cohorts.get(key) ?? 0; - cohorts.set(key, nth + 1); - index.set(node.nodeId, nth); - } - return { index, cohorts }; -} - -function cohortKey(role: string, name: string): string { - return `${role}\u0000${name}`; -} - -/** Merge a run of two or more consecutive StaticText siblings (text split by inline markup) into one node. */ -function staticTextRun(tree: Map, childIds: string[], start: number): { node: AXNode; end: number } | undefined { - let end = start; - const parts: string[] = []; - while (end < childIds.length) { - const node = tree.get(childIds[end]!); - if (!node || node.ignored || node.role?.value !== "StaticText") break; - const text = node.name?.value ?? ""; - if (text) parts.push(text); - end += 1; - } - if (end - start < 2) return undefined; - const first = tree.get(childIds[start]!)!; - return { node: { ...first, name: { value: parts.join(" ") }, childIds: [] }, end: end - 1 }; -} - -function collectStates(node: AXNode): string[] { - const states: string[] = []; - for (const property of node.properties ?? []) { - const value = property.value?.value; - switch (property.name) { - case "checked": - case "pressed": - case "expanded": - // False is meaningful here: it distinguishes an unchecked checkbox or - // collapsed disclosure from an element without the state at all. - if (value === true || value === "true") states.push(property.name); - else if (value === false || value === "false") states.push(`${property.name}=false`); - else if (value === "mixed") states.push(`${property.name}=mixed`); - break; - case "disabled": - case "selected": - case "required": - if (value === true || value === "true") states.push(property.name); - break; - case "level": - if (typeof value === "number") states.push(`level=${value}`); - break; - } - } - const value = node.value?.value; - if (value !== undefined && value !== "" && String(value) !== (node.name?.value ?? "")) { - states.push(`value=${JSON.stringify(String(value))}`); - } - return states; -} - function shortTabId(targetId: string): string { return targetId.slice(0, 10).toUpperCase(); } @@ -1113,51 +1083,6 @@ const FILL_FUNCTION = `function(value) { el.dispatchEvent(new Event("change", { bubbles: true })); }`; -const INTERACTIVE_ROLES: ReadonlySet = new Set([ - "button", - "link", - "textbox", - "searchbox", - "checkbox", - "radio", - "combobox", - "listbox", - "option", - "menuitem", - "menuitemcheckbox", - "menuitemradio", - "slider", - "spinbutton", - "switch", - "tab", - "treeitem", -]); - -const SKIPPED_ROLES: ReadonlySet = new Set(["none", "generic", "InlineTextBox", "LineBreak", "StaticText"]); - -const FRAME_ROLES: ReadonlySet = new Set(["Iframe", "IframePresentational"]); - -/** Non-interactive roles that get refs when named, so scroll_to / ref-scoped snapshots can target them. */ -const CONTENT_ROLES: ReadonlySet = new Set([ - "heading", - "cell", - "gridcell", - "columnheader", - "rowheader", - "row", - "listitem", - "article", - "region", - "main", - "navigation", - "banner", - "contentinfo", - "complementary", - "tabpanel", - "figure", - "image", -]); - const CURSOR_SCAN_GROUP = "cua-cursor-scan"; const CURSOR_POINTER_SCAN = `(() => { diff --git a/packages/agent/src/translator/cdp.ts b/packages/agent/src/translator/cdp.ts index c38604a..7408f0f 100644 --- a/packages/agent/src/translator/cdp.ts +++ b/packages/agent/src/translator/cdp.ts @@ -8,6 +8,7 @@ */ interface PendingCommand { + readonly method: string; resolve(result: unknown): void; reject(error: Error): void; } @@ -28,6 +29,19 @@ export interface CdpTargetInfo { export type CdpEventListener = (event: CdpEventMessage) => void; +/** A command error returned by the CDP JSON-RPC endpoint. */ +export class CdpProtocolError extends Error { + constructor( + readonly method: string, + readonly code: number, + readonly protocolMessage: string, + readonly data?: string, + ) { + super(protocolMessage); + this.name = "CdpProtocolError"; + } +} + export class CdpConnection { private socket?: WebSocket; private opening?: Promise; @@ -47,7 +61,7 @@ export class CdpConnection { const id = this.nextId++; const message = JSON.stringify({ id, method, params, ...(sessionId ? { sessionId } : {}) }); return new Promise((resolve, reject) => { - this.pending.set(id, { resolve: resolve as (result: unknown) => void, reject }); + this.pending.set(id, { method, resolve: resolve as (result: unknown) => void, reject }); // A non-OPEN socket silently discards send() per the WebSocket spec, // which would leave this command pending forever. if (socket.readyState !== WebSocket.OPEN) { @@ -123,7 +137,14 @@ export class CdpConnection { } private handleMessage(data: string): void { - let message: { id?: number; result?: unknown; error?: { message?: string }; method?: string; params?: Record; sessionId?: string }; + let message: { + id?: number; + result?: unknown; + error?: { code?: number; message?: string; data?: string }; + method?: string; + params?: Record; + sessionId?: string; + }; try { message = JSON.parse(data); } catch { @@ -133,8 +154,16 @@ export class CdpConnection { const pending = this.pending.get(message.id); if (!pending) return; this.pending.delete(message.id); - if (message.error) pending.reject(new Error(message.error.message ?? "CDP command failed")); - else pending.resolve(message.result ?? {}); + if (message.error) { + pending.reject( + new CdpProtocolError( + pending.method, + message.error.code ?? -1, + message.error.message ?? "CDP command failed", + message.error.data, + ), + ); + } else pending.resolve(message.result ?? {}); return; } if (typeof message.method === "string") { diff --git a/packages/agent/test/cdp.test.ts b/packages/agent/test/cdp.test.ts index 6fe84fd..8bc1edf 100644 --- a/packages/agent/test/cdp.test.ts +++ b/packages/agent/test/cdp.test.ts @@ -1,5 +1,5 @@ import { afterEach, describe, expect, it, vi } from "vitest"; -import { CdpConnection } from "../src/translator/cdp"; +import { CdpConnection, CdpProtocolError } from "../src/translator/cdp"; class FakeSocket { static instances: FakeSocket[] = []; @@ -66,4 +66,23 @@ describe("CdpConnection send", () => { FakeSocket.instances[0]!.close(); await expect(pending).rejects.toThrow(/closed/); }); + + it("preserves command and protocol error metadata", async () => { + vi.stubGlobal("WebSocket", Object.assign(FakeSocket, { OPEN: 1 })); + const cdp = new CdpConnection("wss://fake.test/cdp"); + const pending = cdp.send("Accessibility.getFullAXTree", { frameId: "F1" }, "session-1"); + await vi.waitFor(() => expect(FakeSocket.instances[0]!.sent).toHaveLength(1)); + FakeSocket.instances[0]!.emit("message", { + data: JSON.stringify({ id: 1, error: { code: -32000, message: "Frame with the given id was not found.", data: "gone" } }), + }); + + const error = await pending.catch((cause: unknown) => cause); + expect(error).toBeInstanceOf(CdpProtocolError); + expect(error).toMatchObject({ + method: "Accessibility.getFullAXTree", + code: -32000, + protocolMessage: "Frame with the given id was not found.", + data: "gone", + }); + }); }); diff --git a/packages/agent/test/translator-browser.test.ts b/packages/agent/test/translator-browser.test.ts index b345b62..eb74f4e 100644 --- a/packages/agent/test/translator-browser.test.ts +++ b/packages/agent/test/translator-browser.test.ts @@ -3,7 +3,7 @@ import sharp from "sharp"; import { describe, expect, it } from "vitest"; import type { CuaBrowserAction } from "@onkernel/cua-ai"; import { BrowserExecutor } from "../src/translator/browser"; -import type { CdpConnection } from "../src/translator/cdp"; +import { CdpProtocolError, type CdpConnection } from "../src/translator/cdp"; import { buildCuaComputerTools } from "../src/tools"; import { InternalComputerTranslator, type KernelBrowser } from "../src/translator/translator"; import type { BatchReadResult } from "../src/translator/types"; @@ -110,7 +110,12 @@ function createFakeCdp(initialNodes: unknown[] = []) { const frameTrees = new Map>(); const iframeFrameIds = new Map(); const autoAttachFrames: Array<{ targetId: string; sessionId: string }> = []; - const failMethods = new Set(); + const methodFailures = new Map(); + let failureProvider: ((method: string, params: Record, sessionId?: string) => Error | undefined) | undefined; + let axRead = 0; + let targetRead = 0; + let onAxRead: ((read: number, params: Record, sessionId?: string) => void) | undefined; + let targetProvider: ((read: number) => Array<{ targetId: string; type: string; title: string; url: string }>) | undefined; const emit = (event: FakeCdpEvent) => { for (const listener of listeners) listener(event); }; @@ -129,10 +134,14 @@ function createFakeCdp(initialNodes: unknown[] = []) { }, send: async (method: string, params: Record = {}, sessionId?: string) => { sent.push({ method, params, sessionId }); - if (failMethods.has(method)) throw new Error(`${method} rejected`); + const failure = methodFailures.get(method) ?? failureProvider?.(method, params, sessionId); + if (failure) throw failure; switch (method) { - case "Accessibility.getFullAXTree": - return { nodes: treeFor(sessionId, params.frameId) }; + case "Accessibility.getFullAXTree": { + const tree = treeFor(sessionId, params.frameId); + onAxRead?.(++axRead, params, sessionId); + return { nodes: tree }; + } case "Target.setAutoAttach": for (const frame of autoAttachFrames.splice(0)) { emit({ @@ -170,7 +179,8 @@ function createFakeCdp(initialNodes: unknown[] = []) { return {}; } }, - pageTargets: async () => [{ targetId: "TARGET-1", type: "page", title: "Page", url: "https://a.test/" }], + pageTargets: async () => + targetProvider?.(++targetRead) ?? [{ targetId: "TARGET-1", type: "page", title: "Page", url: "https://a.test/" }], attachToTarget: async () => "session-1", createTarget: async () => "TARGET-2", close: () => {}, @@ -193,8 +203,17 @@ function createFakeCdp(initialNodes: unknown[] = []) { const addAutoAttachFrame = (frame: { targetId: string; sessionId: string }) => { autoAttachFrames.push(frame); }; - const failOn = (method: string) => { - failMethods.add(method); + const failOn = (method: string, error = new Error(`${method} rejected`)) => { + methodFailures.set(method, error); + }; + const setFailureProvider = (provider: typeof failureProvider) => { + failureProvider = provider; + }; + const setAxReadHook = (hook: typeof onAxRead) => { + onAxRead = hook; + }; + const setTargetProvider = (provider: typeof targetProvider) => { + targetProvider = provider; }; return { sent, @@ -206,6 +225,9 @@ function createFakeCdp(initialNodes: unknown[] = []) { setIframeFrame, addAutoAttachFrame, failOn, + setFailureProvider, + setAxReadHook, + setTargetProvider, cdp: fake as unknown as CdpConnection, }; } @@ -587,6 +609,15 @@ describe("BrowserExecutor cursor-pointer hints", () => { expect(sent.some((cmd) => cmd.method === "DOM.describeNode")).toBe(true); expect(sent.some((cmd) => cmd.method === "Runtime.releaseObjectGroup")).toBe(true); }); + + it("does not scan cursor metadata for find", async () => { + const { cdp, failOn, sent } = createFakeCdp(BUTTON_TREE); + failOn("Runtime.evaluate"); + const executor = new BrowserExecutor(cdp); + const results = await executor.execute({ type: "browser_find", query: "save button" } as CuaBrowserAction); + expect((results[0] as { text: string }).text).toContain('button "Save" [e1]'); + expect(sent.some((command) => command.method === "Runtime.evaluate")).toBe(false); + }); }); describe("BrowserExecutor dialog guard", () => { @@ -736,6 +767,236 @@ describe("BrowserExecutor iframe stitching", () => { const text = await snapshotText(executor); expect(text).toContain('button "Pay" [e'); }); + + it("invalidates and releases a same-process frame when it detaches and rotates", async () => { + const root = [ + ax({ nodeId: "1", role: "RootWebArea", name: "Page", childIds: ["2"] }), + ax({ nodeId: "2", role: "Iframe", backendDOMNodeId: 50, parentId: "1" }), + ]; + const fake = createFakeCdp(root); + fake.setIframeFrame(50, "FRAME-SP"); + fake.setFrameTree("FRAME-SP", [ax({ nodeId: "old", role: "button", name: "Old", backendDOMNodeId: 60 })]); + const executor = new BrowserExecutor(fake.cdp); + await snapshotText(executor); + + fake.emit({ method: "Page.frameDetached", params: { frameId: "FRAME-SP", reason: "swap" }, sessionId: "session-1" }); + await expect(executor.execute({ type: "browser_click", ref: "e2" } as CuaBrowserAction)).rejects.toThrow(/stale/); + + fake.setFrameTree("FRAME-SP", [ax({ nodeId: "new", role: "button", name: "New", backendDOMNodeId: 61 })]); + expect(await snapshotText(executor)).toContain('button "New" [e'); + fake.emit({ method: "Page.frameDetached", params: { frameId: "FRAME-SP", reason: "remove" }, sessionId: "session-1" }); + expect(executor.exportRefState().generations.map(([key]) => key)).toEqual(["TARGET-1"]); + }); + + it("does not retain generation state for rotating unreferenced frames", async () => { + const root = [ + ax({ nodeId: "1", role: "RootWebArea", name: "Page", childIds: ["2"] }), + ax({ nodeId: "2", role: "Iframe", backendDOMNodeId: 50, parentId: "1" }), + ]; + const fake = createFakeCdp(root); + const executor = new BrowserExecutor(fake.cdp); + for (let index = 0; index < 25; index += 1) { + const frameId = `FRAME-${index}`; + fake.setIframeFrame(50, frameId); + fake.setFrameTree(frameId, [ax({ nodeId: `root-${index}`, role: "RootWebArea", name: "Embed" })]); + await snapshotText(executor); + } + expect(executor.exportRefState().generations).toEqual([["TARGET-1", 0]]); + }); + + it("does not retain a frame generation when its AX fetch is transiently inaccessible", async () => { + const root = [ + ax({ nodeId: "1", role: "RootWebArea", name: "Page", childIds: ["2"] }), + ax({ nodeId: "2", role: "Iframe", backendDOMNodeId: 50, parentId: "1" }), + ]; + const fake = createFakeCdp(root); + fake.setIframeFrame(50, "FRAME-SP"); + fake.setFailureProvider((method, params) => + method === "Accessibility.getFullAXTree" && params.frameId === "FRAME-SP" + ? new CdpProtocolError(method, -32000, "Frame with the given id was not found.") + : undefined, + ); + const executor = new BrowserExecutor(fake.cdp); + expect(await snapshotText(executor)).toContain("Iframe [e1]"); + expect(executor.exportRefState().generations).toEqual([["TARGET-1", 0]]); + }); + + it("propagates unexpected frame protocol failures with frame diagnostics and no registration leak", async () => { + const root = [ + ax({ nodeId: "1", role: "RootWebArea", name: "Page", childIds: ["2"] }), + ax({ nodeId: "2", role: "Iframe", backendDOMNodeId: 50, parentId: "1" }), + ]; + const fake = createFakeCdp(root); + fake.setIframeFrame(50, "FRAME-SP"); + fake.setFailureProvider((method, params) => + method === "Accessibility.getFullAXTree" && params.frameId === "FRAME-SP" ? new Error("decoder exploded") : undefined, + ); + const executor = new BrowserExecutor(fake.cdp); + await expect(snapshotText(executor)).rejects.toThrow( + /Failed to collect iframe FRAME-SP at backend node 50 during Accessibility\.getFullAXTree: decoder exploded/, + ); + expect(executor.exportRefState().generations).toEqual([["TARGET-1", 0]]); + }); + + it("propagates malformed frame trees as indexed collection failures", async () => { + const root = [ + ax({ nodeId: "1", role: "RootWebArea", name: "Page", childIds: ["2"] }), + ax({ nodeId: "2", role: "Iframe", backendDOMNodeId: 50, parentId: "1" }), + ]; + const fake = createFakeCdp(root); + fake.setIframeFrame(50, "FRAME-SP"); + fake.setFrameTree("FRAME-SP", null as unknown as unknown[]); + const executor = new BrowserExecutor(fake.cdp); + await expect(snapshotText(executor)).rejects.toThrow(/FRAME-SP.*building the accessibility index/); + expect(executor.exportRefState().generations).toEqual([["TARGET-1", 0]]); + }); + + it("propagates unexpected iframe description failures instead of treating them as inaccessible", async () => { + const root = [ + ax({ nodeId: "1", role: "RootWebArea", name: "Page", childIds: ["2"] }), + ax({ nodeId: "2", role: "Iframe", backendDOMNodeId: 50, parentId: "1" }), + ]; + const fake = createFakeCdp(root); + fake.failOn("DOM.describeNode", new CdpProtocolError("DOM.describeNode", -32603, "Internal error")); + const executor = new BrowserExecutor(fake.cdp); + await expect(snapshotText(executor)).rejects.toThrow(/backend node 50 during DOM\.describeNode.*Internal error/); + expect(fake.sent.filter((command) => command.method === "DOM.describeNode")).toHaveLength(1); + }); + + it("retries a transiently omitted owning frame instead of staling its scoped ref", async () => { + const root = [ + ax({ nodeId: "1", role: "RootWebArea", name: "Page", childIds: ["2", "3"] }), + ax({ nodeId: "2", role: "button", name: "Pay", backendDOMNodeId: 40, parentId: "1" }), + ax({ nodeId: "3", role: "Iframe", backendDOMNodeId: 50, parentId: "1" }), + ]; + const fake = createFakeCdp(root); + fake.setIframeFrame(50, "FRAME-SP"); + fake.setFrameTree("FRAME-SP", [ax({ nodeId: "f1", role: "button", name: "Pay", backendDOMNodeId: 70 })]); + const executor = new BrowserExecutor(fake.cdp); + await snapshotText(executor); + + let omitted = false; + fake.setFailureProvider((method) => { + if (method !== "DOM.describeNode" || omitted) return undefined; + omitted = true; + return new CdpProtocolError(method, -32000, "Could not find node with given id"); + }); + await expect(snapshotText(executor, { ref: "e3" })).resolves.toContain('button "Pay"'); + expect(fake.sent.filter((command) => command.method === "DOM.describeNode")).toHaveLength(3); + }); + + it("reports a persistently omitted owning frame as unverifiable rather than stale", async () => { + const root = [ + ax({ nodeId: "1", role: "RootWebArea", name: "Page", childIds: ["2"] }), + ax({ nodeId: "2", role: "Iframe", backendDOMNodeId: 50, parentId: "1" }), + ]; + const fake = createFakeCdp(root); + fake.setIframeFrame(50, "FRAME-SP"); + fake.setFrameTree("FRAME-SP", [ax({ nodeId: "f1", role: "button", name: "Pay", backendDOMNodeId: 70 })]); + const executor = new BrowserExecutor(fake.cdp); + await snapshotText(executor); + + fake.failOn("DOM.describeNode", new CdpProtocolError("DOM.describeNode", -32000, "Could not find node with given id")); + await expect(snapshotText(executor, { ref: "e2" })).rejects.toThrow(/could not verify ref e2/i); + expect(fake.sent.filter((command) => command.method === "DOM.describeNode")).toHaveLength(4); + }); +}); + +describe("BrowserExecutor observation fencing", () => { + const STALE_TREE = [ + ax({ nodeId: "1", role: "RootWebArea", name: "Old", childIds: ["2"] }), + ax({ nodeId: "2", role: "button", name: "Stale", backendDOMNodeId: 41, parentId: "1" }), + ]; + const STABLE_TREE = [ + ax({ nodeId: "1", role: "RootWebArea", name: "New", childIds: ["2"] }), + ax({ nodeId: "2", role: "button", name: "Stable", backendDOMNodeId: 42, parentId: "1" }), + ]; + + it.each(["browser_snapshot", "browser_find"] as const)("discards a navigated AX collection for %s before minting refs", async (type) => { + const fake = createFakeCdp(STALE_TREE); + fake.setAxReadHook((read, params) => { + if (read !== 1 || params.frameId) return; + fake.setNodes(STABLE_TREE); + fake.emit({ method: "Page.frameNavigated", params: { frame: { id: "TARGET-1" } }, sessionId: "session-1" }); + }); + const executor = new BrowserExecutor(fake.cdp); + const results = await executor.execute( + (type === "browser_snapshot" ? { type } : { type, query: "stable button" }) as CuaBrowserAction, + ); + const text = (results[0] as { text: string }).text; + expect(text).toContain('button "Stable" [e1]'); + expect(text).not.toContain("Stale"); + expect([...refsOf(executor).keys()]).toEqual(["e1"]); + }); + + it.each([ + ["url", { title: "Page", url: "https://b.test/" }], + ["title", { title: "Changed", url: "https://a.test/" }], + ] as const)("retries when target %s changes across collection", async (_field, changed) => { + const fake = createFakeCdp(BUTTON_TREE); + fake.setTargetProvider((read) => [ + { targetId: "TARGET-1", type: "page", ...(read < 3 ? { title: "Page", url: "https://a.test/" } : changed) }, + ]); + const executor = new BrowserExecutor(fake.cdp); + expect(await snapshotText(executor)).toContain('button "Save" [e1]'); + expect(fake.sent.filter((command) => command.method === "Accessibility.getFullAXTree")).toHaveLength(2); + }); + + it("retries when a stitched frame changes during its AX read", async () => { + const root = [ + ax({ nodeId: "1", role: "RootWebArea", name: "Page", childIds: ["2"] }), + ax({ nodeId: "2", role: "Iframe", backendDOMNodeId: 50, parentId: "1" }), + ]; + const oldChild = [ax({ nodeId: "f1", role: "button", name: "Old child", backendDOMNodeId: 60 })]; + const newChild = [ax({ nodeId: "f1", role: "button", name: "New child", backendDOMNodeId: 61 })]; + const fake = createFakeCdp(root); + fake.setIframeFrame(50, "FRAME-SP"); + fake.setFrameTree("FRAME-SP", oldChild); + fake.setAxReadHook((_read, params) => { + if (params.frameId !== "FRAME-SP") return; + fake.setAxReadHook(undefined); + fake.setFrameTree("FRAME-SP", newChild); + fake.emit({ method: "Page.frameNavigated", params: { frame: { id: "FRAME-SP", parentId: "TARGET-1" } }, sessionId: "session-1" }); + }); + const executor = new BrowserExecutor(fake.cdp); + const text = await snapshotText(executor); + expect(text).toContain('button "New child"'); + expect(text).not.toContain("Old child"); + }); + + it("retries when a generation-zero OOPIF detaches after its AX read", async () => { + const root = [ + ax({ nodeId: "1", role: "RootWebArea", name: "Page", childIds: ["2"] }), + ax({ nodeId: "2", role: "Iframe", backendDOMNodeId: 50, parentId: "1" }), + ]; + const fake = createFakeCdp(root); + fake.setIframeFrame(50, "FRAME-OOP"); + fake.addAutoAttachFrame({ targetId: "FRAME-OOP", sessionId: "session-oop" }); + fake.setSessionTree("session-oop", [ax({ nodeId: "f1", role: "button", name: "Detached", backendDOMNodeId: 60 })]); + fake.setFrameTree("FRAME-OOP", [ax({ nodeId: "f1", role: "button", name: "Current", backendDOMNodeId: 61 })]); + fake.setAxReadHook((_read, _params, sessionId) => { + if (sessionId !== "session-oop") return; + fake.setAxReadHook(undefined); + fake.emit({ method: "Target.detachedFromTarget", params: { sessionId: "session-oop" } }); + }); + const executor = new BrowserExecutor(fake.cdp); + const text = await snapshotText(executor); + expect(text).toContain('button "Current"'); + expect(text).not.toContain("Detached"); + }); + + it.each(["browser_snapshot", "browser_find"] as const)("fails %s after three changed collections without minting refs", async (type) => { + const fake = createFakeCdp(BUTTON_TREE); + fake.setAxReadHook((_read, params) => { + if (!params.frameId) fake.emit({ method: "Page.frameNavigated", params: { frame: { id: "TARGET-1" } }, sessionId: "session-1" }); + }); + const executor = new BrowserExecutor(fake.cdp); + await expect( + executor.execute((type === "browser_snapshot" ? { type } : { type, query: "save" }) as CuaBrowserAction), + ).rejects.toThrow(/observation changed/i); + expect(fake.sent.filter((command) => command.method === "Accessibility.getFullAXTree")).toHaveLength(3); + expect(refsOf(executor).size).toBe(0); + }); }); describe("navigation tool grounding frame", () => { @@ -814,6 +1075,27 @@ describe("BrowserExecutor ref state export/import", () => { expect(pressed).toBeDefined(); }); + it("restores a same-process frame ref with its owning generation", async () => { + const root = [ + ax({ nodeId: "1", role: "RootWebArea", name: "Page", childIds: ["2"] }), + ax({ nodeId: "2", role: "Iframe", backendDOMNodeId: 50, parentId: "1" }), + ]; + const firstFake = createFakeCdp(root); + firstFake.setIframeFrame(50, "FRAME-SP"); + firstFake.setFrameTree("FRAME-SP", [ax({ nodeId: "f1", role: "button", name: "Pay", backendDOMNodeId: 70 })]); + const first = new BrowserExecutor(firstFake.cdp); + await snapshotText(first); + const state = first.exportRefState(); + + const secondFake = createFakeCdp(root); + secondFake.setIframeFrame(50, "FRAME-SP"); + secondFake.setFrameTree("FRAME-SP", [ax({ nodeId: "f1", role: "button", name: "Pay", backendDOMNodeId: 70 })]); + const second = new BrowserExecutor(secondFake.cdp); + second.importRefState(state); + await second.execute({ type: "browser_click", ref: "e2" } as CuaBrowserAction); + expect(secondFake.sent.some((command) => command.method === "DOM.getBoxModel" && command.params.backendNodeId === 70)).toBe(true); + }); + it("keeps minting unique refs after import and invalidates imported refs on navigation", async () => { const first = new BrowserExecutor(createFakeCdp(BUTTON_TREE).cdp); await snapshotText(first);