From 67e705650036b408cdab37efb61128275c53223a Mon Sep 17 00:00:00 2001 From: quanruzhuoxiu Date: Mon, 20 Jul 2026 08:10:05 +0000 Subject: [PATCH 1/3] test(computer): synchronize macOS fixture input readiness --- .../ai/fixtures/macos-desktop-smoke-app.swift | 62 ++++++++++++++----- .../tests/ai/macos-desktop-smoke.test.ts | 17 +++-- 2 files changed, 57 insertions(+), 22 deletions(-) diff --git a/packages/computer/tests/ai/fixtures/macos-desktop-smoke-app.swift b/packages/computer/tests/ai/fixtures/macos-desktop-smoke-app.swift index cbacfb3a67..ba07cc9ac6 100644 --- a/packages/computer/tests/ai/fixtures/macos-desktop-smoke-app.swift +++ b/packages/computer/tests/ai/fixtures/macos-desktop-smoke-app.swift @@ -10,6 +10,15 @@ final class FlippedDocumentView: NSView { final class SmokeButton: NSButton { var onMouseDown: (() -> Void)? + // AppKit normally consumes the first click while a programmatically + // activated application is still transitioning to the foreground. The + // hosted macOS runner can remain in that transition even after System + // Events says the process is frontmost. Accepting the activation click + // keeps this fixture focused on whether the global mouse event arrived. + override func acceptsFirstMouse(for event: NSEvent?) -> Bool { + true + } + override func mouseDown(with event: NSEvent) { onMouseDown?() super.mouseDown(with: event) @@ -28,6 +37,7 @@ final class FixtureController: NSObject, NSApplicationDelegate, NSTextFieldDeleg private var activationSource: DispatchSourceSignal? private var activationCount = 0 + private var inputReadyGeneration = 0 private var clickCount = 0 private var buttonActionCount = 0 private var lastKey = "" @@ -108,12 +118,7 @@ final class FixtureController: NSObject, NSApplicationDelegate, NSTextFieldDeleg installActivationSignal() activateFixture() - window.makeFirstResponder(textField) writeState() - - DispatchQueue.main.asyncAfter(deadline: .now() + 0.5) { [weak self] in - self?.writeReadyMetadata() - } } func applicationShouldTerminateAfterLastWindowClosed(_ sender: NSApplication) -> Bool { @@ -140,22 +145,46 @@ final class FixtureController: NSObject, NSApplicationDelegate, NSTextFieldDeleg } private func activateFixture() { - focusFixture() activationCount += 1 + let activation = activationCount + NSApplication.shared.unhide(nil) + NSApplication.shared.activate(ignoringOtherApps: true) + NSRunningApplication.current.activate(options: [.activateAllWindows]) writeState() - DispatchQueue.main.asyncAfter(deadline: .now() + 0.15) { [weak self] in - self?.focusFixture() - self?.writeState() + + // Present the window on the next AppKit turn, after the activation request + // has reached the application. A readiness generation is published only + // after AppKit itself reports a visible key window; callers synchronize on + // that generation instead of sleeping for an arbitrary duration. + DispatchQueue.main.async { [weak self] in + guard let self else { return } + self.window.orderFrontRegardless() + self.window.makeKeyAndOrderFront(nil) + self.window.makeFirstResponder(self.textField) + self.publishInputReady(for: activation, attemptsRemaining: 40) } } - private func focusFixture() { - NSApplication.shared.unhide(nil) - window.orderFrontRegardless() - window.makeKeyAndOrderFront(nil) - NSApplication.shared.activate(ignoringOtherApps: true) - NSRunningApplication.current.activate(options: [.activateAllWindows]) - window.makeFirstResponder(textField) + private func publishInputReady(for activation: Int, attemptsRemaining: Int) { + guard activation == activationCount else { return } + if window.isVisible && window.isKeyWindow && NSApplication.shared.isActive { + inputReadyGeneration += 1 + writeState() + if inputReadyGeneration == 1 { + writeReadyMetadata() + } + return + } + guard attemptsRemaining > 0 else { + writeState() + return + } + DispatchQueue.main.asyncAfter(deadline: .now() + 0.05) { [weak self] in + self?.publishInputReady( + for: activation, + attemptsRemaining: attemptsRemaining - 1 + ) + } } private func installActivationSignal() { @@ -255,6 +284,7 @@ final class FixtureController: NSObject, NSApplicationDelegate, NSTextFieldDeleg "active": NSApplication.shared.isActive, "keyWindow": window.isKeyWindow, "activationCount": activationCount, + "inputReadyGeneration": inputReadyGeneration, "clickCount": clickCount, "buttonActionCount": buttonActionCount, "text": textField.stringValue, diff --git a/packages/computer/tests/ai/macos-desktop-smoke.test.ts b/packages/computer/tests/ai/macos-desktop-smoke.test.ts index b8528935a2..af8866c364 100644 --- a/packages/computer/tests/ai/macos-desktop-smoke.test.ts +++ b/packages/computer/tests/ai/macos-desktop-smoke.test.ts @@ -70,6 +70,7 @@ interface FixtureState { active: boolean; keyWindow: boolean; activationCount: number; + inputReadyGeneration: number; clickCount: number; buttonActionCount: number; text: string; @@ -161,6 +162,10 @@ function normalizeState(value: unknown): FixtureState { raw.activationCount, 'state.activationCount', ), + inputReadyGeneration: asFiniteNumber( + raw.inputReadyGeneration, + 'state.inputReadyGeneration', + ), clickCount: asFiniteNumber(raw.clickCount, 'state.clickCount'), buttonActionCount: asFiniteNumber( raw.buttonActionCount, @@ -261,15 +266,15 @@ async function retryFixtureAction(options: { await waitForJson( options.stateFile, normalizeState, - // GitHub's hosted macOS session reports the fixture as frontmost to - // System Events while AppKit still reports inactive/non-key. The - // fixture's activation signal is the reliable synchronization point; - // each caller subsequently verifies that the real input was received. - (state) => state.activationCount > beforeActivation.activationCount, + (state) => + state.activationCount > beforeActivation.activationCount && + state.inputReadyGeneration > beforeActivation.inputReadyGeneration && + state.visible && + state.active && + state.keyWindow, ACTIVATION_TIMEOUT_MS, options.fixtureProcess, ); - await sleep(250); await options.action(); const state = await waitForJson( options.stateFile, From 3a00c0415af949b7172c8f59ebdf4a9dea719ecb Mon Sep 17 00:00:00 2001 From: quanruzhuoxiu Date: Mon, 20 Jul 2026 08:10:05 +0000 Subject: [PATCH 2/3] test(computer): accept macOS activation clicks in text field --- .../tests/ai/fixtures/macos-desktop-smoke-app.swift | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/packages/computer/tests/ai/fixtures/macos-desktop-smoke-app.swift b/packages/computer/tests/ai/fixtures/macos-desktop-smoke-app.swift index ba07cc9ac6..be6bfbe419 100644 --- a/packages/computer/tests/ai/fixtures/macos-desktop-smoke-app.swift +++ b/packages/computer/tests/ai/fixtures/macos-desktop-smoke-app.swift @@ -25,6 +25,13 @@ final class SmokeButton: NSButton { } } +@MainActor +final class SmokeTextField: NSTextField { + override func acceptsFirstMouse(for event: NSEvent?) -> Bool { + true + } +} + @MainActor final class FixtureController: NSObject, NSApplicationDelegate, NSTextFieldDelegate { private let readyURL: URL @@ -32,7 +39,7 @@ final class FixtureController: NSObject, NSApplicationDelegate, NSTextFieldDeleg private var window: NSWindow! private var button: SmokeButton! - private var textField: NSTextField! + private var textField: SmokeTextField! private var scrollView: NSScrollView! private var activationSource: DispatchSourceSignal? @@ -90,7 +97,7 @@ final class FixtureController: NSObject, NSApplicationDelegate, NSTextFieldDeleg self.writeState() } - textField = NSTextField(frame: NSRect(x: 120, y: 275, width: 400, height: 44)) + textField = SmokeTextField(frame: NSRect(x: 120, y: 275, width: 400, height: 44)) textField.placeholderString = "Type smoke text" textField.delegate = self textField.target = self From 883e26908fdeb1680be579e2df84d6a2b5d9f6eb Mon Sep 17 00:00:00 2001 From: quanruzhuoxiu Date: Mon, 20 Jul 2026 08:10:05 +0000 Subject: [PATCH 3/3] fix(computer): harden macOS keyboard target focus --- packages/computer/src/device.ts | 41 ++++++++++--------- .../tests/unit-test/device-security.test.ts | 21 ++++++++++ 2 files changed, 43 insertions(+), 19 deletions(-) diff --git a/packages/computer/src/device.ts b/packages/computer/src/device.ts index 24442b77e5..39908c2a4d 100644 --- a/packages/computer/src/device.ts +++ b/packages/computer/src/device.ts @@ -828,14 +828,7 @@ export class ComputerDevice implements AbstractInterface { const element = opts?.target as LocateResultElement | undefined; if (element) { - const [x, y] = element.center; - const target = this.toGlobalPoint({ x, y }); - this.inputDriver.moveMouse( - Math.round(target.x), - Math.round(target.y), - ); - this.inputDriver.mouseClick('left'); - await this.inputDriver.delay(INPUT_FOCUS_DELAY); + await this.focusKeyboardTarget(element, INPUT_FOCUS_DELAY); if (opts?.replace !== false) { await this.selectAllAndDelete(); @@ -848,23 +841,14 @@ export class ComputerDevice implements AbstractInterface { keyboardPress: async (keyName, opts) => { const target = opts?.target as LocateResultElement | undefined; if (target) { - const [x, y] = target.center; - const point = this.toGlobalPoint({ x, y }); - this.inputDriver.moveMouse(Math.round(point.x), Math.round(point.y)); - this.inputDriver.mouseClick('left'); - await this.inputDriver.delay(50); + await this.focusKeyboardTarget(target, 50); } await this.pressKeyboardShortcut(keyName); }, clearInput: async (target) => { if (target) { - const element = target as LocateResultElement; - const [x, y] = element.center; - const point = this.toGlobalPoint({ x, y }); - this.inputDriver.moveMouse(Math.round(point.x), Math.round(point.y)); - this.inputDriver.mouseClick('left'); - await this.inputDriver.delay(100); + await this.focusKeyboardTarget(target as LocateResultElement, 100); } await this.selectAllAndDelete(); @@ -885,6 +869,25 @@ export class ComputerDevice implements AbstractInterface { process.platform === 'darwin' && options?.keyboardDriver !== 'libnut'; } + private async focusKeyboardTarget( + element: LocateResultElement, + delayMs: number, + ): Promise { + const [x, y] = element.center; + if (process.platform === 'darwin') { + // Reuse the macOS tap path, which holds mouse-down long enough for + // AppKit and follows up when the click changes the foreground process. + // A bare libnut mouseClick can be consumed solely as an activation click + // on hosted desktops, leaving subsequent AppleScript typing unfocused. + await this.inputPrimitives.pointer.tap({ x, y }); + } else { + const point = this.toGlobalPoint({ x, y }); + this.inputDriver.moveMouse(Math.round(point.x), Math.round(point.y)); + this.inputDriver.mouseClick('left'); + } + await this.inputDriver.delay(delayMs); + } + describe(): string { return this.description || 'Computer Device'; } diff --git a/packages/computer/tests/unit-test/device-security.test.ts b/packages/computer/tests/unit-test/device-security.test.ts index c9c722269f..fb49c7b133 100644 --- a/packages/computer/tests/unit-test/device-security.test.ts +++ b/packages/computer/tests/unit-test/device-security.test.ts @@ -403,4 +403,25 @@ describe('ComputerDevice pointer input', () => { 'left', ); }); + + it('uses the held tap path when focusing a macOS keyboard target', async () => { + const device = await createConnectedDevice(); + + await device.inputPrimitives.keyboard.keyboardPress('Enter', { + target: { center: [100, 120] }, + }); + + expect(mockState.libnut.mouseClick).not.toHaveBeenCalled(); + expect(mockState.libnut.mouseToggle).toHaveBeenCalledTimes(2); + expect(mockState.libnut.mouseToggle).toHaveBeenNthCalledWith( + 1, + 'down', + 'left', + ); + expect(mockState.libnut.mouseToggle).toHaveBeenNthCalledWith( + 2, + 'up', + 'left', + ); + }); });