Merge pull request #260 from askui/fix/SOLENG-332-scroll-scaling

fix: remove incorrect coordinate scaling from mouse scroll deltas (SO…

Author: programminx-askui

src/askui/computer_agent.py

@@ -231,15 +231,15 @@ def mouse_move(
     @validate_call
     def mouse_scroll(
         self,
-        x: int,
-        y: int,
+        dx: int,
+        dy: int,
     ) -> None:
         """
         Simulates scrolling the mouse wheel by the specified horizontal and vertical amounts.
 
         Args:
-            x (int): The horizontal scroll amount. Positive values typically scroll right, negative values scroll left.
-            y (int): The vertical scroll amount. Positive values typically scroll down, negative values scroll up.
+            dx (int): The horizontal scroll amount. Positive values typically scroll right, negative values scroll left.
+            dy (int): The vertical scroll amount. Positive values typically scroll down, negative values scroll up.
 
         Note:
             The actual scroll direction depends on the operating system's configuration.
@@ -258,8 +258,8 @@ def mouse_scroll(
                 agent.mouse_scroll(3, 0)   # Usually scrolls right 3 units
             ```
         """
-        self._reporter.add_message("User", f'mouse_scroll: "{x}", "{y}"')
-        self.tools.os.mouse_scroll(x, y)
+        self._reporter.add_message("User", f'mouse_scroll: "{dx}", "{dy}"')
+        self.tools.os.mouse_scroll(dx, dy)
 
     @telemetry.record_call(exclude={"text", "locator"})
     @validate_call(config=ConfigDict(arbitrary_types_allowed=True))

src/askui/tools/agent_os.py

@@ -319,14 +319,14 @@ def mouse_up(self, button: MouseButton = "left") -> None:
         raise NotImplementedError
 
     @abstractmethod
-    def mouse_scroll(self, x: int, y: int) -> None:
+    def mouse_scroll(self, dx: int, dy: int) -> None:
         """
         Simulates scrolling the mouse wheel.
 
         Args:
-            x (int): The horizontal scroll amount. Positive values scroll right,
+            dx (int): The horizontal scroll amount. Positive values scroll right,
                 negative values scroll left.
-            y (int): The vertical scroll amount. Positive values scroll down,
+            dy (int): The vertical scroll amount. Positive values scroll down,
                 negative values scroll up.
         """
         raise NotImplementedError

src/askui/tools/askui/askui_controller.py

@@ -503,37 +503,37 @@ def mouse_up(self, button: Literal["left", "middle", "right"] = "left") -> None:
 
     @telemetry.record_call()
     @override
-    def mouse_scroll(self, x: int, y: int) -> None:
+    def mouse_scroll(self, dx: int, dy: int) -> None:
         """
         Scroll the mouse wheel.
 
         Args:
-            x (int): The horizontal scroll amount. Positive values scroll right,
+            dx (int): The horizontal scroll amount. Positive values scroll right,
                 negative values scroll left.
-            y (int): The vertical scroll amount. Positive values scroll down,
+            dy (int): The vertical scroll amount. Positive values scroll down,
                 negative values scroll up.
         """
-        self._reporter.add_message("AgentOS", f"mouse_scroll({x}, {y})")
-        if x != 0:
+        self._reporter.add_message("AgentOS", f"mouse_scroll({dx}, {dy})")
+        if dx != 0:
             self._run_recorder_action(
                 acion_class_id=controller_v1_pbs.ActionClassID_MouseWheelScroll,
                 action_parameters=controller_v1_pbs.ActionParameters(
                     mouseWheelScroll=controller_v1_pbs.ActionParameters_MouseWheelScroll(
                         direction=controller_v1_pbs.MouseWheelScrollDirection.MouseWheelScrollDirection_Horizontal,
                         deltaType=controller_v1_pbs.MouseWheelDeltaType.MouseWheelDelta_Raw,
-                        delta=x,
+                        delta=dx,
                         milliseconds=50,
                     )
                 ),
             )
-        if y != 0:
+        if dy != 0:
             self._run_recorder_action(
                 acion_class_id=controller_v1_pbs.ActionClassID_MouseWheelScroll,
                 action_parameters=controller_v1_pbs.ActionParameters(
                     mouseWheelScroll=controller_v1_pbs.ActionParameters_MouseWheelScroll(
                         direction=controller_v1_pbs.MouseWheelScrollDirection.MouseWheelScrollDirection_Vertical,
                         deltaType=controller_v1_pbs.MouseWheelDeltaType.MouseWheelDelta_Raw,
-                        delta=y,
+                        delta=dy,
                         milliseconds=50,
                     )
                 ),

src/askui/tools/computer/mouse_scroll_tool.py

@@ -8,22 +8,37 @@ class ComputerMouseScrollTool(ComputerBaseTool):
     def __init__(self, agent_os: ComputerAgentOsFacade | None = None) -> None:
         super().__init__(
             name="mouse_scroll",
-            description="Scroll the mouse wheel at the current position.",
+            description=(
+                "Scroll the mouse wheel at the current "
+                "position. The scroll amount depends on "
+                "the operating system. Start with "
+                "dy=150 or dy=-150 for a normal scroll "
+                "and adjust based on the result. "
+                "On Windows, positive dy scrolls down. "
+                "On macOS (default settings), negative "
+                "dy scrolls down. If scrolling moves in "
+                "the opposite direction than expected, "
+                "invert the values."
+            ),
             input_schema={
                 "type": "object",
                 "properties": {
                     "dx": {
                         "type": "integer",
                         "description": (
                             "The horizontal scroll amount. "
-                            "Positive values scroll right, negative values scroll left."
+                            "Positive values scroll right, "
+                            "negative values scroll left. "
+                            "Use 0 if no horizontal "
+                            "scrolling is needed."
                         ),
                     },
                     "dy": {
                         "type": "integer",
                         "description": (
                             "The vertical scroll amount. "
-                            "Positive values scroll down, negative values scroll up."
+                            "Positive values scroll down, negative values scroll up. "
+                            "Use 0 if no vertical scrolling is needed."
                         ),
                     },
                 },

src/askui/tools/computer_agent_os_facade.py

@@ -84,11 +84,8 @@ def mouse_down(self, button: MouseButton = "left") -> None:
     def mouse_up(self, button: MouseButton = "left") -> None:
         self._agent_os.mouse_up(button)
 
-    def mouse_scroll(self, x: int, y: int) -> None:
-        scaled_x, scaled_y = self._scale_coordinates_back(
-            x, y, check_coordinates_in_bounds=False
-        )
-        self._agent_os.mouse_scroll(scaled_x, scaled_y)
+    def mouse_scroll(self, dx: int, dy: int) -> None:
+        self._agent_os.mouse_scroll(dx, dy)
 
     def keyboard_pressed(
         self, key: PcKey | ModifierKey, modifier_keys: list[ModifierKey] | None = None

src/askui/tools/playwright/agent_os.py

@@ -259,21 +259,21 @@ def mouse_up(self, button: Literal["left", "middle", "right"] = "left") -> None:
         self._page.mouse.up(button=button)
 
     @override
-    def mouse_scroll(self, x: int, y: int) -> None:
+    def mouse_scroll(self, dx: int, dy: int) -> None:
         """
         Simulates scrolling the mouse wheel.
 
         Args:
-            x (int): The horizontal scroll amount. Positive values scroll right,
+            dx (int): The horizontal scroll amount. Positive values scroll right,
                 negative values scroll left.
-            y (int): The vertical scroll amount. Positive values scroll down,
+            dy (int): The vertical scroll amount. Positive values scroll down,
                 negative values scroll up.
         """
         if not self._page:
             error_msg = "No active page. Call connect() first."
             raise RuntimeError(error_msg)
 
-        self._page.mouse.wheel(delta_x=x, delta_y=y)
+        self._page.mouse.wheel(delta_x=dx, delta_y=dy)
 
     @override
     def keyboard_pressed(