fix: bypass X11 for scroll using CDP pixel-precise deltas

images/chromium-headful/client/src/components/video.vue

@@ -552,6 +552,10 @@
     }
 
     beforeDestroy() {
+      if (this._scrollFlushTimeout != null) {
+        clearTimeout(this._scrollFlushTimeout)
+        this._scrollFlushTimeout = null
+      }
       this.observer.disconnect()
       this.$accessor.video.setPlayable(false)
       /* Guacamole Keyboard does not provide destroy functions */
@@ -708,7 +712,53 @@
       })
     }
 
-    wheelThrottle = false
+    _scrollAccX = 0
+    _scrollAccY = 0
+    _scrollLastSendTime = 0
+    _scrollFlushTimeout: ReturnType<typeof setTimeout> | null = null
+    _scrollLastClientX = 0
+    _scrollLastClientY = 0
+    _scrollApiUrl: string | null = null
+
+    _getScrollApiUrl(): string {
+      if (this._scrollApiUrl) return this._scrollApiUrl
+      // The kernel-images API is exposed on port 444 (maps to 10001 inside the
+      // container) in both Docker and unikernel deployments.
+      this._scrollApiUrl = `${location.protocol}//${location.hostname}:444/live-view/scroll`
+      return this._scrollApiUrl
+    }
+
+    _clearScrollFlushTimeout() {
+      if (this._scrollFlushTimeout != null) {
+        clearTimeout(this._scrollFlushTimeout)
+        this._scrollFlushTimeout = null
+      }
+    }
+
+    _sendScrollAccumulated(clientX: number, clientY: number) {
+      if (this._scrollAccX === 0 && this._scrollAccY === 0) {
+        return
+      }
+      const { w, h } = this.$accessor.video.resolution
+      const rect = this._overlay.getBoundingClientRect()
+      const sx = Math.round((w / rect.width) * (clientX - rect.left))
+      const sy = Math.round((h / rect.height) * (clientY - rect.top))
+
+      const dx = this._scrollAccX
+      const dy = this._scrollAccY
+      this._scrollAccX = 0
+      this._scrollAccY = 0
+      this._scrollLastSendTime = Date.now()
+
+      const url = this._getScrollApiUrl()
+      fetch(url, {
+        method: 'POST',
+        headers: { 'Content-Type': 'application/json' },
+        body: JSON.stringify({ x: sx, y: sy, delta_x: -dx, delta_y: -dy }),
+        keepalive: true,
+      }).catch(() => {})
+    }
+
     onWheel(e: WheelEvent) {
       if (!this.hosting || this.locked) {
         return
@@ -717,8 +767,6 @@
       let x = e.deltaX
       let y = e.deltaY
 
-      // Normalize to pixel units. deltaMode 1 = lines, 2 = pages; convert
-      // both to approximate pixel values so the divisor below works uniformly.
       if (e.deltaMode !== 0) {
         x *= WHEEL_LINE_HEIGHT
         y *= WHEEL_LINE_HEIGHT
@@ -729,26 +777,29 @@
         y = y * -1
       }
 
-      // The server sends one XTestFakeButtonEvent per unit we pass here,
-      // and each event scrolls Chromium by ~120 px. Raw pixel deltas from
-      // trackpads are already in pixels (~120 per notch), so dividing by
-      // PIXELS_PER_TICK converts them to discrete scroll "ticks". The
-      // result is clamped to [-scroll, scroll] (the user-facing sensitivity
-      // setting) so fast swipes don't over-scroll.
-      const PIXELS_PER_TICK = 120
-      x = x === 0 ? 0 : Math.min(Math.max(Math.round(x / PIXELS_PER_TICK) || Math.sign(x), -this.scroll), this.scroll)
-      y = y === 0 ? 0 : Math.min(Math.max(Math.round(y / PIXELS_PER_TICK) || Math.sign(y), -this.scroll), this.scroll)
+      this._scrollAccX += x
+      this._scrollAccY += y
 
-      this.sendMousePos(e)
+      if (this._scrollAccX === 0 && this._scrollAccY === 0) {
+        return
+      }
 
-      if (!this.wheelThrottle) {
-        this.wheelThrottle = true
-        this.$client.sendData('wheel', { x, y })
+      this._scrollLastClientX = e.clientX
+      this._scrollLastClientY = e.clientY
 
-        window.setTimeout(() => {
-          this.wheelThrottle = false
-        }, 100)
+      const now = Date.now()
+      if (now - this._scrollLastSendTime < 50) {
+        if (this._scrollFlushTimeout == null) {
+          this._scrollFlushTimeout = setTimeout(() => {
+            this._scrollFlushTimeout = null
+            this._sendScrollAccumulated(this._scrollLastClientX, this._scrollLastClientY)
+          }, 50)
+        }
+        return
       }
+
+      this._clearScrollFlushTimeout()
+      this._sendScrollAccumulated(e.clientX, e.clientY)
     }
 
     onTouchHandler(e: TouchEvent) {

server/cmd/api/api/computer.go

@@ -16,6 +16,7 @@ import (
 	"syscall"
 	"time"
 
+	"github.com/onkernel/kernel-images/server/lib/cdpclient"
 	"github.com/onkernel/kernel-images/server/lib/logger"
 	"github.com/onkernel/kernel-images/server/lib/mousetrajectory"
 	oapi "github.com/onkernel/kernel-images/server/lib/oapi"
@@ -748,6 +749,48 @@ func (s *ApiService) PressKey(ctx context.Context, request oapi.PressKeyRequestO
 	return oapi.PressKey200Response{}, nil
 }
 
+const pixelsPerScrollTick = 120
+
+// CDP Input.dispatchMouseEvent modifier bitmask (Alt=1, Ctrl=2, Meta=4, Shift=8).
+const (
+	cdpModAlt   = 1
+	cdpModCtrl  = 2
+	cdpModMeta  = 4
+	cdpModShift = 8
+)
+
+// holdKeysToCDPModifiers maps xdotool-style hold key names to CDP mouse event modifiers.
+func holdKeysToCDPModifiers(keys []string) int {
+	var m int
+	for _, raw := range keys {
+		k := strings.ToLower(strings.TrimSpace(raw))
+		if k == "" {
+			continue
+		}
+		switch k {
+		case "ctrl", "control", "control_l", "control_r":
+			m |= cdpModCtrl
+		case "shift", "shift_l", "shift_r":
+			m |= cdpModShift
+		case "alt", "alt_l", "alt_r":
+			m |= cdpModAlt
+		case "meta", "super", "super_l", "super_r", "command", "command_l", "command_r":
+			m |= cdpModMeta
+		default:
+			if strings.HasPrefix(k, "control") {
+				m |= cdpModCtrl
+			} else if strings.HasPrefix(k, "shift") {
+				m |= cdpModShift
+			} else if strings.HasPrefix(k, "alt") {
+				m |= cdpModAlt
+			} else if strings.HasPrefix(k, "super") || strings.HasPrefix(k, "meta") {
+				m |= cdpModMeta
+			}
+		}
+	}
+	return m
+}
+
 func (s *ApiService) doScroll(ctx context.Context, body oapi.ScrollRequest) error {
 	log := logger.FromContext(ctx)
 
@@ -769,50 +812,103 @@ func (s *ApiService) doScroll(ctx context.Context, body oapi.ScrollRequest) erro
 		return &validationError{msg: fmt.Sprintf("coordinates exceed screen bounds (max: %dx%d)", screenWidth-1, screenHeight-1)}
 	}
 
-	args := []string{}
-	if body.HoldKeys != nil {
+	// Hold keys via xdotool (CDP doesn't have a direct modifier-hold mechanism
+	// that persists across separate commands).
+	if body.HoldKeys != nil && len(*body.HoldKeys) > 0 {
+		var keydownArgs []string
 		for _, key := range *body.HoldKeys {
-			args = append(args, "keydown", key)
+			keydownArgs = append(keydownArgs, "keydown", key)
+		}
+		if _, err := defaultXdoTool.Run(ctx, keydownArgs...); err != nil {
+			log.Error("xdotool keydown failed", "err", err)
 		}
+		defer func() {
+			var keyupArgs []string
+			for _, key := range *body.HoldKeys {
+				keyupArgs = append(keyupArgs, "keyup", key)
+			}
+			if _, err := defaultXdoTool.Run(context.Background(), keyupArgs...); err != nil {
+				log.Error("xdotool keyup failed", "err", err)
+			}
+		}()
 	}
-	args = append(args, "mousemove", strconv.Itoa(body.X), strconv.Itoa(body.Y))
 
-	// Apply vertical ticks first (sequential as specified)
-	if body.DeltaY != nil && *body.DeltaY != 0 {
-		count := *body.DeltaY
-		btn := "5" // down
-		if count < 0 {
-			btn = "4" // up
-			count = -count
-		}
-		args = append(args, "click", "--repeat", strconv.Itoa(count), "--delay", "0", btn)
+	// Convert tick counts to CSS pixel deltas for CDP. The API contract
+	// specifies delta_x/delta_y as discrete scroll ticks (matching the old
+	// xdotool button-click model). Each tick ≈ 120 CSS pixels.
+	var deltaXPx, deltaYPx float64
+	if body.DeltaX != nil {
+		deltaXPx = float64(*body.DeltaX) * pixelsPerScrollTick
 	}
-	// Then horizontal ticks
-	if body.DeltaX != nil && *body.DeltaX != 0 {
-		count := *body.DeltaX
-		btn := "7" // right
-		if count < 0 {
-			btn = "6" // left
-			count = -count
-		}
-		args = append(args, "click", "--repeat", strconv.Itoa(count), "--delay", "0", btn)
+	if body.DeltaY != nil {
+		deltaYPx = float64(*body.DeltaY) * pixelsPerScrollTick
 	}
 
+	modifiers := 0
 	if body.HoldKeys != nil {
-		for _, key := range *body.HoldKeys {
-			args = append(args, "keyup", key)
-		}
+		modifiers = holdKeysToCDPModifiers(*body.HoldKeys)
 	}
 
-	log.Info("executing xdotool", "args", args)
-	output, err := defaultXdoTool.Run(ctx, args...)
+	upstreamURL := s.upstreamMgr.Current()
+	if upstreamURL == "" {
+		return &executionError{msg: "devtools upstream not available"}
+	}
+
+	cdpCtx, cancel := context.WithTimeout(ctx, 5*time.Second)
+	defer cancel()
+
+	client, err := cdpclient.Dial(cdpCtx, upstreamURL)
 	if err != nil {
-		log.Error("xdotool scroll failed", "err", err, "output", string(output))
-		return &executionError{msg: fmt.Sprintf("failed to perform scroll: %s", string(output))}
+		return &executionError{msg: fmt.Sprintf("failed to connect to devtools for scroll: %s", err)}
+	}
+	defer client.Close()
+
+	log.Info("dispatching CDP mouseWheel", "x", body.X, "y", body.Y, "deltaX", deltaXPx, "deltaY", deltaYPx, "modifiers", modifiers)
+	if err := client.DispatchMouseWheelEvent(cdpCtx, body.X, body.Y, deltaXPx, deltaYPx, modifiers); err != nil {
+		return &executionError{msg: fmt.Sprintf("CDP mouseWheel failed: %s", err)}
 	}
+
 	return nil
 }
 
+func (s *ApiService) LiveViewScroll(ctx context.Context, request oapi.LiveViewScrollRequestObject) (oapi.LiveViewScrollResponseObject, error) {
+	if request.Body == nil {
+		return oapi.LiveViewScroll400JSONResponse{BadRequestErrorJSONResponse: oapi.BadRequestErrorJSONResponse{Message: "missing request body"}}, nil
+	}
+
+	var deltaX, deltaY float64
+	if request.Body.DeltaX != nil {
+		deltaX = *request.Body.DeltaX
+	}
+	if request.Body.DeltaY != nil {
+		deltaY = *request.Body.DeltaY
+	}
+
+	if deltaX == 0 && deltaY == 0 {
+		return oapi.LiveViewScroll200Response{}, nil
+	}
+
+	upstreamURL := s.upstreamMgr.Current()
+	if upstreamURL == "" {
+		return oapi.LiveViewScroll503JSONResponse{Message: "devtools upstream not available"}, nil
+	}
+
+	cdpCtx, cancel := context.WithTimeout(ctx, 3*time.Second)
+	defer cancel()
+
+	client, err := cdpclient.Dial(cdpCtx, upstreamURL)
+	if err != nil {
+		return oapi.LiveViewScroll500JSONResponse{InternalErrorJSONResponse: oapi.InternalErrorJSONResponse{Message: fmt.Sprintf("cdp dial failed: %s", err)}}, nil
+	}
+	defer client.Close()
+
+	if err := client.DispatchMouseWheelEvent(cdpCtx, request.Body.X, request.Body.Y, deltaX, deltaY, 0); err != nil {
+		return oapi.LiveViewScroll500JSONResponse{InternalErrorJSONResponse: oapi.InternalErrorJSONResponse{Message: fmt.Sprintf("cdp scroll failed: %s", err)}}, nil
+	}
+
+	return oapi.LiveViewScroll200Response{}, nil
+}
+
 func (s *ApiService) Scroll(ctx context.Context, request oapi.ScrollRequestObject) (oapi.ScrollResponseObject, error) {
 	s.inputMu.Lock()
 	defer s.inputMu.Unlock()

server/cmd/api/main.go

@@ -54,6 +54,20 @@ func main() {
 	r.Use(
 		chiMiddleware.Logger,
 		chiMiddleware.Recoverer,
+		func(next http.Handler) http.Handler {
+			return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+				if strings.HasPrefix(r.URL.Path, "/live-view/") {
+					w.Header().Set("Access-Control-Allow-Origin", "*")
+					w.Header().Set("Access-Control-Allow-Methods", "GET, POST, PUT, PATCH, DELETE, OPTIONS")
+					w.Header().Set("Access-Control-Allow-Headers", "Content-Type, Authorization")
+					if r.Method == http.MethodOptions {
+						w.WriteHeader(http.StatusNoContent)
+						return
+					}
+				}
+				next.ServeHTTP(w, r)
+			})
+		},
 		func(next http.Handler) http.Handler {
 			return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
 				ctxWithLogger := logger.AddToContext(r.Context(), slogger)

server/go.sum

@@ -127,8 +127,6 @@ github.com/morikuni/aec v1.0.0 h1:nP9CBfwrvYnBRgY6qfDQkygYDmYwOilePFkwzv4dU8A=
 github.com/morikuni/aec v1.0.0/go.mod h1:BbKIizmSmc5MMPqRYbxO4ZU0S0+P200+tUnFx7PXmsc=
 github.com/nrednav/cuid2 v1.1.0 h1:Y2P9Fo1Iz7lKuwcn+fS0mbxkNvEqoNLUtm0+moHCnYc=
 github.com/nrednav/cuid2 v1.1.0/go.mod h1:jBjkJAI+QLM4EUGvtwGDHC1cP1QQrRNfLo/A7qJFDhA=
-github.com/oapi-codegen/runtime v1.1.2 h1:P2+CubHq8fO4Q6fV1tqDBZHCwpVpvPg7oKiYzQgXIyI=
-github.com/oapi-codegen/runtime v1.1.2/go.mod h1:SK9X900oXmPWilYR5/WKPzt3Kqxn/uS/+lbpREv+eCg=
 github.com/oapi-codegen/runtime v1.2.0 h1:RvKc1CVS1QeKSNzO97FBQbSMZyQ8s6rZd+LpmzwHMP4=
 github.com/oapi-codegen/runtime v1.2.0/go.mod h1:Y7ZhmmlE8ikZOmuHRRndiIm7nf3xcVv+YMweKgG1DT0=
 github.com/oasdiff/yaml v0.0.0-20250309154309-f31be36b4037 h1:G7ERwszslrBzRxj//JalHPu/3yz+De2J+4aLtSRlHiY=
@@ -175,9 +173,8 @@ github.com/tklauser/go-sysconf v0.3.12 h1:0QaGUFOdQaIVdPgfITYzaTegZvdCjmYO52cSFA
 github.com/tklauser/go-sysconf v0.3.12/go.mod h1:Ho14jnntGE1fpdOqQEEaiKRpvIavV0hSfmBq8nJbHYI=
 github.com/tklauser/numcpus v0.6.1 h1:ng9scYS7az0Bk4OZLvrNXNSAO2Pxr1XXRAPyjhIx+Fk=
 github.com/tklauser/numcpus v0.6.1/go.mod h1:1XfjsgE2zo8GVw7POkMbHENHzVg3GzmoZ9fESEdAacY=
-github.com/ugorji/go/codec v1.2.11 h1:BMaWp1Bb6fHwEtbplGBGJ498wD+LKlNSl25MjdZY4dU=
-github.com/ugorji/go/codec v1.2.11/go.mod h1:UNopzCgEMSXjBc6AOMqYvWC1ktqTAfzJZUZgYf6w6lg=
 github.com/ugorji/go/codec v1.2.12 h1:9LC83zGrHhuUA9l16C9AHXAqEV/2wBQ4nkvumAE65EE=
+github.com/ugorji/go/codec v1.2.12/go.mod h1:UNopzCgEMSXjBc6AOMqYvWC1ktqTAfzJZUZgYf6w6lg=
 github.com/yusufpapurcu/wmi v1.2.4 h1:zFUKzehAFReQwLys1b/iSMl+JQGSCSjtVqQn9bBrPo0=
 github.com/yusufpapurcu/wmi v1.2.4/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0=
 go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64=