feat: binder-mcp P0 shell tools — screenshot, input, UI, apps, settings, shell

Author: xaionaro@dx.center

cmd/binder-mcp/device.go

@@ -71,6 +71,7 @@ func runDevice(cmd *cobra.Command, _ []string) error {
 	)
 
 	tools.Register(mcpServer)
+	RegisterShellTools(mcpServer)
 
 	logger.Debugf(ctx, "serving MCP over stdio")
 

cmd/binder-mcp/remote.go

@@ -92,6 +92,7 @@ func runRemoteMode(
 	)
 
 	tools.Register(mcpServer)
+	RegisterShellTools(mcpServer)
 
 	logger.Debugf(ctx, "serving MCP over stdio (remote mode)")
 

cmd/binder-mcp/shell.go

@@ -0,0 +1,26 @@
+//go:build linux
+
+package main
+
+import (
+	"fmt"
+	"os/exec"
+	"strings"
+)
+
+// shellExec runs a shell command and returns the combined stdout+stderr output.
+func shellExec(command string) (string, error) {
+	cmd := exec.Command("sh", "-c", command)
+	out, err := cmd.CombinedOutput()
+	result := strings.TrimSpace(string(out))
+	if err != nil {
+		return result, fmt.Errorf("command %q: %w (output: %s)", command, err, result)
+	}
+	return result, nil
+}
+
+// shellQuote quotes a string for safe use in shell commands,
+// preventing injection attacks.
+func shellQuote(s string) string {
+	return "'" + strings.ReplaceAll(s, "'", "'\"'\"'") + "'"
+}

cmd/binder-mcp/tool_click_ui_element.go

@@ -0,0 +1,104 @@
+//go:build linux
+
+package main
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+
+	"github.com/facebookincubator/go-belt/tool/logger"
+	"github.com/mark3labs/mcp-go/mcp"
+	"github.com/mark3labs/mcp-go/server"
+)
+
+// ClickUIResult describes the click_ui_element response.
+type ClickUIResult struct {
+	MatchedText        string `json:"matched_text,omitempty"`
+	MatchedResourceID  string `json:"matched_resource_id,omitempty"`
+	MatchedContentDesc string `json:"matched_content_desc,omitempty"`
+	TappedX            int    `json:"tapped_x"`
+	TappedY            int    `json:"tapped_y"`
+}
+
+func registerClickUIElement(s *server.MCPServer) {
+	tool := mcp.NewTool("click_ui_element",
+		mcp.WithDescription(
+			"Find a UI element by text, resource-id, or content-desc, "+
+				"compute the center of its bounds, and tap it. "+
+				"This is a convenience tool that combines find_ui_element + tap. "+
+				"Returns an error if no matching element is found or if multiple "+
+				"elements match (use more specific criteria).",
+		),
+		mcp.WithString("text",
+			mcp.Description("Match element whose text contains this substring (case-insensitive)"),
+		),
+		mcp.WithString("resource_id",
+			mcp.Description("Match element whose resource-id contains this substring"),
+		),
+		mcp.WithString("content_desc",
+			mcp.Description("Match element whose content-desc contains this substring (case-insensitive)"),
+		),
+		mcp.WithDestructiveHintAnnotation(true),
+		mcp.WithIdempotentHintAnnotation(false),
+	)
+
+	s.AddTool(tool, handleClickUIElement)
+}
+
+func handleClickUIElement(
+	ctx context.Context,
+	request mcp.CallToolRequest,
+) (*mcp.CallToolResult, error) {
+	logger.Tracef(ctx, "handleClickUIElement")
+	defer func() { logger.Tracef(ctx, "/handleClickUIElement") }()
+
+	textFilter := request.GetString("text", "")
+	resourceIDFilter := request.GetString("resource_id", "")
+	contentDescFilter := request.GetString("content_desc", "")
+
+	if textFilter == "" && resourceIDFilter == "" && contentDescFilter == "" {
+		return mcp.NewToolResultError("at least one search filter is required (text, resource_id, or content_desc)"), nil
+	}
+
+	xmlData, err := dumpUIXML()
+	if err != nil {
+		return mcp.NewToolResultError(fmt.Sprintf("dumping UI: %v", err)), nil
+	}
+
+	elements := parseAndFilterUI(xmlData, textFilter, resourceIDFilter, contentDescFilter, "")
+
+	switch len(elements) {
+	case 0:
+		return mcp.NewToolResultError("no matching UI element found"), nil
+	case 1:
+		// Exactly one match — tap it.
+	default:
+		data, _ := json.Marshal(elements)
+		return mcp.NewToolResultError(fmt.Sprintf(
+			"found %d matching elements (use more specific criteria): %s",
+			len(elements), string(data),
+		)), nil
+	}
+
+	elem := elements[0]
+	cmd := fmt.Sprintf("input tap %d %d", elem.CenterX, elem.CenterY)
+	if _, err := shellExec(cmd); err != nil {
+		return mcp.NewToolResultError(fmt.Sprintf("input tap: %v", err)), nil
+	}
+
+	result := ClickUIResult{
+		MatchedText:        elem.Text,
+		MatchedResourceID:  elem.ResourceID,
+		MatchedContentDesc: elem.ContentDesc,
+		TappedX:            elem.CenterX,
+		TappedY:            elem.CenterY,
+	}
+
+	data, err := json.Marshal(result)
+	if err != nil {
+		return nil, fmt.Errorf("marshaling click result: %w", err)
+	}
+
+	return mcp.NewToolResultText(string(data)), nil
+}

cmd/binder-mcp/tool_dump_service.go

@@ -0,0 +1,53 @@
+//go:build linux
+
+package main
+
+import (
+	"context"
+	"fmt"
+
+	"github.com/facebookincubator/go-belt/tool/logger"
+	"github.com/mark3labs/mcp-go/mcp"
+	"github.com/mark3labs/mcp-go/server"
+)
+
+func registerDumpService(s *server.MCPServer) {
+	tool := mcp.NewTool("dump_service",
+		mcp.WithDescription(
+			"Dump the state of an Android system service using 'dumpsys'. "+
+				"Returns the full text dump. Common services: activity, window, "+
+				"power, battery, wifi, connectivity, meminfo, cpuinfo, package, "+
+				"notification, alarm, audio, display, input.",
+		),
+		mcp.WithString("service",
+			mcp.Required(),
+			mcp.Description("Service name (e.g. 'activity', 'battery', 'meminfo')"),
+		),
+		mcp.WithReadOnlyHintAnnotation(true),
+		mcp.WithDestructiveHintAnnotation(false),
+		mcp.WithIdempotentHintAnnotation(true),
+	)
+
+	s.AddTool(tool, handleDumpService)
+}
+
+func handleDumpService(
+	ctx context.Context,
+	request mcp.CallToolRequest,
+) (*mcp.CallToolResult, error) {
+	logger.Tracef(ctx, "handleDumpService")
+	defer func() { logger.Tracef(ctx, "/handleDumpService") }()
+
+	service, err := request.RequireString("service")
+	if err != nil {
+		return mcp.NewToolResultError(err.Error()), nil
+	}
+
+	cmd := fmt.Sprintf("dumpsys %s", shellQuote(service))
+	out, err := shellExec(cmd)
+	if err != nil {
+		return mcp.NewToolResultError(fmt.Sprintf("dumpsys: %v", err)), nil
+	}
+
+	return mcp.NewToolResultText(out), nil
+}

cmd/binder-mcp/tool_dump_ui_hierarchy.go

@@ -0,0 +1,64 @@
+//go:build linux
+
+package main
+
+import (
+	"context"
+	"fmt"
+
+	"github.com/facebookincubator/go-belt/tool/logger"
+	"github.com/mark3labs/mcp-go/mcp"
+	"github.com/mark3labs/mcp-go/server"
+)
+
+const uiDumpPath = "/data/local/tmp/ui.xml"
+
+func registerDumpUIHierarchy(s *server.MCPServer) {
+	tool := mcp.NewTool("dump_ui_hierarchy",
+		mcp.WithDescription(
+			"Dump the current UI hierarchy as XML using 'uiautomator dump'. "+
+				"Returns the full XML tree of all visible UI elements with their "+
+				"properties (text, resource-id, class, bounds, etc.). "+
+				"This operation can take 2-3 seconds.",
+		),
+		mcp.WithReadOnlyHintAnnotation(true),
+		mcp.WithDestructiveHintAnnotation(false),
+		mcp.WithIdempotentHintAnnotation(true),
+	)
+
+	s.AddTool(tool, handleDumpUIHierarchy)
+}
+
+func handleDumpUIHierarchy(
+	ctx context.Context,
+	_ mcp.CallToolRequest,
+) (*mcp.CallToolResult, error) {
+	logger.Tracef(ctx, "handleDumpUIHierarchy")
+	defer func() { logger.Tracef(ctx, "/handleDumpUIHierarchy") }()
+
+	cmd := fmt.Sprintf("uiautomator dump %s && cat %s",
+		shellQuote(uiDumpPath), shellQuote(uiDumpPath))
+
+	out, err := shellExec(cmd)
+	if err != nil {
+		return mcp.NewToolResultError(fmt.Sprintf("uiautomator dump: %v", err)), nil
+	}
+
+	// uiautomator dump prints a status line before the XML in some versions.
+	// The XML always starts with "<?xml" — strip any prefix.
+	xml := extractXML(out)
+
+	return mcp.NewToolResultText(xml), nil
+}
+
+// extractXML strips any non-XML prefix from uiautomator output.
+// The dump command sometimes prints "UI hierchary dumped to: /path"
+// before the XML when using 'dump && cat'.
+func extractXML(output string) string {
+	for i := 0; i < len(output); i++ {
+		if output[i] == '<' {
+			return output[i:]
+		}
+	}
+	return output
+}