From 17d8d0df78d4a2b0df981e2f476c618a45dcabd5 Mon Sep 17 00:00:00 2001 From: julscezar Date: Mon, 6 Apr 2026 05:01:23 -0600 Subject: [PATCH] fix: conditionally declare tools when OpenClaw is configured (fixes #12) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When OpenClaw is not configured, Gemini still received tool declarations in the setup message. This caused it to attempt tool calls (like web search) that had no backend to handle them, getting permanently stuck in "executing" state. Changes: - Only include tools array in Gemini setup when isOpenClawConfigured - Use a no-tools system instruction when OpenClaw is absent, so Gemini tells the user to enable OpenClaw instead of attempting tool calls - Add fast-fail in ToolCallRouter for unconfigured OpenClaw (defense in depth — returns immediate error instead of hanging) - Applied to both iOS (Swift) and Android (Kotlin) Co-Authored-By: Claude Opus 4.6 (1M context) --- .../CameraAccess/Gemini/GeminiConfig.swift | 8 ++ .../Gemini/GeminiLiveService.swift | 83 ++++++++++--------- .../OpenClaw/ToolCallRouter.swift | 12 +++ .../cameraaccess/gemini/GeminiConfig.kt | 6 ++ .../cameraaccess/gemini/GeminiLiveService.kt | 20 ++++- .../cameraaccess/openclaw/ToolCallRouter.kt | 12 +++ 6 files changed, 99 insertions(+), 42 deletions(-) diff --git a/samples/CameraAccess/CameraAccess/Gemini/GeminiConfig.swift b/samples/CameraAccess/CameraAccess/Gemini/GeminiConfig.swift index 5c124f66..73a979d6 100644 --- a/samples/CameraAccess/CameraAccess/Gemini/GeminiConfig.swift +++ b/samples/CameraAccess/CameraAccess/Gemini/GeminiConfig.swift @@ -42,6 +42,14 @@ enum GeminiConfig { For messages, confirm recipient and content before delegating unless clearly urgent. """ + static let noToolsSystemInstruction = """ + You are an AI assistant for someone wearing Meta Ray-Ban smart glasses. You can see through their camera and have a voice conversation. Keep responses concise and natural. + + You do NOT have any tools. You cannot send messages, search the web, manage lists, set reminders, or take any actions. You are a voice + vision assistant only. + + If the user asks you to do something that requires an action (send a message, search, add to a list, etc.), let them know that OpenClaw is not connected and they need to set it up in Settings to enable those features. + """ + // User-configurable values (Settings screen overrides, falling back to Secrets.swift) static var apiKey: String { SettingsManager.shared.geminiAPIKey } static var openClawHost: String { SettingsManager.shared.openClawHost } diff --git a/samples/CameraAccess/CameraAccess/Gemini/GeminiLiveService.swift b/samples/CameraAccess/CameraAccess/Gemini/GeminiLiveService.swift index 248f2f02..d23598da 100644 --- a/samples/CameraAccess/CameraAccess/Gemini/GeminiLiveService.swift +++ b/samples/CameraAccess/CameraAccess/Gemini/GeminiLiveService.swift @@ -178,46 +178,53 @@ class GeminiLiveService: ObservableObject { } private func sendSetupMessage() { - let setup: [String: Any] = [ - "setup": [ - "model": GeminiConfig.model, - "generationConfig": [ - "responseModalities": ["AUDIO"], - "thinkingConfig": [ - "thinkingBudget": 0 - ] - ], - "systemInstruction": [ - "parts": [ - ["text": GeminiConfig.systemInstruction] - ] - ], - "tools": [ - [ - "functionDeclarations": ToolDeclarations.allDeclarations() - ] - ], - "realtimeInputConfig": [ - "automaticActivityDetection": [ - "disabled": false, - "startOfSpeechSensitivity": "START_SENSITIVITY_HIGH", - "endOfSpeechSensitivity": "END_SENSITIVITY_LOW", - "silenceDurationMs": 500, - "prefixPaddingMs": 40 - ], - "activityHandling": "START_OF_ACTIVITY_INTERRUPTS", - "turnCoverage": "TURN_INCLUDES_ALL_INPUT" - ], - "contextWindowCompression": [ - "slidingWindow": [ - "targetTokens": 80000 - ] + var setupContent: [String: Any] = [ + "model": GeminiConfig.model, + "generationConfig": [ + "responseModalities": ["AUDIO"], + "thinkingConfig": [ + "thinkingBudget": 0 + ] + ], + "systemInstruction": [ + "parts": [ + ["text": GeminiConfig.isOpenClawConfigured + ? GeminiConfig.systemInstruction + : GeminiConfig.noToolsSystemInstruction] + ] + ], + "realtimeInputConfig": [ + "automaticActivityDetection": [ + "disabled": false, + "startOfSpeechSensitivity": "START_SENSITIVITY_HIGH", + "endOfSpeechSensitivity": "END_SENSITIVITY_LOW", + "silenceDurationMs": 500, + "prefixPaddingMs": 40 ], - "inputAudioTranscription": [:] as [String: Any], - "outputAudioTranscription": [:] as [String: Any] - ] + "activityHandling": "START_OF_ACTIVITY_INTERRUPTS", + "turnCoverage": "TURN_INCLUDES_ALL_INPUT" + ], + "contextWindowCompression": [ + "slidingWindow": [ + "targetTokens": 80000 + ] + ], + "inputAudioTranscription": [:] as [String: Any], + "outputAudioTranscription": [:] as [String: Any] ] - sendJSON(setup) + + // Only declare tools when OpenClaw is configured — otherwise Gemini + // will attempt tool calls that have no backend, getting stuck in "executing" + if GeminiConfig.isOpenClawConfigured { + setupContent["tools"] = [ + ["functionDeclarations": ToolDeclarations.allDeclarations()] + ] + NSLog("[Gemini] Setup with tools (OpenClaw configured)") + } else { + NSLog("[Gemini] Setup without tools (OpenClaw not configured)") + } + + sendJSON(["setup": setupContent]) } private func sendJSON(_ json: [String: Any]) { diff --git a/samples/CameraAccess/CameraAccess/OpenClaw/ToolCallRouter.swift b/samples/CameraAccess/CameraAccess/OpenClaw/ToolCallRouter.swift index a20babf4..a50c0e81 100644 --- a/samples/CameraAccess/CameraAccess/OpenClaw/ToolCallRouter.swift +++ b/samples/CameraAccess/CameraAccess/OpenClaw/ToolCallRouter.swift @@ -23,6 +23,18 @@ class ToolCallRouter { NSLog("[ToolCall] Received: %@ (id: %@) args: %@", callName, callId, String(describing: call.args)) + // Fast-fail if OpenClaw is not configured + if !GeminiConfig.isOpenClawConfigured { + NSLog("[ToolCall] OpenClaw not configured, rejecting tool call %@", callId) + let errorResult: ToolResult = .failure( + "OpenClaw is not configured. Tool calls are unavailable. " + + "Please tell the user to set up OpenClaw in Settings to enable actions like web search, messaging, and more." + ) + let response = buildToolResponse(callId: callId, name: callName, result: errorResult) + sendResponse(response) + return + } + // Circuit breaker: stop sending tool calls after repeated failures if consecutiveFailures >= maxConsecutiveFailures { NSLog("[ToolCall] Circuit breaker open (%d consecutive failures), rejecting %@", diff --git a/samples/CameraAccessAndroid/app/src/main/java/com/meta/wearable/dat/externalsampleapps/cameraaccess/gemini/GeminiConfig.kt b/samples/CameraAccessAndroid/app/src/main/java/com/meta/wearable/dat/externalsampleapps/cameraaccess/gemini/GeminiConfig.kt index 10ba908e..aae15890 100644 --- a/samples/CameraAccessAndroid/app/src/main/java/com/meta/wearable/dat/externalsampleapps/cameraaccess/gemini/GeminiConfig.kt +++ b/samples/CameraAccessAndroid/app/src/main/java/com/meta/wearable/dat/externalsampleapps/cameraaccess/gemini/GeminiConfig.kt @@ -45,4 +45,10 @@ object GeminiConfig { get() = openClawGatewayToken != "YOUR_OPENCLAW_GATEWAY_TOKEN" && openClawGatewayToken.isNotEmpty() && openClawHost != "http://YOUR_MAC_HOSTNAME.local" + + const val NO_TOOLS_SYSTEM_INSTRUCTION = """You are an AI assistant for someone wearing Meta Ray-Ban smart glasses. You can see through their camera and have a voice conversation. Keep responses concise and natural. + +You do NOT have any tools. You cannot send messages, search the web, manage lists, set reminders, or take any actions. You are a voice + vision assistant only. + +If the user asks you to do something that requires an action (send a message, search, add to a list, etc.), let them know that OpenClaw is not connected and they need to set it up in Settings to enable those features.""" } diff --git a/samples/CameraAccessAndroid/app/src/main/java/com/meta/wearable/dat/externalsampleapps/cameraaccess/gemini/GeminiLiveService.kt b/samples/CameraAccessAndroid/app/src/main/java/com/meta/wearable/dat/externalsampleapps/cameraaccess/gemini/GeminiLiveService.kt index d046d306..d2370a1d 100644 --- a/samples/CameraAccessAndroid/app/src/main/java/com/meta/wearable/dat/externalsampleapps/cameraaccess/gemini/GeminiLiveService.kt +++ b/samples/CameraAccessAndroid/app/src/main/java/com/meta/wearable/dat/externalsampleapps/cameraaccess/gemini/GeminiLiveService.kt @@ -211,6 +211,8 @@ class GeminiLiveService { } private fun sendSetupMessage() { + val isOpenClawConfigured = GeminiConfig.isOpenClawConfigured + val setup = JSONObject().apply { put("setup", JSONObject().apply { put("model", GeminiConfig.MODEL) @@ -222,12 +224,22 @@ class GeminiLiveService { }) put("systemInstruction", JSONObject().apply { put("parts", JSONArray().put(JSONObject().apply { - put("text", GeminiConfig.systemInstruction) + put("text", if (isOpenClawConfigured) + GeminiConfig.systemInstruction + else + GeminiConfig.NO_TOOLS_SYSTEM_INSTRUCTION) })) }) - put("tools", JSONArray().put(JSONObject().apply { - put("functionDeclarations", ToolDeclarations.allDeclarationsJSON()) - })) + // Only declare tools when OpenClaw is configured — otherwise Gemini + // will attempt tool calls that have no backend, getting stuck in "executing" + if (isOpenClawConfigured) { + put("tools", JSONArray().put(JSONObject().apply { + put("functionDeclarations", ToolDeclarations.allDeclarationsJSON()) + })) + Log.d(TAG, "Setup with tools (OpenClaw configured)") + } else { + Log.d(TAG, "Setup without tools (OpenClaw not configured)") + } put("realtimeInputConfig", JSONObject().apply { put("automaticActivityDetection", JSONObject().apply { put("disabled", false) diff --git a/samples/CameraAccessAndroid/app/src/main/java/com/meta/wearable/dat/externalsampleapps/cameraaccess/openclaw/ToolCallRouter.kt b/samples/CameraAccessAndroid/app/src/main/java/com/meta/wearable/dat/externalsampleapps/cameraaccess/openclaw/ToolCallRouter.kt index 35337e14..8def5189 100644 --- a/samples/CameraAccessAndroid/app/src/main/java/com/meta/wearable/dat/externalsampleapps/cameraaccess/openclaw/ToolCallRouter.kt +++ b/samples/CameraAccessAndroid/app/src/main/java/com/meta/wearable/dat/externalsampleapps/cameraaccess/openclaw/ToolCallRouter.kt @@ -1,6 +1,7 @@ package com.meta.wearable.dat.externalsampleapps.cameraaccess.openclaw import android.util.Log +import com.meta.wearable.dat.externalsampleapps.cameraaccess.gemini.GeminiConfig import kotlinx.coroutines.CoroutineScope import kotlinx.coroutines.Job import kotlinx.coroutines.launch @@ -28,6 +29,17 @@ class ToolCallRouter( Log.d(TAG, "Received: $callName (id: $callId) args: ${call.args}") + // Fast-fail if OpenClaw is not configured + if (!GeminiConfig.isOpenClawConfigured) { + Log.w(TAG, "OpenClaw not configured, rejecting tool call $callId") + val errorResult = ToolResult.Failure( + "OpenClaw is not configured. Tool calls are unavailable. " + + "Please tell the user to set up OpenClaw in Settings to enable actions like web search, messaging, and more." + ) + sendResponse(buildToolResponse(callId, callName, errorResult)) + return + } + // Circuit breaker: stop sending tool calls after repeated failures if (consecutiveFailures >= MAX_CONSECUTIVE_FAILURES) { Log.d(TAG, "Circuit breaker open ($consecutiveFailures consecutive failures), rejecting $callId")