diff --git a/samples/CameraAccess/CameraAccess/Gemini/GeminiConfig.swift b/samples/CameraAccess/CameraAccess/Gemini/GeminiConfig.swift index 5c124f66..73a979d6 100644 --- a/samples/CameraAccess/CameraAccess/Gemini/GeminiConfig.swift +++ b/samples/CameraAccess/CameraAccess/Gemini/GeminiConfig.swift @@ -42,6 +42,14 @@ enum GeminiConfig { For messages, confirm recipient and content before delegating unless clearly urgent. """ + static let noToolsSystemInstruction = """ + You are an AI assistant for someone wearing Meta Ray-Ban smart glasses. You can see through their camera and have a voice conversation. Keep responses concise and natural. + + You do NOT have any tools. You cannot send messages, search the web, manage lists, set reminders, or take any actions. You are a voice + vision assistant only. + + If the user asks you to do something that requires an action (send a message, search, add to a list, etc.), let them know that OpenClaw is not connected and they need to set it up in Settings to enable those features. + """ + // User-configurable values (Settings screen overrides, falling back to Secrets.swift) static var apiKey: String { SettingsManager.shared.geminiAPIKey } static var openClawHost: String { SettingsManager.shared.openClawHost } diff --git a/samples/CameraAccess/CameraAccess/Gemini/GeminiLiveService.swift b/samples/CameraAccess/CameraAccess/Gemini/GeminiLiveService.swift index 248f2f02..d23598da 100644 --- a/samples/CameraAccess/CameraAccess/Gemini/GeminiLiveService.swift +++ b/samples/CameraAccess/CameraAccess/Gemini/GeminiLiveService.swift @@ -178,46 +178,53 @@ class GeminiLiveService: ObservableObject { } private func sendSetupMessage() { - let setup: [String: Any] = [ - "setup": [ - "model": GeminiConfig.model, - "generationConfig": [ - "responseModalities": ["AUDIO"], - "thinkingConfig": [ - "thinkingBudget": 0 - ] - ], - "systemInstruction": [ - "parts": [ - ["text": GeminiConfig.systemInstruction] - ] - ], - "tools": [ - [ - "functionDeclarations": ToolDeclarations.allDeclarations() - ] - ], - "realtimeInputConfig": [ - "automaticActivityDetection": [ - "disabled": false, - "startOfSpeechSensitivity": "START_SENSITIVITY_HIGH", - "endOfSpeechSensitivity": "END_SENSITIVITY_LOW", - "silenceDurationMs": 500, - "prefixPaddingMs": 40 - ], - "activityHandling": "START_OF_ACTIVITY_INTERRUPTS", - "turnCoverage": "TURN_INCLUDES_ALL_INPUT" - ], - "contextWindowCompression": [ - "slidingWindow": [ - "targetTokens": 80000 - ] + var setupContent: [String: Any] = [ + "model": GeminiConfig.model, + "generationConfig": [ + "responseModalities": ["AUDIO"], + "thinkingConfig": [ + "thinkingBudget": 0 + ] + ], + "systemInstruction": [ + "parts": [ + ["text": GeminiConfig.isOpenClawConfigured + ? GeminiConfig.systemInstruction + : GeminiConfig.noToolsSystemInstruction] + ] + ], + "realtimeInputConfig": [ + "automaticActivityDetection": [ + "disabled": false, + "startOfSpeechSensitivity": "START_SENSITIVITY_HIGH", + "endOfSpeechSensitivity": "END_SENSITIVITY_LOW", + "silenceDurationMs": 500, + "prefixPaddingMs": 40 ], - "inputAudioTranscription": [:] as [String: Any], - "outputAudioTranscription": [:] as [String: Any] - ] + "activityHandling": "START_OF_ACTIVITY_INTERRUPTS", + "turnCoverage": "TURN_INCLUDES_ALL_INPUT" + ], + "contextWindowCompression": [ + "slidingWindow": [ + "targetTokens": 80000 + ] + ], + "inputAudioTranscription": [:] as [String: Any], + "outputAudioTranscription": [:] as [String: Any] ] - sendJSON(setup) + + // Only declare tools when OpenClaw is configured — otherwise Gemini + // will attempt tool calls that have no backend, getting stuck in "executing" + if GeminiConfig.isOpenClawConfigured { + setupContent["tools"] = [ + ["functionDeclarations": ToolDeclarations.allDeclarations()] + ] + NSLog("[Gemini] Setup with tools (OpenClaw configured)") + } else { + NSLog("[Gemini] Setup without tools (OpenClaw not configured)") + } + + sendJSON(["setup": setupContent]) } private func sendJSON(_ json: [String: Any]) { diff --git a/samples/CameraAccess/CameraAccess/OpenClaw/ToolCallRouter.swift b/samples/CameraAccess/CameraAccess/OpenClaw/ToolCallRouter.swift index a20babf4..a50c0e81 100644 --- a/samples/CameraAccess/CameraAccess/OpenClaw/ToolCallRouter.swift +++ b/samples/CameraAccess/CameraAccess/OpenClaw/ToolCallRouter.swift @@ -23,6 +23,18 @@ class ToolCallRouter { NSLog("[ToolCall] Received: %@ (id: %@) args: %@", callName, callId, String(describing: call.args)) + // Fast-fail if OpenClaw is not configured + if !GeminiConfig.isOpenClawConfigured { + NSLog("[ToolCall] OpenClaw not configured, rejecting tool call %@", callId) + let errorResult: ToolResult = .failure( + "OpenClaw is not configured. Tool calls are unavailable. " + + "Please tell the user to set up OpenClaw in Settings to enable actions like web search, messaging, and more." + ) + let response = buildToolResponse(callId: callId, name: callName, result: errorResult) + sendResponse(response) + return + } + // Circuit breaker: stop sending tool calls after repeated failures if consecutiveFailures >= maxConsecutiveFailures { NSLog("[ToolCall] Circuit breaker open (%d consecutive failures), rejecting %@", diff --git a/samples/CameraAccessAndroid/app/src/main/java/com/meta/wearable/dat/externalsampleapps/cameraaccess/gemini/GeminiConfig.kt b/samples/CameraAccessAndroid/app/src/main/java/com/meta/wearable/dat/externalsampleapps/cameraaccess/gemini/GeminiConfig.kt index 10ba908e..aae15890 100644 --- a/samples/CameraAccessAndroid/app/src/main/java/com/meta/wearable/dat/externalsampleapps/cameraaccess/gemini/GeminiConfig.kt +++ b/samples/CameraAccessAndroid/app/src/main/java/com/meta/wearable/dat/externalsampleapps/cameraaccess/gemini/GeminiConfig.kt @@ -45,4 +45,10 @@ object GeminiConfig { get() = openClawGatewayToken != "YOUR_OPENCLAW_GATEWAY_TOKEN" && openClawGatewayToken.isNotEmpty() && openClawHost != "http://YOUR_MAC_HOSTNAME.local" + + const val NO_TOOLS_SYSTEM_INSTRUCTION = """You are an AI assistant for someone wearing Meta Ray-Ban smart glasses. You can see through their camera and have a voice conversation. Keep responses concise and natural. + +You do NOT have any tools. You cannot send messages, search the web, manage lists, set reminders, or take any actions. You are a voice + vision assistant only. + +If the user asks you to do something that requires an action (send a message, search, add to a list, etc.), let them know that OpenClaw is not connected and they need to set it up in Settings to enable those features.""" } diff --git a/samples/CameraAccessAndroid/app/src/main/java/com/meta/wearable/dat/externalsampleapps/cameraaccess/gemini/GeminiLiveService.kt b/samples/CameraAccessAndroid/app/src/main/java/com/meta/wearable/dat/externalsampleapps/cameraaccess/gemini/GeminiLiveService.kt index d046d306..d2370a1d 100644 --- a/samples/CameraAccessAndroid/app/src/main/java/com/meta/wearable/dat/externalsampleapps/cameraaccess/gemini/GeminiLiveService.kt +++ b/samples/CameraAccessAndroid/app/src/main/java/com/meta/wearable/dat/externalsampleapps/cameraaccess/gemini/GeminiLiveService.kt @@ -211,6 +211,8 @@ class GeminiLiveService { } private fun sendSetupMessage() { + val isOpenClawConfigured = GeminiConfig.isOpenClawConfigured + val setup = JSONObject().apply { put("setup", JSONObject().apply { put("model", GeminiConfig.MODEL) @@ -222,12 +224,22 @@ class GeminiLiveService { }) put("systemInstruction", JSONObject().apply { put("parts", JSONArray().put(JSONObject().apply { - put("text", GeminiConfig.systemInstruction) + put("text", if (isOpenClawConfigured) + GeminiConfig.systemInstruction + else + GeminiConfig.NO_TOOLS_SYSTEM_INSTRUCTION) })) }) - put("tools", JSONArray().put(JSONObject().apply { - put("functionDeclarations", ToolDeclarations.allDeclarationsJSON()) - })) + // Only declare tools when OpenClaw is configured — otherwise Gemini + // will attempt tool calls that have no backend, getting stuck in "executing" + if (isOpenClawConfigured) { + put("tools", JSONArray().put(JSONObject().apply { + put("functionDeclarations", ToolDeclarations.allDeclarationsJSON()) + })) + Log.d(TAG, "Setup with tools (OpenClaw configured)") + } else { + Log.d(TAG, "Setup without tools (OpenClaw not configured)") + } put("realtimeInputConfig", JSONObject().apply { put("automaticActivityDetection", JSONObject().apply { put("disabled", false) diff --git a/samples/CameraAccessAndroid/app/src/main/java/com/meta/wearable/dat/externalsampleapps/cameraaccess/openclaw/ToolCallRouter.kt b/samples/CameraAccessAndroid/app/src/main/java/com/meta/wearable/dat/externalsampleapps/cameraaccess/openclaw/ToolCallRouter.kt index 35337e14..8def5189 100644 --- a/samples/CameraAccessAndroid/app/src/main/java/com/meta/wearable/dat/externalsampleapps/cameraaccess/openclaw/ToolCallRouter.kt +++ b/samples/CameraAccessAndroid/app/src/main/java/com/meta/wearable/dat/externalsampleapps/cameraaccess/openclaw/ToolCallRouter.kt @@ -1,6 +1,7 @@ package com.meta.wearable.dat.externalsampleapps.cameraaccess.openclaw import android.util.Log +import com.meta.wearable.dat.externalsampleapps.cameraaccess.gemini.GeminiConfig import kotlinx.coroutines.CoroutineScope import kotlinx.coroutines.Job import kotlinx.coroutines.launch @@ -28,6 +29,17 @@ class ToolCallRouter( Log.d(TAG, "Received: $callName (id: $callId) args: ${call.args}") + // Fast-fail if OpenClaw is not configured + if (!GeminiConfig.isOpenClawConfigured) { + Log.w(TAG, "OpenClaw not configured, rejecting tool call $callId") + val errorResult = ToolResult.Failure( + "OpenClaw is not configured. Tool calls are unavailable. " + + "Please tell the user to set up OpenClaw in Settings to enable actions like web search, messaging, and more." + ) + sendResponse(buildToolResponse(callId, callName, errorResult)) + return + } + // Circuit breaker: stop sending tool calls after repeated failures if (consecutiveFailures >= MAX_CONSECUTIVE_FAILURES) { Log.d(TAG, "Circuit breaker open ($consecutiveFailures consecutive failures), rejecting $callId")