From 10c4fa26c611968f5f4088b091a6ebd9f33f4176 Mon Sep 17 00:00:00 2001 From: Andrew Walker Date: Thu, 14 May 2026 07:44:46 -0500 Subject: [PATCH 1/2] Add CloakBrowser browser fallback --- docs/BROWSER_PROVIDER_PROTOCOL.md | 22 +++ src/tool/browser.rs | 236 +++++++++++++++++++++++++++++- src/tool/browser_tests.rs | 19 ++- 3 files changed, 266 insertions(+), 11 deletions(-) diff --git a/docs/BROWSER_PROVIDER_PROTOCOL.md b/docs/BROWSER_PROVIDER_PROTOCOL.md index 78641cbbd0..4d590c8f1c 100644 --- a/docs/BROWSER_PROVIDER_PROTOCOL.md +++ b/docs/BROWSER_PROVIDER_PROTOCOL.md @@ -9,6 +9,7 @@ Audience: jcode core, browser bridge authors, adapter authors jcode should expose a single first-class `browser` tool while remaining compatible with multiple browser automation backends: - Firefox Agent Bridge +- CloakBrowser Playwright fallback - Chrome Agent Bridge - Chrome remote debugging / CDP adapters - WebDriver / BiDi adapters @@ -663,6 +664,27 @@ Semantics or qualities that influence jcode behavior: - `extension_required` - `remote_debugging_required` +--- + +## Built-in CloakBrowser fallback + +jcode includes an optional CloakBrowser-backed provider for Chromium/Playwright automation. +It is intentionally a fallback, not the default setup path: + +- `browser="auto"` still prefers the Firefox Agent Bridge when it is ready. +- If Firefox is not ready and the Python `cloakbrowser` module is installed, `auto` falls back to CloakBrowser. +- `browser="chrome"` selects CloakBrowser directly. +- `browser action="setup" browser="chrome"` installs the Python `cloakbrowser` wrapper with `python3 -m pip install cloakbrowser`. +- `JCODE_CLOAKBROWSER_PYTHON` can point at a specific Python interpreter or virtualenv. + +The fallback currently supports the core page actions `open`, `snapshot`, `get_content`, +`screenshot`, `eval`, `click`, `type`, and `wait`. It uses a persistent profile under +`~/.jcode/cloakbrowser/` and enables CloakBrowser's `humanize` behavior. + +Because CloakBrowser downloads and launches a custom Chromium build, users should treat it +as an explicitly trusted optional backend and use it only for legitimate browser automation, +testing, or authorized access. + ### Stability Each feature or method may optionally include a stability tag: diff --git a/src/tool/browser.rs b/src/tool/browser.rs index 03bdd3c703..9a30a3418e 100644 --- a/src/tool/browser.rs +++ b/src/tool/browser.rs @@ -6,10 +6,12 @@ use serde::Deserialize; use serde_json::{Map, Value, json}; use std::path::PathBuf; use std::time::{SystemTime, UNIX_EPOCH}; +use tokio::io::AsyncWriteExt; pub struct BrowserTool; static FIREFOX_PROVIDER: FirefoxBridgeProvider = FirefoxBridgeProvider; +static CLOAK_PROVIDER: CloakBrowserProvider = CloakBrowserProvider; impl BrowserTool { pub fn new() -> Self { @@ -275,7 +277,7 @@ impl Tool for BrowserTool { async fn execute(&self, input: Value, ctx: ToolContext) -> Result { let params: BrowserInput = serde_json::from_value(input)?; - let provider = resolve_provider(params.browser.as_deref())?; + let provider = resolve_provider(params.browser.as_deref()).await?; match params.action.as_str() { "status" => provider.status(&ctx).await, @@ -332,18 +334,246 @@ fn attach_browser_metadata( output } -fn resolve_provider(browser: Option<&str>) -> Result<&'static dyn BrowserProvider> { +async fn resolve_provider(browser: Option<&str>) -> Result<&'static dyn BrowserProvider> { let browser = browser.unwrap_or("auto"); + if browser == "auto" { + if FIREFOX_PROVIDER.ensure_ready().await.is_ok() { + return Ok(&FIREFOX_PROVIDER); + } + if CLOAK_PROVIDER.ensure_ready().await.is_ok() { + return Ok(&CLOAK_PROVIDER); + } + return Ok(&FIREFOX_PROVIDER); + } if FIREFOX_PROVIDER.supported_browsers().contains(&browser) { return Ok(&FIREFOX_PROVIDER); } + if CLOAK_PROVIDER.supported_browsers().contains(&browser) { + return Ok(&CLOAK_PROVIDER); + } anyhow::bail!( - "Browser backend '{}' is not wired into the built-in browser tool yet. Use auto/firefox for now.", + "Browser backend '{}' is not wired into the built-in browser tool yet. Use auto/firefox/chrome for now.", browser ) } +struct CloakBrowserProvider; + +#[async_trait] +impl BrowserProvider for CloakBrowserProvider { + fn id(&self) -> &'static str { + "cloakbrowser_playwright" + } + fn supported_browsers(&self) -> &'static [&'static str] { + &["chrome"] + } + + async fn status(&self, _ctx: &ToolContext) -> Result { + match cloak_python_check().await { + Ok(version) => Ok(ToolOutput::new(format!("CloakBrowser fallback is available via Python module cloakbrowser ({}).", version)) + .with_title("browser status") + .with_metadata(json!({"ready": true, "backend": self.id(), "browser": "chrome", "module_installed": true}))), + Err(err) => Ok(ToolOutput::new(format!("CloakBrowser fallback is not available yet. Install it with `python3 -m pip install cloakbrowser`, or run browser action='setup' with browser='chrome'.\n\n{}", err)) + .with_title("browser status") + .with_metadata(json!({"ready": false, "backend": self.id(), "browser": "chrome", "module_installed": false}))), + } + } + + async fn setup(&self) -> Result { + let output = tokio::process::Command::new(cloak_python_bin()) + .args(["-m", "pip", "install", "cloakbrowser"]) + .stdin(std::process::Stdio::null()) + .output() + .await + .context("failed to run python3 -m pip install cloakbrowser")?; + let mut log = String::new(); + log.push_str(&String::from_utf8_lossy(&output.stdout)); + log.push_str(&String::from_utf8_lossy(&output.stderr)); + let ready = output.status.success() && cloak_python_check().await.is_ok(); + Ok(ToolOutput::new(log) + .with_title(if ready { + "browser setup" + } else { + "browser setup (incomplete)" + }) + .with_metadata(json!({ + "ready": ready, "backend": self.id(), "browser": "chrome", "module_installed": ready + }))) + } + + async fn ensure_ready(&self) -> Result> { + cloak_python_check().await.map(|_| None) + } + + async fn execute( + &self, + action: &str, + input: &BrowserInput, + ctx: &ToolContext, + ) -> Result { + let result = cloak_run_action(action, input, ctx).await?; + if action == "screenshot" { + return cloak_screenshot_output(result, self.id(), "chrome").await; + } + Ok(attach_browser_metadata( + render_browser_output(action, format!("browser {}", action), result), + self.id(), + "chrome", + )) + } +} + +async fn cloak_screenshot_output( + result: Value, + backend: &'static str, + browser: &'static str, +) -> Result { + let saved = result + .get("saved") + .and_then(|v| v.as_str()) + .map(PathBuf::from); + let mut output = ToolOutput::new(match &saved { + Some(path) => format!("Captured browser screenshot to {}.", path.display()), + None => "Captured browser screenshot.".to_string(), + }) + .with_title("browser screenshot") + .with_metadata(result.clone()); + + if let Some(path) = saved + && let Ok(bytes) = tokio::fs::read(&path).await + { + output = output.with_labeled_image( + "image/png", + STANDARD.encode(&bytes), + format!("browser screenshot: {}", path.display()), + ); + let _ = tokio::fs::remove_file(path).await; + } + + Ok(attach_browser_metadata(output, backend, browser)) +} + +fn cloak_python_bin() -> String { + std::env::var("JCODE_CLOAKBROWSER_PYTHON").unwrap_or_else(|_| "python3".to_string()) +} + +async fn cloak_python_check() -> Result { + let output = tokio::process::Command::new(cloak_python_bin()) + .args([ + "-c", + "import cloakbrowser; print(getattr(cloakbrowser, '__version__', 'installed'))", + ]) + .stdin(std::process::Stdio::null()) + .output() + .await?; + if output.status.success() { + Ok(String::from_utf8_lossy(&output.stdout).trim().to_string()) + } else { + anyhow::bail!(String::from_utf8_lossy(&output.stderr).trim().to_string()) + } +} + +async fn cloak_run_action(action: &str, input: &BrowserInput, ctx: &ToolContext) -> Result { + if !matches!( + action, + "open" | "snapshot" | "get_content" | "screenshot" | "eval" | "click" | "type" | "wait" + ) { + anyhow::bail!( + "CloakBrowser fallback currently supports open, snapshot, get_content, screenshot, eval, click, type, and wait. Use Firefox bridge for '{}'.", + action + ); + } + let request = json!({ + "action": action, + "url": input.url, + "selector": input.selector, + "text": input.text, + "script": input.script, + "format": input.format, + "wait": input.wait, + "timeout_ms": input.timeout_ms, + "screenshot_path": if action == "screenshot" { Some(temp_screenshot_path().to_string_lossy().to_string()) } else { None:: }, + "profile_dir": cloak_profile_dir(&ctx.session_id).to_string_lossy().to_string(), + }); + let mut child = tokio::process::Command::new(cloak_python_bin()) + .arg("-") + .stdin(std::process::Stdio::piped()) + .stdout(std::process::Stdio::piped()) + .stderr(std::process::Stdio::piped()) + .spawn() + .context("failed to start CloakBrowser Python helper")?; + let script = format!( + "{}\nREQ = {}\nmain(REQ)\n", + CLOAK_HELPER_PY, + serde_json::to_string(&request)? + ); + child + .stdin + .as_mut() + .unwrap() + .write_all(script.as_bytes()) + .await?; + drop(child.stdin.take()); + let output = child.wait_with_output().await?; + let stdout = String::from_utf8_lossy(&output.stdout).trim().to_string(); + let stderr = String::from_utf8_lossy(&output.stderr).trim().to_string(); + if !output.status.success() { + anyhow::bail!(if stderr.is_empty() { stdout } else { stderr }); + } + serde_json::from_str(&stdout).or_else(|_| Ok(json!({"raw": stdout}))) +} + +fn cloak_profile_dir(session_id: &str) -> PathBuf { + dirs::home_dir() + .unwrap_or_else(std::env::temp_dir) + .join(".jcode") + .join("cloakbrowser") + .join(session_id) +} + +const CLOAK_HELPER_PY: &str = r#" +import json, pathlib, sys +from cloakbrowser import launch_persistent_context + +def main(req): + pathlib.Path(req['profile_dir']).mkdir(parents=True, exist_ok=True) + ctx = launch_persistent_context(req['profile_dir'], headless=True, humanize=True) + page = ctx.pages[0] if ctx.pages else ctx.new_page() + try: + action = req['action'] + timeout = req.get('timeout_ms') or 30000 + if req.get('url') and action != 'open': + page.goto(req['url'], wait_until='domcontentloaded', timeout=timeout) + if action == 'open': + page.goto(req['url'], wait_until='domcontentloaded', timeout=timeout) + result = {'ok': True, 'url': page.url, 'title': page.title()} + elif action in ('snapshot', 'get_content'): + fmt = 'annotated' if action == 'snapshot' else (req.get('format') or 'text') + content = page.content() if fmt == 'html' else page.locator('body').inner_text(timeout=timeout) + result = {'content': content, 'url': page.url, 'title': page.title(), 'format': fmt} + elif action == 'screenshot': + page.screenshot(path=req['screenshot_path'], full_page=True, timeout=timeout) + result = {'saved': req['screenshot_path'], 'url': page.url, 'title': page.title()} + elif action == 'eval': + result = {'result': page.evaluate(req['script']), 'url': page.url} + elif action == 'click': + page.click(req['selector'], timeout=timeout) + result = {'ok': True, 'url': page.url} + elif action == 'type': + page.fill(req['selector'], req.get('text') or '', timeout=timeout) + result = {'ok': True, 'url': page.url} + elif action == 'wait': + if req.get('selector'): + page.wait_for_selector(req['selector'], timeout=timeout) + elif req.get('text'): + page.get_by_text(req['text']).wait_for(timeout=timeout) + result = {'ok': True, 'url': page.url} + print(json.dumps(result)) + finally: + ctx.close() +"#; + async fn firefox_status( provider: &FirefoxBridgeProvider, _ctx: &ToolContext, diff --git a/src/tool/browser_tests.rs b/src/tool/browser_tests.rs index eb658da045..290ce7cb2e 100644 --- a/src/tool/browser_tests.rs +++ b/src/tool/browser_tests.rs @@ -169,17 +169,20 @@ fn schema_exposes_advanced_browser_fields() { assert!(props.contains_key("scroll_to")); } -#[test] -fn resolve_provider_accepts_auto_and_firefox() { - assert!(resolve_provider(Some("auto")).is_ok()); - assert!(resolve_provider(Some("firefox")).is_ok()); +#[tokio::test] +async fn resolve_provider_accepts_auto_firefox_and_chrome() { + assert!(resolve_provider(Some("auto")).await.is_ok()); + assert!(resolve_provider(Some("firefox")).await.is_ok()); + let chrome = resolve_provider(Some("chrome")).await.unwrap(); + assert_eq!(chrome.id(), "cloakbrowser_playwright"); } -#[test] -fn resolve_provider_rejects_unsupported_browser() { - let err = resolve_provider(Some("chrome")) +#[tokio::test] +async fn resolve_provider_rejects_unsupported_browser() { + let err = resolve_provider(Some("edge")) + .await .err() - .expect("chrome should not resolve yet"); + .expect("edge should not resolve yet"); assert!( err.to_string() .contains("not wired into the built-in browser tool") From 97a396246f5fabe2255040016731da6e940385d7 Mon Sep 17 00:00:00 2001 From: Andrew Walker Date: Fri, 5 Jun 2026 19:13:09 -0500 Subject: [PATCH 2/2] fix(tui): sync copy selection via OSC52 --- src/tui/app/helpers.rs | 57 ++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 55 insertions(+), 2 deletions(-) diff --git a/src/tui/app/helpers.rs b/src/tui/app/helpers.rs index 0e232656bb..f56b23b467 100644 --- a/src/tui/app/helpers.rs +++ b/src/tui/app/helpers.rs @@ -247,8 +247,15 @@ pub(super) fn format_tokens(tokens: u64) -> String { } } -/// Copy text to clipboard, trying wl-copy first (Wayland), then arboard as fallback. +/// Copy text to clipboard. +/// +/// In browser-backed terminals such as cmux, the process clipboard APIs can +/// succeed while only updating the remote host clipboard. Emit OSC 52 as well +/// so the terminal frontend can synchronize the user's local clipboard when it +/// supports clipboard escape sequences. pub(super) fn copy_to_clipboard(text: &str) -> bool { + let osc52_success = copy_to_terminal_clipboard_osc52(text); + if let Ok(mut child) = std::process::Command::new("wl-copy") .stdin(std::process::Stdio::piped()) .stdout(std::process::Stdio::null()) @@ -260,12 +267,58 @@ pub(super) fn copy_to_clipboard(text: &str) -> bool { && stdin.write_all(text.as_bytes()).is_ok() { drop(child.stdin.take()); - return child.wait().map(|s| s.success()).unwrap_or(false); + return child.wait().map(|s| s.success()).unwrap_or(false) || osc52_success; } } arboard::Clipboard::new() .and_then(|mut cb| cb.set_text(text.to_string())) .is_ok() + || osc52_success +} + +fn copy_to_terminal_clipboard_osc52(text: &str) -> bool { + use std::io::Write; + + let sequence = osc52_clipboard_sequence(text); + let mut stdout = std::io::stdout().lock(); + stdout.write_all(sequence.as_bytes()).is_ok() && stdout.flush().is_ok() +} + +fn osc52_clipboard_sequence(text: &str) -> String { + osc52_clipboard_sequence_for_tmux(text, std::env::var_os("TMUX").is_some()) +} + +fn osc52_clipboard_sequence_for_tmux(text: &str, in_tmux: bool) -> String { + use base64::Engine; + + let encoded = base64::engine::general_purpose::STANDARD.encode(text.as_bytes()); + let osc = format!("\x1b]52;c;{encoded}\x07"); + if in_tmux { + format!("\x1bPtmux;\x1b{osc}\x1b\\") + } else { + osc + } +} + +#[cfg(test)] +mod clipboard_tests { + use super::osc52_clipboard_sequence_for_tmux; + + #[test] + fn osc52_clipboard_sequence_encodes_text() { + assert_eq!( + osc52_clipboard_sequence_for_tmux("hello", false), + "\x1b]52;c;aGVsbG8=\x07" + ); + } + + #[test] + fn osc52_clipboard_sequence_wraps_for_tmux() { + assert_eq!( + osc52_clipboard_sequence_for_tmux("hello", true), + "\x1bPtmux;\x1b\x1b]52;c;aGVsbG8=\x07\x1b\\" + ); + } } pub(super) fn effort_display_label(effort: &str) -> &str {