diff --git a/bt-daemon/README.md b/bt-daemon/README.md index 0a1c5127..12cbff14 100644 --- a/bt-daemon/README.md +++ b/bt-daemon/README.md @@ -293,6 +293,16 @@ transcript are not invented. Muse Code 1.3+ supports live tracing, managed runs, session import, and attach on macOS and Linux. Use `import muse --all` to import completed sessions. +Muse image attachments appear in the Turn input when the local session log and +its `input-assets/v1/sha256` files are retained (verified with Muse 1.4.14). +Capture reads the last 8 MiB of the session log and includes up to 8 MiB of image +bytes per turn, verifying each asset's size and SHA-256 digest. Images use the +same data URL format as the other agent translators, allowing Braintrust's +backend attachment conversion to produce previews. Missing, corrupt, oversized, +or unavailable assets leave the text prompt intact. Export JSON alone does not +include asset bytes; import and attach need the original local Muse session +store. Muse's LLM hook summaries still contain image placeholders. + Add `--attach` to keep following an active Codex, Claude, Antigravity, or Muse session until Ctrl-C. Detaching from Muse leaves the session running; attach again or import it later to collect its completion. diff --git a/bt-daemon/src/hook.rs b/bt-daemon/src/hook.rs index 9f5feb35..f4f6204f 100644 --- a/bt-daemon/src/hook.rs +++ b/bt-daemon/src/hook.rs @@ -222,6 +222,12 @@ async fn capture_hook( .or_else(|| json_str_field(&payload, &args.event_field)) .unwrap_or_default(); + if crate::translate::canonical_source_name(&args.source) == "muse" + && event == "UserPromptSubmit" + { + crate::muse::images::enrich(&mut payload); + } + if args.flush_on_turn_end { route.flush_mode = wire::FlushMode::FlushOnTurnEnd; } diff --git a/bt-daemon/src/muse/hooks.rs b/bt-daemon/src/muse/hooks.rs index 99f79522..9587250a 100644 --- a/bt-daemon/src/muse/hooks.rs +++ b/bt-daemon/src/muse/hooks.rs @@ -135,6 +135,8 @@ impl SessionMetadata { #[derive(Debug, Clone, Default, Serialize, Deserialize)] pub(crate) struct PromptSubmitted { pub prompt: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub image_content: Option, } #[derive(Debug, Clone, Default, Serialize, Deserialize)] pub(crate) struct ModelDetails { diff --git a/bt-daemon/src/muse/images.rs b/bt-daemon/src/muse/images.rs new file mode 100644 index 00000000..c0876d75 --- /dev/null +++ b/bt-daemon/src/muse/images.rs @@ -0,0 +1,162 @@ +//! Recover image bytes from Muse's content-addressed session assets, never source paths. +use base64::Engine; +use serde_json::{json, Value}; +use sha2::{Digest, Sha256}; +use std::io::{Read, Seek, SeekFrom}; +use std::path::PathBuf; + +const MAX_BYTES: u64 = 8 * 1024 * 1024; + +fn session_dir(session: &str) -> Option { + let id = uuid::Uuid::parse_str(session).ok()?; + let millis = (id.as_u128() >> 80) as i64; + let date = chrono::DateTime::from_timestamp_millis(millis)?; + let root = std::env::var_os("XDG_DATA_HOME") + .map(PathBuf::from) + .unwrap_or_else(|| { + PathBuf::from( + std::env::var_os("HOME") + .or_else(|| std::env::var_os("USERPROFILE")) + .unwrap_or_default(), + ) + .join(".local/share") + }); + Some( + root.join("muse/sessions") + .join(date.format("%Y/%m/%d").to_string()) + .join(session), + ) +} + +pub(crate) fn content(session: &str, turn: &str) -> Option { + let dir = session_dir(session)?; + content_at(&dir, turn) +} + +fn content_at(dir: &std::path::Path, turn: &str) -> Option { + let mut file = std::fs::File::open(dir.join("session.jsonl")).ok()?; + let len = file.metadata().ok()?.len(); + let start = len.saturating_sub(MAX_BYTES); + file.seek(SeekFrom::Start(start)).ok()?; + let mut bytes = Vec::new(); + file.take(MAX_BYTES).read_to_end(&mut bytes).ok()?; + let lines = bytes.split(|b| *b == b'\n'); + for line in lines.rev() { + let Ok(record) = serde_json::from_slice::(line) else { + continue; + }; + if record["payload_type"] != "runtime.user_intent.accepted" { + continue; + } + let payload = &record["payload"]; + if payload["intent_id"] != turn { + continue; + } + return asset_content(dir, payload["refill_blocks"].as_array()?); + } + None +} + +fn asset_content(dir: &std::path::Path, blocks: &[Value]) -> Option { + let mut content = Vec::new(); + let mut found = false; + let mut total = 0; + for block in blocks { + if block["kind"] == "text" { + if let Some(text) = block["text"].as_str() { + content.push(json!({"type":"text","text":text})); + } + } else if block["kind"] == "asset" { + let asset = &block["asset"]; + let digest = asset["digest"].as_str()?; + let mime = asset["media_type"].as_str()?; + let size = asset["byte_length"].as_u64()?; + total += size.min(MAX_BYTES + 1); + if asset["schema_version"] != 2 + || asset["algorithm"] != "sha256" + || digest.len() != 64 + || !digest.bytes().all(|b| b.is_ascii_hexdigit()) + || !mime.starts_with("image/") + || total > MAX_BYTES + { + return None; + } + let path = dir + .join("input-assets/v1/sha256") + .join(&digest[..2]) + .join(digest); + let file = std::fs::File::open(path).ok()?; + if !file.metadata().ok()?.is_file() { + return None; + } + let mut bytes = Vec::new(); + file.take(MAX_BYTES + 1).read_to_end(&mut bytes).ok()?; + if bytes.len() as u64 != size || format!("{:x}", Sha256::digest(&bytes)) != digest { + return None; + } + let data = base64::engine::general_purpose::STANDARD.encode(bytes); + content.push(json!({"type":"image_url","image_url":{"url":format!("data:{mime};base64,{data}")}})); + found = true; + } + } + found.then_some(Value::Array(content)) +} + +pub(crate) fn enrich(payload: &mut Value) { + if !payload["prompt"] + .as_str() + .is_some_and(|text| text.contains("[Image #")) + { + return; + } + let Some(session) = payload["session_id"].as_str() else { + return; + }; + let Some(turn) = payload["turn_id"].as_str() else { + return; + }; + if let Some(content) = content(session, turn) { + payload["image_content"] = content; + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn captured_asset_uses_durable_bytes_and_rejects_corruption() { + let dir = tempfile::tempdir().unwrap(); + let bytes = base64::engine::general_purpose::STANDARD.decode("iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mNk+A8AAQUBAScY42YAAAAASUVORK5CYII=").unwrap(); + let digest = format!("{:x}", Sha256::digest(&bytes)); + let parent = dir.path().join("input-assets/v1/sha256").join(&digest[..2]); + std::fs::create_dir_all(&parent).unwrap(); + let path = parent.join(&digest); + std::fs::write(&path, &bytes).unwrap(); + // Captured Muse 1.4.14 runtime.user_intent.accepted refill blocks. + let blocks = json!([ + {"kind":"text","text":"[Image #1] Describe this image"}, + {"kind":"asset","asset":{"schema_version":2,"algorithm":"sha256", + "digest":digest,"media_type":"image/png","byte_length":68, + "width":1,"height":1,"source_path":"/never/read/the/original.png"}} + ]); + let content = asset_content(dir.path(), blocks.as_array().unwrap()).unwrap(); + assert_eq!(content[0]["text"], "[Image #1] Describe this image"); + assert_eq!(content[1]["type"], "image_url"); + assert!(content[1]["image_url"]["url"] + .as_str() + .unwrap() + .starts_with("data:image/png;base64,")); + let record = json!({"payload_type":"runtime.user_intent.accepted", "payload":{"intent_id":"turn-image", "refill_blocks":blocks}}); + std::fs::write(dir.path().join("session.jsonl"), format!("{}\n", record)).unwrap(); + assert_eq!(content_at(dir.path(), "turn-image"), Some(content)); + assert!(content_at(dir.path(), "another-turn").is_none()); + std::fs::write(&path, vec![0; 68]).unwrap(); + assert!(asset_content(dir.path(), blocks.as_array().unwrap()).is_none()); + std::fs::remove_file(&path).unwrap(); + assert!(asset_content(dir.path(), blocks.as_array().unwrap()).is_none()); + let mut invalid = blocks.clone(); + invalid[1]["asset"]["digest"] = json!("../../secret"); + assert!(asset_content(dir.path(), invalid.as_array().unwrap()).is_none()); + } +} diff --git a/bt-daemon/src/muse/mod.rs b/bt-daemon/src/muse/mod.rs index c8a1ab98..0d24389b 100644 --- a/bt-daemon/src/muse/mod.rs +++ b/bt-daemon/src/muse/mod.rs @@ -5,6 +5,7 @@ pub(crate) mod export_v1; pub(crate) mod field; pub(crate) mod hooks; pub(crate) mod ids; +pub(crate) mod images; pub(crate) mod messages; pub(crate) mod metadata; pub(crate) mod msp; diff --git a/bt-daemon/src/transcript_import/muse.rs b/bt-daemon/src/transcript_import/muse.rs index 558085cd..edd6737b 100644 --- a/bt-daemon/src/transcript_import/muse.rs +++ b/bt-daemon/src/transcript_import/muse.rs @@ -176,7 +176,10 @@ impl Result<()>> Reader<'_, F> { ts, MuseEvent::UserPromptSubmit(Hook { context: self.context(Some(run_id), None), - body: PromptSubmitted { prompt: None }, + body: PromptSubmitted { + prompt: None, + ..Default::default() + }, }), ) } @@ -334,7 +337,15 @@ impl Result<()>> Reader<'_, F> { ts, MuseEvent::UserPromptSubmit(Hook { context: self.context(Some(&run_id), None), - body: PromptSubmitted { prompt }, + body: PromptSubmitted { + prompt: prompt.clone(), + image_content: prompt + .as_deref() + .filter(|text| text.contains("[Image #")) + .and_then(|_| { + crate::muse::images::content(&self.session_id, &run_id) + }), + }, }), )?; } diff --git a/bt-daemon/src/translate/muse.rs b/bt-daemon/src/translate/muse.rs index ff757aee..46141887 100644 --- a/bt-daemon/src/translate/muse.rs +++ b/bt-daemon/src/translate/muse.rs @@ -387,7 +387,7 @@ impl AgentTranslator for MuseTranslator { name: "Turn".into(), span_type: SpanType::Task, start_ms: Some(e.ts_ms), - input: hook.body.prompt.map(serde_json::to_value).transpose()?, + input: hook.body.image_content.map(|content| serde_json::json!([{ "role": "user", "content": content }])).or_else(|| hook.body.prompt.map(Value::String)), is_turn: true, turn_span_id: Some(turn_span_id), ..Default::default() @@ -497,6 +497,33 @@ mod tests { } } + #[test] + fn turn_input_preserves_image_messages() { + let mut translator = MuseTranslator::new( + "muse-fixture-session", + Arc::new(GitMetadataCache::default()), + ); + let content = json!([{"type":"text","text":"Describe this image"}, {"type":"image_url","image_url":{"url":"data:image/png;base64,aGVsbG8="}}]); + let rows = translator + .handle( + &event( + "UserPromptSubmit", + json!({"turn_id":"t1","prompt":"Describe this image","image_content":content}), + 1, + ), + &context(), + ) + .unwrap(); + let row = rows + .iter() + .find_map(|op| match op { + SpanOp::Insert(row) if row.name == "Turn" => Some(row), + _ => None, + }) + .unwrap(); + assert_eq!(row.input.as_ref().unwrap()[0]["content"], content); + } + fn fixture() -> Vec { include_str!("../../tests/fixtures/muse-hooks-1.1.1.jsonl") .lines()