diff --git a/.changeset/sep-directive-and-images.md b/.changeset/sep-directive-and-images.md new file mode 100644 index 00000000..99c3572c --- /dev/null +++ b/.changeset/sep-directive-and-images.md @@ -0,0 +1,9 @@ +--- +"@smooai/smooth-operator": minor +--- + +Two additive SEP-protocol enhancements on the streaming path (directive nav + business-card images), both optional and back-compatible. + +**Directive-over-SEP.** `eventual_response` gains an optional `directive` field — an opaque client-side directive (e.g. a Navigate / ApplyView instruction) a host tool emitted this turn. The runner threads a `directive_sink` into the `ToolProviderContext` (new `with_directive_sink` builder), drains it after the turn (last-write-wins, mirroring the citation sink), and carries the value onto `TurnResult::directive`. The protocol layer never interprets the shape — the host client owns it, exactly like `response`. Absent when no host tool wrote one, so the event is byte-for-byte unchanged for existing clients. Added to `spec/events/eventual-response.schema.json` and `spec/actions/send-message.schema.json` `$defs/Response`, and to the TypeScript SDK. + +**Image-through-SEP.** `send_message` gains an optional `images` array (`{ url, detail? }`) for multimodal turns. A new facade `UserImage` type flows from the inbound request into `TurnRequest::images` and the `ToolProviderContext` (new `with_images` builder); when non-empty the runner maps each onto a core `ImageContent` and attaches them to the engine's user message via `AgentConfig::with_user_images` (requires core `0.16.2`). Parsing is fail-soft (a malformed `images` entry is dropped, never rejects the turn). Empty/absent ⇒ a text-only turn, unchanged. Added to `spec/actions/send-message.schema.json` `$defs/Request` and the TypeScript SDK. diff --git a/rust/examples/dev-support/tests/serve_smoke.rs b/rust/examples/dev-support/tests/serve_smoke.rs index 667ce97c..a37b5c89 100644 --- a/rust/examples/dev-support/tests/serve_smoke.rs +++ b/rust/examples/dev-support/tests/serve_smoke.rs @@ -364,6 +364,7 @@ async fn grounded_turn_over_served_storage_answers_from_the_ingested_repo() { conversation_id: &conversation_id, request_id: "sm-grounded", user_message: "How big is the Frobnicator's ring buffer?", + model_max_output: None, access: AccessContext::anonymous(), llm_provider: Some(Arc::new(mock.clone())), reranker: None, @@ -380,6 +381,7 @@ async fn grounded_turn_over_served_storage_answers_from_the_ingested_repo() { auth_gate: None, tool_configs: None, extensions: None, + images: vec![], }, &tx, ) diff --git a/rust/smooth-operator-lambda/src/dispatch.rs b/rust/smooth-operator-lambda/src/dispatch.rs index 78a9af8f..a590cc84 100644 --- a/rust/smooth-operator-lambda/src/dispatch.rs +++ b/rust/smooth-operator-lambda/src/dispatch.rs @@ -571,6 +571,8 @@ async fn send_message( auth_gate: None, tool_configs: None, extensions: None, + // The lambda flavor is text-only; no multimodal attachments. + images: vec![], }, &tx, ) @@ -593,6 +595,7 @@ async fn send_message( false, &turn.citations, turn.usage, + turn.directive, )) .await?; } diff --git a/rust/smooth-operator-server/src/handler.rs b/rust/smooth-operator-server/src/handler.rs index bdc091bb..fb0c47cd 100644 --- a/rust/smooth-operator-server/src/handler.rs +++ b/rust/smooth-operator-server/src/handler.rs @@ -1046,6 +1046,13 @@ async fn handle_send_message( } }; + // Optional multimodal attachments. Fail-soft: absent ⇒ text-only; a malformed + // `images` array is dropped rather than rejecting the turn (per the schema). + let images: Vec = parsed + .get("images") + .and_then(|v| serde_json::from_value(v.clone()).ok()) + .unwrap_or_default(); + let Some(session) = state.get_session(session_id) else { let _ = sink.send(protocol::error( Some(request_id), @@ -1385,6 +1392,8 @@ async fn handle_send_message( tool_configs, // SEP — the per-turn extension host (None unless allowlisted). extensions, + // Optional multimodal attachments (empty ⇒ text-only, unchanged). + images, }, &sink_owned, ) @@ -1452,6 +1461,7 @@ async fn handle_send_message( false, &turn.citations, turn.usage, + turn.directive, )); // Best-effort auto-title (fires only while the conversation is diff --git a/rust/smooth-operator-server/src/protocol.rs b/rust/smooth-operator-server/src/protocol.rs index 15c38416..1a6d0c0c 100644 --- a/rust/smooth-operator-server/src/protocol.rs +++ b/rust/smooth-operator-server/src/protocol.rs @@ -138,6 +138,12 @@ pub struct TurnUsage { /// client can accumulate live session cost. Absent when the engine reported no /// usage (e.g. an offline mock turn), keeping the event back-compatible with /// clients that predate cost reporting. +/// +/// `directive`, when `Some`, attaches an opaque client-side directive a host tool +/// emitted this turn as a sibling `data.data.directive` value. The protocol layer +/// never interprets the shape (the host client owns it, like `response`). Absent +/// when the turn produced no directive, keeping the event back-compatible with +/// clients that predate directives. #[must_use] pub fn eventual_response( request_id: &str, @@ -147,6 +153,7 @@ pub fn eventual_response( needs_escalation: bool, citations: &[smooth_operator::domain::Citation], usage: Option, + directive: Option, ) -> Value { let mut inner = json!({ "messageId": message_id, @@ -165,6 +172,10 @@ pub fn eventual_response( "completionTokens": usage.completion_tokens, }); } + // Optional + back-compat: only emit `directive` when a host tool wrote one. + if let Some(directive) = directive { + inner["directive"] = directive; + } json!({ "type": "eventual_response", "requestId": request_id, @@ -435,7 +446,10 @@ mod tests { assert_eq!(ev["type"], "stream_preamble"); // …but shaped exactly like stream_token so they can reuse the render path. assert_eq!(ev["token"], "Let me pull up your recent conversations."); - assert_eq!(ev["data"]["token"], "Let me pull up your recent conversations."); + assert_eq!( + ev["data"]["token"], + "Let me pull up your recent conversations." + ); assert_eq!(ev["data"]["requestId"], "r1"); } @@ -458,6 +472,7 @@ mod tests { false, &[], None, + None, ); assert_eq!(ev["type"], "eventual_response"); assert_eq!(ev["status"], 200); @@ -477,6 +492,7 @@ mod tests { false, &[], None, + None, ); assert!( ev["data"]["data"].get("citations").is_none(), @@ -495,6 +511,7 @@ mod tests { false, &[], None, + None, ); assert!( ev["data"]["data"].get("usage").is_none(), @@ -517,6 +534,7 @@ mod tests { false, &[], Some(usage), + None, ); let u = &ev["data"]["data"]["usage"]; assert!( @@ -555,6 +573,7 @@ mod tests { false, &citations, None, + None, ); let cites = &ev["data"]["data"]["citations"]; assert!(cites.is_array(), "citations should be an array"); @@ -583,6 +602,41 @@ mod tests { assert_eq!(cites[1]["id"], "doc-2"); } + #[test] + fn eventual_response_omits_directive_when_none() { + // Back-compat: no `directive` key at all when no host tool wrote one. + let ev = eventual_response( + "r1", + 200, + "m1", + json!({"responseParts": ["hi"]}), + false, + &[], + None, + None, + ); + assert!( + ev["data"]["data"].get("directive").is_none(), + "directive must be absent when None for back-compat" + ); + } + + #[test] + fn eventual_response_attaches_directive_when_present() { + let directive = json!({"kind": "Navigate", "path": "/crm/contacts/42"}); + let ev = eventual_response( + "r1", + 200, + "m1", + json!({"responseParts": ["hi"]}), + false, + &[], + None, + Some(directive.clone()), + ); + assert_eq!(ev["data"]["data"]["directive"], directive); + } + #[test] fn write_confirmation_required_matches_spec_shape() { let ev = write_confirmation_required( diff --git a/rust/smooth-operator-server/src/runner.rs b/rust/smooth-operator-server/src/runner.rs index 16547280..e1b4febc 100644 --- a/rust/smooth-operator-server/src/runner.rs +++ b/rust/smooth-operator-server/src/runner.rs @@ -216,6 +216,12 @@ pub struct TurnResult { /// [`crate::suggestions`]). Empty when the model emitted none. Carried onto /// the `eventual_response`'s `suggestedNextActions`. pub suggested_next_actions: Vec, + /// An optional client-side directive a host tool emitted this turn (drained + /// from the [`ToolProviderContext::directive_sink`]). Opaque + /// `serde_json::Value` (last-write-wins) — carried onto the + /// `eventual_response`'s `directive` field. `None` when no host tool wrote + /// one (or no tool provider was installed), keeping the event back-compatible. + pub directive: Option, } /// One tool call captured during the turn, used to emit a `gen_ai.tool` child @@ -345,6 +351,12 @@ pub struct TurnRequest<'a> { /// [`crate::extensions::build_extension_host`] (only when /// `SMOOTH_EXTENSIONS_ALLOW` is non-empty). pub extensions: Option, + /// Image attachments for a multimodal turn. When non-empty, the runner maps + /// each onto a core `ImageContent` and attaches them to the engine's user + /// message via `AgentConfig::with_user_images`, and also carries them into the + /// [`ToolProviderContext`] so a host tool can see them. Empty (the default) + /// ⇒ a text-only turn, byte-for-byte unchanged. + pub images: Vec, } /// Runs one knowledge-grounded, streaming turn for a session's conversation and @@ -393,6 +405,7 @@ pub async fn run_streaming_turn( auth_gate, tool_configs, extensions, + images, } = req; // Capture the OTel turn-span attributes up front, since `llm` is moved into @@ -437,6 +450,11 @@ pub async fn run_streaming_turn( // Sink the knowledge_search tool records its structured results into, for // citations built from the sources the agent's searches surfaced. let tool_sources: KnowledgeResultSink = Arc::new(Mutex::new(Vec::new())); + // Slot a host tool writes a client-side directive into this turn. Drained + // after the run into `TurnResult::directive` (Null ⇒ absent). Only threaded + // into the `ToolProviderContext` when a provider is installed, so with no + // host tools it stays `Null` and the `eventual_response` omits `directive`. + let directive_sink = Arc::new(Mutex::new(serde_json::Value::Null)); // 1. Load prior turns for memory BEFORE persisting the new inbound message, // so prior_messages is exactly the history-up-to-now. @@ -481,12 +499,26 @@ pub async fn run_streaming_turn( // that emits no trailer costs nothing and yields empty suggestions. sections.push(crate::suggestions::SUGGESTED_REPLIES_PROMPT_SECTION.to_string()); let resolved_prompt = sections.join("\n\n"); - let config = AgentConfig::new("smooth-agent-chat", &resolved_prompt, llm) + let mut config = AgentConfig::new("smooth-agent-chat", &resolved_prompt, llm) .with_max_iterations(max_iterations) .with_knowledge(Arc::clone(&knowledge)) .with_prior_messages(prior) // Clamp max_tokens to the model's output ceiling (None ⇒ unclamped). .with_model_ceiling(model_max_output); + // Multimodal turn: attach the turn's images to the engine's user message as + // OpenAI `image_url` content parts. Empty (the default) leaves the turn + // text-only — byte-for-byte unchanged. + if !images.is_empty() { + config = config.with_user_images( + images + .iter() + .map(|img| smooth_operator_core::conversation::ImageContent { + url: img.url.clone(), + detail: img.detail.clone(), + }) + .collect(), + ); + } let mut tools = ToolRegistry::new(); // Build the knowledge_search tool over the SAME ACL-filtered handle, with the @@ -553,8 +585,13 @@ pub async fn run_streaming_turn( // Thread the per-turn handles the runner already has — the conversation // this turn runs in and the resolved per-org gateway key — so a host's // conversation-persisting / retrieval tools aren't degraded to no-ops. - let mut ctx = - ToolProviderContext::new(org_id, access.clone()).with_conversation_id(conversation_id); + let mut ctx = ToolProviderContext::new(org_id, access.clone()) + .with_conversation_id(conversation_id) + // Directive-over-SEP: give host tools the slot to write a client-side + // directive; drained after the turn onto `eventual_response.directive`. + .with_directive_sink(Arc::clone(&directive_sink)) + // Multimodal: let a host tool see the turn's image attachments. + .with_images(images.clone()); if let Some(key) = gateway_key { ctx = ctx.with_gateway_key(key); } @@ -997,6 +1034,21 @@ pub async fn run_streaming_turn( }; let citations = collect_citations(&auto_sources, &tool_sources); + // Drain the directive sink (mirrors the citation drain above). A host tool + // that ran this turn may have written a client-side directive; `Null` ⇒ none + // was written, so `eventual_response` omits `directive` (back-compat). + let directive = match Arc::try_unwrap(directive_sink) { + Ok(mutex) => mutex + .into_inner() + .unwrap_or_else(std::sync::PoisonError::into_inner), + Err(arc) => arc.lock().unwrap_or_else(|p| p.into_inner()).clone(), + }; + let directive = if directive.is_null() { + None + } else { + Some(directive) + }; + // 6. Conversation-workflow advancement (SMOODEV-590 parity). When the agent // has a workflow, ask the cheap judge whether THIS turn satisfied the // current step's criteria and compute the next step id. Failure-tolerant: @@ -1024,6 +1076,7 @@ pub async fn run_streaming_turn( usage, next_step_id, suggested_next_actions, + directive, }) } diff --git a/rust/smooth-operator-server/tests/acl_chat_leak.rs b/rust/smooth-operator-server/tests/acl_chat_leak.rs index 6a66c8b4..832b3a14 100644 --- a/rust/smooth-operator-server/tests/acl_chat_leak.rs +++ b/rust/smooth-operator-server/tests/acl_chat_leak.rs @@ -140,6 +140,7 @@ async fn run_turn_as( auth_gate: None, tool_configs: None, extensions: None, + images: vec![], }, &tx, ) diff --git a/rust/smooth-operator-server/tests/acl_trusted_mode.rs b/rust/smooth-operator-server/tests/acl_trusted_mode.rs index 084ebc3d..4399bcd2 100644 --- a/rust/smooth-operator-server/tests/acl_trusted_mode.rs +++ b/rust/smooth-operator-server/tests/acl_trusted_mode.rs @@ -145,6 +145,7 @@ async fn run_turn_as( auth_gate: None, tool_configs: None, extensions: None, + images: vec![], }, &tx, ) diff --git a/rust/smooth-operator-server/tests/agent_tool_auth.rs b/rust/smooth-operator-server/tests/agent_tool_auth.rs index c98da9ac..f72531ce 100644 --- a/rust/smooth-operator-server/tests/agent_tool_auth.rs +++ b/rust/smooth-operator-server/tests/agent_tool_auth.rs @@ -141,6 +141,7 @@ async fn run( auth_gate, tool_configs, extensions: None, + images: vec![], }, &tx, ) diff --git a/rust/smooth-operator-server/tests/confirm_tool_action.rs b/rust/smooth-operator-server/tests/confirm_tool_action.rs index c85c4c36..39c1cb0b 100644 --- a/rust/smooth-operator-server/tests/confirm_tool_action.rs +++ b/rust/smooth-operator-server/tests/confirm_tool_action.rs @@ -157,6 +157,7 @@ fn spawn_turn( auth_gate: None, tool_configs: None, extensions: None, + images: vec![], }, &sink, ) diff --git a/rust/smooth-operator-server/tests/conversation_workflow.rs b/rust/smooth-operator-server/tests/conversation_workflow.rs index 3fb82f52..0ef5a0a2 100644 --- a/rust/smooth-operator-server/tests/conversation_workflow.rs +++ b/rust/smooth-operator-server/tests/conversation_workflow.rs @@ -105,6 +105,7 @@ async fn run_turn( auth_gate: None, tool_configs: None, extensions: None, + images: vec![], }, &tx, ) @@ -283,6 +284,7 @@ async fn run_turn_on( auth_gate: None, tool_configs: None, extensions: None, + images: vec![], }, &tx, ) diff --git a/rust/smooth-operator-server/tests/empty_reply_fallback.rs b/rust/smooth-operator-server/tests/empty_reply_fallback.rs index 96cd9cb5..bfdbf180 100644 --- a/rust/smooth-operator-server/tests/empty_reply_fallback.rs +++ b/rust/smooth-operator-server/tests/empty_reply_fallback.rs @@ -139,6 +139,7 @@ async fn empty_terminal_content_falls_back_to_streamed_reply() { auth_gate: None, tool_configs: None, extensions: None, + images: vec![], }, &tx, ) diff --git a/rust/smooth-operator-server/tests/gateway_wire_empty_reply.rs b/rust/smooth-operator-server/tests/gateway_wire_empty_reply.rs index c1285544..112838c7 100644 --- a/rust/smooth-operator-server/tests/gateway_wire_empty_reply.rs +++ b/rust/smooth-operator-server/tests/gateway_wire_empty_reply.rs @@ -89,6 +89,7 @@ async fn run_against(chunks: Vec) -> (String, Vec, Vec, auth_gate: None, tool_configs: None, extensions: None, + images: vec![], }, &tx, ) diff --git a/rust/smooth-operator-server/tests/injection_seams.rs b/rust/smooth-operator-server/tests/injection_seams.rs index cc063372..1c4c0df9 100644 --- a/rust/smooth-operator-server/tests/injection_seams.rs +++ b/rust/smooth-operator-server/tests/injection_seams.rs @@ -140,6 +140,7 @@ async fn run_turn_with_key( auth_gate: None, tool_configs: None, extensions: None, + images: vec![], }, &tx, ) diff --git a/rust/smooth-operator-server/tests/interactions.rs b/rust/smooth-operator-server/tests/interactions.rs index ef059081..d73c2423 100644 --- a/rust/smooth-operator-server/tests/interactions.rs +++ b/rust/smooth-operator-server/tests/interactions.rs @@ -194,6 +194,7 @@ fn spawn_turn( auth_gate: None, tool_configs: None, extensions: None, + images: vec![], }, &sink, ) diff --git a/rust/smooth-operator-server/tests/knowledge_org_scoping.rs b/rust/smooth-operator-server/tests/knowledge_org_scoping.rs index 16ff4581..1c75c425 100644 --- a/rust/smooth-operator-server/tests/knowledge_org_scoping.rs +++ b/rust/smooth-operator-server/tests/knowledge_org_scoping.rs @@ -223,6 +223,7 @@ async fn run_turn_as(storage: Arc, access: AccessContext) { auth_gate: None, tool_configs: None, extensions: None, + images: vec![], }, &tx, ) diff --git a/rust/smooth-operator-server/tests/suggested_replies.rs b/rust/smooth-operator-server/tests/suggested_replies.rs index cba3cac5..824d2e7c 100644 --- a/rust/smooth-operator-server/tests/suggested_replies.rs +++ b/rust/smooth-operator-server/tests/suggested_replies.rs @@ -69,6 +69,7 @@ async fn run_turn(deltas: &[&str]) -> (TurnResult, Vec, String) { auth_gate: None, tool_configs: None, extensions: None, + images: vec![], }, &tx, ) diff --git a/rust/smooth-operator-server/tests/telemetry.rs b/rust/smooth-operator-server/tests/telemetry.rs index 5e006c32..36577986 100644 --- a/rust/smooth-operator-server/tests/telemetry.rs +++ b/rust/smooth-operator-server/tests/telemetry.rs @@ -187,6 +187,7 @@ async fn streaming_turn_emits_gen_ai_spans_with_org_and_tool_args() { auth_gate: None, tool_configs: None, extensions: None, + images: vec![], }, &tx, ) diff --git a/rust/smooth-operator/src/lib.rs b/rust/smooth-operator/src/lib.rs index 32f12525..02850792 100644 --- a/rust/smooth-operator/src/lib.rs +++ b/rust/smooth-operator/src/lib.rs @@ -84,7 +84,7 @@ pub use settings::{ AgentSettings, InMemorySettingsStore, SettingsStore, DEFAULT_MODEL, DEFAULT_SYSTEM_PROMPT, }; pub use telemetry::init_telemetry; -pub use tool_provider::{ToolProvider, ToolProviderContext}; +pub use tool_provider::{ToolProvider, ToolProviderContext, UserImage}; pub use tools::{ builtin_tools, interaction_channel, ConversationHistoryTool, FetchUrlTool, InteractionAttach, InteractionChannelPair, KnowledgeResultSink, KnowledgeSearchTool, NoopWebSearchProvider, diff --git a/rust/smooth-operator/src/tool_provider.rs b/rust/smooth-operator/src/tool_provider.rs index 47091aff..9cb725c9 100644 --- a/rust/smooth-operator/src/tool_provider.rs +++ b/rust/smooth-operator/src/tool_provider.rs @@ -30,13 +30,31 @@ //! gateway this turn was billed/scoped to). Both are carried as `Option` and //! the shared crate never interprets them — it only threads them through. -use std::sync::Arc; +use std::sync::{Arc, Mutex}; use async_trait::async_trait; +use serde::{Deserialize, Serialize}; use smooth_operator_core::Tool; use crate::access_control::AccessContext; +/// An image attachment on a multimodal turn's user message. `url` is a `data:` +/// image URL (`data:image/png;base64,...`) or a remote `https` URL; `detail` +/// (`"low"`/`"high"`/`"auto"`) is the optional OpenAI vision hint, omitted when +/// absent. The wire shape mirrors the core `ImageContent` — the server maps one +/// to the other before handing the turn to the engine. Shared by the inbound +/// `send_message` request (`images`) and the [`ToolProviderContext`] so a host +/// tool can see what the turn carried. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct UserImage { + /// A `data:`/`https` image URL, emitted to the model as an OpenAI + /// `image_url` content part. + pub url: String, + /// Optional OpenAI vision detail hint (`"low"`/`"high"`/`"auto"`). + #[serde(default, skip_serializing_if = "Option::is_none")] + pub detail: Option, +} + /// The per-turn context a [`ToolProvider`] sees when asked for tools. /// /// Carries everything a host needs to decide which tools a turn gets WITHOUT @@ -69,6 +87,18 @@ pub struct ToolProviderContext { /// for this agent's turn. Empty when no tool carries config; the shared crate /// does not interpret it. pub tool_specific_config: std::collections::HashMap, + /// Optional **directive sink** — where a host tool writes a client-side + /// directive (a navigation / view-application instruction) for this turn. The + /// runner drains it after the turn and carries the value onto the + /// `eventual_response`'s `directive` field. Opaque `serde_json::Value` + /// (last-write-wins): the shared crate never interprets the shape, exactly as + /// `response` is left loose. `None` (the default) ⇒ no host directive path, so + /// behavior is byte-for-byte unchanged. + pub directive_sink: Option>>, + /// The image attachments this turn carried (multimodal turns). A host tool may + /// read them; empty for the text-only common case. The runner also maps these + /// onto the engine's user message via core `with_user_images`. + pub images: Vec, } impl ToolProviderContext { @@ -86,6 +116,8 @@ impl ToolProviderContext { conversation_id: None, gateway_key: None, tool_specific_config: std::collections::HashMap::new(), + directive_sink: None, + images: Vec::new(), } } @@ -112,6 +144,21 @@ impl ToolProviderContext { self.gateway_key = Some(gateway_key.into()); self } + + /// Set the turn's [`directive_sink`](Self::directive_sink) — the slot a host + /// tool writes a client-side directive into for this turn. + #[must_use] + pub fn with_directive_sink(mut self, sink: Arc>) -> Self { + self.directive_sink = Some(sink); + self + } + + /// Set the turn's [`images`](Self::images) — the attachments the turn carried. + #[must_use] + pub fn with_images(mut self, images: Vec) -> Self { + self.images = images; + self + } } /// Host seam for contributing EXTRA tools to a turn's [`ToolRegistry`]. @@ -190,6 +237,8 @@ mod tests { let ctx = ToolProviderContext::new(Some("org-a".into()), AccessContext::anonymous()); assert_eq!(ctx.conversation_id, None); assert_eq!(ctx.gateway_key, None); + assert!(ctx.directive_sink.is_none()); + assert!(ctx.images.is_empty()); } #[test] @@ -201,6 +250,39 @@ mod tests { assert_eq!(ctx.gateway_key.as_deref(), Some("sk-org-a")); } + #[test] + fn builder_sets_directive_sink_and_images() { + let sink = Arc::new(Mutex::new(serde_json::Value::Null)); + let ctx = ToolProviderContext::new(Some("org-a".into()), AccessContext::anonymous()) + .with_directive_sink(Arc::clone(&sink)) + .with_images(vec![UserImage { + url: "https://x/y.png".into(), + detail: Some("high".into()), + }]); + // A host tool writes through the sink; the runner reads it afterward. + *ctx.directive_sink.as_ref().unwrap().lock().unwrap() = + serde_json::json!({"kind": "Navigate"}); + assert_eq!( + *sink.lock().unwrap(), + serde_json::json!({"kind": "Navigate"}) + ); + assert_eq!(ctx.images.len(), 1); + assert_eq!(ctx.images[0].url, "https://x/y.png"); + assert_eq!(ctx.images[0].detail.as_deref(), Some("high")); + } + + #[test] + fn user_image_omits_detail_when_absent() { + // Back-compat wire shape: no `detail` key when None. + let v = serde_json::to_value(UserImage { + url: "data:image/png;base64,AAAA".into(), + detail: None, + }) + .unwrap(); + assert!(v.get("detail").is_none()); + assert_eq!(v["url"], "data:image/png;base64,AAAA"); + } + #[tokio::test] async fn empty_provider_leaves_registry_unchanged() { let provider = StubProvider { names: vec![] }; diff --git a/spec/actions/send-message.schema.json b/spec/actions/send-message.schema.json index 7e046825..ee8bbc71 100644 --- a/spec/actions/send-message.schema.json +++ b/spec/actions/send-message.schema.json @@ -40,6 +40,26 @@ "model": { "type": "string", "description": "Optional gateway model id to run THIS turn on (e.g. a /smooth-mode preset). Absent → the server's configured default model." + }, + "images": { + "type": "array", + "description": "Optional image attachments for a multimodal turn. Each item is a `data:` or `https` image URL with an optional OpenAI vision `detail` hint. Absent/empty → a text-only turn (byte-identical to before this field existed). Fail-soft: a malformed entry is ignored rather than rejecting the turn.", + "items": { + "type": "object", + "required": ["url"], + "additionalProperties": false, + "properties": { + "url": { + "type": "string", + "description": "A `data:image/...;base64,...` URL or a remote `https` image URL. Emitted to the model as an OpenAI `image_url` content part." + }, + "detail": { + "type": "string", + "enum": ["low", "high", "auto"], + "description": "Optional OpenAI vision detail hint. Omitted when absent." + } + } + } } } }, @@ -67,6 +87,10 @@ "escalationReason": { "type": "string", "description": "Human-readable reason when `needsEscalation` is true." + }, + "directive": { + "type": ["object", "null"], + "description": "An optional client-side directive the agent emitted for this turn (e.g. a navigation or view-application instruction). Opaque at the protocol layer — like `response`, the concrete shape (Navigate / ApplyView / …) is owned by the host client and validated there. Absent when the turn produced no directive. Last-write-wins when multiple were emitted." } } }, diff --git a/spec/events/eventual-response.schema.json b/spec/events/eventual-response.schema.json index 9fb464ed..c80c9a22 100644 --- a/spec/events/eventual-response.schema.json +++ b/spec/events/eventual-response.schema.json @@ -106,6 +106,10 @@ } } } + }, + "directive": { + "type": ["object", "null"], + "description": "An optional client-side directive the agent emitted for this turn (e.g. a navigation or view-application instruction). Opaque at the protocol layer — like `response`, the concrete shape (Navigate / ApplyView / …) is owned by the host client and validated there. Optional and back-compatible: absent when the turn produced no directive. Last-write-wins when multiple were emitted." } } } diff --git a/typescript/src/generated/types.ts b/typescript/src/generated/types.ts index 356848a1..15e109c1 100644 --- a/typescript/src/generated/types.ts +++ b/typescript/src/generated/types.ts @@ -329,6 +329,19 @@ export interface SendMessageRequest { * Optional gateway model id to run THIS turn on (e.g. a /smooth-mode preset). Absent → the server's configured default model. */ model?: string; + /** + * Optional image attachments for a multimodal turn. Each item is a `data:` or `https` image URL with an optional OpenAI vision `detail` hint. Absent/empty → a text-only turn (byte-identical to before this field existed). Fail-soft: a malformed entry is ignored rather than rejecting the turn. + */ + images?: { + /** + * A `data:image/...;base64,...` URL or a remote `https` image URL. Emitted to the model as an OpenAI `image_url` content part. + */ + url: string; + /** + * Optional OpenAI vision detail hint. Omitted when absent. + */ + detail?: 'low' | 'high' | 'auto'; + }[]; } // ── from actions/send-message.schema.json ── @@ -349,6 +362,10 @@ export interface SendMessageResponse { * Human-readable reason when `needsEscalation` is true. */ escalationReason?: string; + /** + * An optional client-side directive the agent emitted for this turn (e.g. a navigation or view-application instruction). Opaque at the protocol layer — like `response`, the concrete shape (Navigate / ApplyView / …) is owned by the host client and validated there. Absent when the turn produced no directive. Last-write-wins when multiple were emitted. + */ + directive?: {} | null; } /** * Structured agent response. @@ -1017,6 +1034,10 @@ export interface EventualResponse { */ score: number; }[]; + /** + * An optional client-side directive the agent emitted for this turn (e.g. a navigation or view-application instruction). Opaque at the protocol layer — like `response`, the concrete shape (Navigate / ApplyView / …) is owned by the host client and validated there. Optional and back-compatible: absent when the turn produced no directive. Last-write-wins when multiple were emitted. + */ + directive?: {} | null; }; }; /**