directive-over-SEP + image-through-SEP (additive)

.changeset/sep-directive-and-images.md

@@ -0,0 +1,9 @@
+---
+"@smooai/smooth-operator": minor
+---
+
+Two additive SEP-protocol enhancements on the streaming path (directive nav + business-card images), both optional and back-compatible.
+
+**Directive-over-SEP.** `eventual_response` gains an optional `directive` field — an opaque client-side directive (e.g. a Navigate / ApplyView instruction) a host tool emitted this turn. The runner threads a `directive_sink` into the `ToolProviderContext` (new `with_directive_sink` builder), drains it after the turn (last-write-wins, mirroring the citation sink), and carries the value onto `TurnResult::directive`. The protocol layer never interprets the shape — the host client owns it, exactly like `response`. Absent when no host tool wrote one, so the event is byte-for-byte unchanged for existing clients. Added to `spec/events/eventual-response.schema.json` and `spec/actions/send-message.schema.json` `$defs/Response`, and to the TypeScript SDK.
+
+**Image-through-SEP.** `send_message` gains an optional `images` array (`{ url, detail? }`) for multimodal turns. A new facade `UserImage` type flows from the inbound request into `TurnRequest::images` and the `ToolProviderContext` (new `with_images` builder); when non-empty the runner maps each onto a core `ImageContent` and attaches them to the engine's user message via `AgentConfig::with_user_images` (requires core `0.16.2`). Parsing is fail-soft (a malformed `images` entry is dropped, never rejects the turn). Empty/absent ⇒ a text-only turn, unchanged. Added to `spec/actions/send-message.schema.json` `$defs/Request` and the TypeScript SDK.

rust/examples/dev-support/tests/serve_smoke.rs

@@ -364,6 +364,7 @@ async fn grounded_turn_over_served_storage_answers_from_the_ingested_repo() {
             conversation_id: &conversation_id,
             request_id: "sm-grounded",
             user_message: "How big is the Frobnicator's ring buffer?",
+            model_max_output: None,
             access: AccessContext::anonymous(),
             llm_provider: Some(Arc::new(mock.clone())),
             reranker: None,
@@ -380,6 +381,7 @@ async fn grounded_turn_over_served_storage_answers_from_the_ingested_repo() {
             auth_gate: None,
             tool_configs: None,
             extensions: None,
+            images: vec![],
         },
         &tx,
     )

rust/smooth-operator-lambda/src/dispatch.rs

@@ -571,6 +571,8 @@ async fn send_message(
             auth_gate: None,
             tool_configs: None,
             extensions: None,
+            // The lambda flavor is text-only; no multimodal attachments.
+            images: vec![],
         },
         &tx,
     )
@@ -593,6 +595,7 @@ async fn send_message(
                     false,
                     &turn.citations,
                     turn.usage,
+                    turn.directive,
                 ))
                 .await?;
         }

rust/smooth-operator-server/src/handler.rs

@@ -1046,6 +1046,13 @@ async fn handle_send_message(
         }
     };
 
+    // Optional multimodal attachments. Fail-soft: absent ⇒ text-only; a malformed
+    // `images` array is dropped rather than rejecting the turn (per the schema).
+    let images: Vec<smooth_operator::tool_provider::UserImage> = parsed
+        .get("images")
+        .and_then(|v| serde_json::from_value(v.clone()).ok())
+        .unwrap_or_default();
+
     let Some(session) = state.get_session(session_id) else {
         let _ = sink.send(protocol::error(
             Some(request_id),
@@ -1385,6 +1392,8 @@ async fn handle_send_message(
                 tool_configs,
                 // SEP — the per-turn extension host (None unless allowlisted).
                 extensions,
+                // Optional multimodal attachments (empty ⇒ text-only, unchanged).
+                images,
             },
             &sink_owned,
         )
@@ -1452,6 +1461,7 @@ async fn handle_send_message(
                     false,
                     &turn.citations,
                     turn.usage,
+                    turn.directive,
                 ));
 
                 // Best-effort auto-title (fires only while the conversation is

rust/smooth-operator-server/src/protocol.rs

@@ -138,6 +138,12 @@ pub struct TurnUsage {
 /// client can accumulate live session cost. Absent when the engine reported no
 /// usage (e.g. an offline mock turn), keeping the event back-compatible with
 /// clients that predate cost reporting.
+///
+/// `directive`, when `Some`, attaches an opaque client-side directive a host tool
+/// emitted this turn as a sibling `data.data.directive` value. The protocol layer
+/// never interprets the shape (the host client owns it, like `response`). Absent
+/// when the turn produced no directive, keeping the event back-compatible with
+/// clients that predate directives.
 #[must_use]
 pub fn eventual_response(
     request_id: &str,
@@ -147,6 +153,7 @@ pub fn eventual_response(
     needs_escalation: bool,
     citations: &[smooth_operator::domain::Citation],
     usage: Option<TurnUsage>,
+    directive: Option<Value>,
 ) -> Value {
     let mut inner = json!({
         "messageId": message_id,
@@ -165,6 +172,10 @@ pub fn eventual_response(
             "completionTokens": usage.completion_tokens,
         });
     }
+    // Optional + back-compat: only emit `directive` when a host tool wrote one.
+    if let Some(directive) = directive {
+        inner["directive"] = directive;
+    }
     json!({
         "type": "eventual_response",
         "requestId": request_id,
@@ -435,7 +446,10 @@ mod tests {
         assert_eq!(ev["type"], "stream_preamble");
         // …but shaped exactly like stream_token so they can reuse the render path.
         assert_eq!(ev["token"], "Let me pull up your recent conversations.");
-        assert_eq!(ev["data"]["token"], "Let me pull up your recent conversations.");
+        assert_eq!(
+            ev["data"]["token"],
+            "Let me pull up your recent conversations."
+        );
         assert_eq!(ev["data"]["requestId"], "r1");
     }
 
@@ -458,6 +472,7 @@ mod tests {
             false,
             &[],
             None,
+            None,
         );
         assert_eq!(ev["type"], "eventual_response");
         assert_eq!(ev["status"], 200);
@@ -477,6 +492,7 @@ mod tests {
             false,
             &[],
             None,
+            None,
         );
         assert!(
             ev["data"]["data"].get("citations").is_none(),
@@ -495,6 +511,7 @@ mod tests {
             false,
             &[],
             None,
+            None,
         );
         assert!(
             ev["data"]["data"].get("usage").is_none(),
@@ -517,6 +534,7 @@ mod tests {
             false,
             &[],
             Some(usage),
+            None,
         );
         let u = &ev["data"]["data"]["usage"];
         assert!(
@@ -555,6 +573,7 @@ mod tests {
             false,
             &citations,
             None,
+            None,
         );
         let cites = &ev["data"]["data"]["citations"];
         assert!(cites.is_array(), "citations should be an array");
@@ -583,6 +602,41 @@ mod tests {
         assert_eq!(cites[1]["id"], "doc-2");
     }
 
+    #[test]
+    fn eventual_response_omits_directive_when_none() {
+        // Back-compat: no `directive` key at all when no host tool wrote one.
+        let ev = eventual_response(
+            "r1",
+            200,
+            "m1",
+            json!({"responseParts": ["hi"]}),
+            false,
+            &[],
+            None,
+            None,
+        );
+        assert!(
+            ev["data"]["data"].get("directive").is_none(),
+            "directive must be absent when None for back-compat"
+        );
+    }
+
+    #[test]
+    fn eventual_response_attaches_directive_when_present() {
+        let directive = json!({"kind": "Navigate", "path": "/crm/contacts/42"});
+        let ev = eventual_response(
+            "r1",
+            200,
+            "m1",
+            json!({"responseParts": ["hi"]}),
+            false,
+            &[],
+            None,
+            Some(directive.clone()),
+        );
+        assert_eq!(ev["data"]["data"]["directive"], directive);
+    }
+
     #[test]
     fn write_confirmation_required_matches_spec_shape() {
         let ev = write_confirmation_required(

rust/smooth-operator-server/src/runner.rs

@@ -216,6 +216,12 @@ pub struct TurnResult {
     /// [`crate::suggestions`]). Empty when the model emitted none. Carried onto
     /// the `eventual_response`'s `suggestedNextActions`.
     pub suggested_next_actions: Vec<String>,
+    /// An optional client-side directive a host tool emitted this turn (drained
+    /// from the [`ToolProviderContext::directive_sink`]). Opaque
+    /// `serde_json::Value` (last-write-wins) — carried onto the
+    /// `eventual_response`'s `directive` field. `None` when no host tool wrote
+    /// one (or no tool provider was installed), keeping the event back-compatible.
+    pub directive: Option<serde_json::Value>,
 }
 
 /// One tool call captured during the turn, used to emit a `gen_ai.tool` child
@@ -345,6 +351,12 @@ pub struct TurnRequest<'a> {
     /// [`crate::extensions::build_extension_host`] (only when
     /// `SMOOTH_EXTENSIONS_ALLOW` is non-empty).
     pub extensions: Option<crate::extensions::ExtensionTurn>,
+    /// Image attachments for a multimodal turn. When non-empty, the runner maps
+    /// each onto a core `ImageContent` and attaches them to the engine's user
+    /// message via `AgentConfig::with_user_images`, and also carries them into the
+    /// [`ToolProviderContext`] so a host tool can see them. Empty (the default)
+    /// ⇒ a text-only turn, byte-for-byte unchanged.
+    pub images: Vec<smooth_operator::tool_provider::UserImage>,
 }
 
 /// Runs one knowledge-grounded, streaming turn for a session's conversation and
@@ -393,6 +405,7 @@ pub async fn run_streaming_turn(
         auth_gate,
         tool_configs,
         extensions,
+        images,
     } = req;
 
     // Capture the OTel turn-span attributes up front, since `llm` is moved into
@@ -437,6 +450,11 @@ pub async fn run_streaming_turn(
     // Sink the knowledge_search tool records its structured results into, for
     // citations built from the sources the agent's searches surfaced.
     let tool_sources: KnowledgeResultSink = Arc::new(Mutex::new(Vec::new()));
+    // Slot a host tool writes a client-side directive into this turn. Drained
+    // after the run into `TurnResult::directive` (Null ⇒ absent). Only threaded
+    // into the `ToolProviderContext` when a provider is installed, so with no
+    // host tools it stays `Null` and the `eventual_response` omits `directive`.
+    let directive_sink = Arc::new(Mutex::new(serde_json::Value::Null));
 
     // 1. Load prior turns for memory BEFORE persisting the new inbound message,
     //    so prior_messages is exactly the history-up-to-now.
@@ -481,12 +499,26 @@ pub async fn run_streaming_turn(
     // that emits no trailer costs nothing and yields empty suggestions.
     sections.push(crate::suggestions::SUGGESTED_REPLIES_PROMPT_SECTION.to_string());
     let resolved_prompt = sections.join("\n\n");
-    let config = AgentConfig::new("smooth-agent-chat", &resolved_prompt, llm)
+    let mut config = AgentConfig::new("smooth-agent-chat", &resolved_prompt, llm)
         .with_max_iterations(max_iterations)
         .with_knowledge(Arc::clone(&knowledge))
         .with_prior_messages(prior)
         // Clamp max_tokens to the model's output ceiling (None ⇒ unclamped).
         .with_model_ceiling(model_max_output);
+    // Multimodal turn: attach the turn's images to the engine's user message as
+    // OpenAI `image_url` content parts. Empty (the default) leaves the turn
+    // text-only — byte-for-byte unchanged.
+    if !images.is_empty() {
+        config = config.with_user_images(
+            images
+                .iter()
+                .map(|img| smooth_operator_core::conversation::ImageContent {
+                    url: img.url.clone(),
+                    detail: img.detail.clone(),
+                })
+                .collect(),
+        );
+    }
 
     let mut tools = ToolRegistry::new();
     // Build the knowledge_search tool over the SAME ACL-filtered handle, with the
@@ -553,8 +585,13 @@ pub async fn run_streaming_turn(
         // Thread the per-turn handles the runner already has — the conversation
         // this turn runs in and the resolved per-org gateway key — so a host's
         // conversation-persisting / retrieval tools aren't degraded to no-ops.
-        let mut ctx =
-            ToolProviderContext::new(org_id, access.clone()).with_conversation_id(conversation_id);
+        let mut ctx = ToolProviderContext::new(org_id, access.clone())
+            .with_conversation_id(conversation_id)
+            // Directive-over-SEP: give host tools the slot to write a client-side
+            // directive; drained after the turn onto `eventual_response.directive`.
+            .with_directive_sink(Arc::clone(&directive_sink))
+            // Multimodal: let a host tool see the turn's image attachments.
+            .with_images(images.clone());
         if let Some(key) = gateway_key {
             ctx = ctx.with_gateway_key(key);
         }
@@ -997,6 +1034,21 @@ pub async fn run_streaming_turn(
     };
     let citations = collect_citations(&auto_sources, &tool_sources);
 
+    // Drain the directive sink (mirrors the citation drain above). A host tool
+    // that ran this turn may have written a client-side directive; `Null` ⇒ none
+    // was written, so `eventual_response` omits `directive` (back-compat).
+    let directive = match Arc::try_unwrap(directive_sink) {
+        Ok(mutex) => mutex
+            .into_inner()
+            .unwrap_or_else(std::sync::PoisonError::into_inner),
+        Err(arc) => arc.lock().unwrap_or_else(|p| p.into_inner()).clone(),
+    };
+    let directive = if directive.is_null() {
+        None
+    } else {
+        Some(directive)
+    };
+
     // 6. Conversation-workflow advancement (SMOODEV-590 parity). When the agent
     //    has a workflow, ask the cheap judge whether THIS turn satisfied the
     //    current step's criteria and compute the next step id. Failure-tolerant:
@@ -1024,6 +1076,7 @@ pub async fn run_streaming_turn(
         usage,
         next_step_id,
         suggested_next_actions,
+        directive,
     })
 }
 

rust/smooth-operator-server/tests/acl_chat_leak.rs

@@ -140,6 +140,7 @@ async fn run_turn_as(
             auth_gate: None,
             tool_configs: None,
             extensions: None,
+            images: vec![],
         },
         &tx,
     )

rust/smooth-operator-server/tests/acl_trusted_mode.rs

@@ -145,6 +145,7 @@ async fn run_turn_as(
             auth_gate: None,
             tool_configs: None,
             extensions: None,
+            images: vec![],
         },
         &tx,
     )

rust/smooth-operator-server/tests/agent_tool_auth.rs

@@ -141,6 +141,7 @@ async fn run(
             auth_gate,
             tool_configs,
             extensions: None,
+            images: vec![],
         },
         &tx,
     )

rust/smooth-operator-server/tests/confirm_tool_action.rs

@@ -157,6 +157,7 @@ fn spawn_turn(
                 auth_gate: None,
                 tool_configs: None,
                 extensions: None,
+                images: vec![],
             },
             &sink,
         )

rust/smooth-operator-server/tests/conversation_workflow.rs

@@ -105,6 +105,7 @@ async fn run_turn(
             auth_gate: None,
             tool_configs: None,
             extensions: None,
+            images: vec![],
         },
         &tx,
     )
@@ -283,6 +284,7 @@ async fn run_turn_on(
             auth_gate: None,
             tool_configs: None,
             extensions: None,
+            images: vec![],
         },
         &tx,
     )

rust/smooth-operator-server/tests/empty_reply_fallback.rs

@@ -139,6 +139,7 @@ async fn empty_terminal_content_falls_back_to_streamed_reply() {
             auth_gate: None,
             tool_configs: None,
             extensions: None,
+            images: vec![],
         },
         &tx,
     )

rust/smooth-operator-server/tests/gateway_wire_empty_reply.rs

@@ -89,6 +89,7 @@ async fn run_against(chunks: Vec<String>) -> (String, Vec<String>, Vec<String>,
             auth_gate: None,
             tool_configs: None,
             extensions: None,
+            images: vec![],
         },
         &tx,
     )

rust/smooth-operator-server/tests/injection_seams.rs

@@ -140,6 +140,7 @@ async fn run_turn_with_key(
             auth_gate: None,
             tool_configs: None,
             extensions: None,
+            images: vec![],
         },
         &tx,
     )

rust/smooth-operator-server/tests/interactions.rs

@@ -194,6 +194,7 @@ fn spawn_turn(
                 auth_gate: None,
                 tool_configs: None,
                 extensions: None,
+                images: vec![],
             },
             &sink,
         )