th-9a5794: stream_preamble — fast-model preamble to cover reasoning TTFT

.changeset/preamble-stream-event.md

@@ -0,0 +1,5 @@
+---
+"@smooai/smooth-operator": patch
+---
+
+Add an optional fast-model **preamble** to streaming turns to cover the reasoning model's time-to-first-token. When the server is configured with `SMOOTH_AGENT_PREAMBLE_MODEL` (e.g. `groq-gpt-oss-20b`), a small fast model runs IN PARALLEL with the main turn and streams ONE short present-tense "what I'm about to do" sentence over a new `stream_preamble` wire event — an ephemeral status line the real answer replaces. It's best-effort (any error/slowness is swallowed on its own task) and guarded: it's dropped if the real answer has already begun streaming, so it can never block or corrupt a turn. Unset ⇒ no extra call and byte-for-byte unchanged behavior. Adds `stream-preamble.schema.json` to the SEP spec and `StreamPreamble` to the TypeScript SDK union.

rust/smooth-operator-server/src/config.rs

@@ -15,6 +15,7 @@
 //! | `SMOOAI_GATEWAY_URL` | `https://llm.smoo.ai/v1` | OpenAI-compatible LLM gateway base URL. |
 //! | `SMOOAI_GATEWAY_KEY` | *(unset)* | Gateway API key. When unset, `send_message` errors cleanly. |
 //! | `SMOOTH_AGENT_MODEL` | `claude-haiku-4-5` | Model id requested from the gateway. |
+//! | `SMOOTH_AGENT_PREAMBLE_MODEL` | *(unset → off)* | When set to a fast model id (e.g. `groq-gpt-oss-20b`), a small model runs in parallel with each streaming turn and emits ONE ephemeral `stream_preamble` sentence ("what I'm about to do") to cover the main model's time-to-first-token. Uses the same gateway/key as `SMOOTH_AGENT_MODEL`. Unset ⇒ no extra call, behavior unchanged. |
 //! | `SMOOTH_AGENT_SEED_KB` | *(unset)* | When `1`, seed a couple of distinctive demo docs on startup. |
 //! | `SMOOTH_AGENT_MAX_ITERATIONS` | `6` | Agent-loop iteration cap per turn. |
 //! | `SMOOTH_AGENT_MAX_TOKENS` | `512` | `max_tokens` sent to the gateway (kept low — paid endpoint). |

rust/smooth-operator-server/src/protocol.rs

@@ -80,6 +80,23 @@ pub fn stream_reasoning(request_id: &str, token: &str) -> Value {
     })
 }
 
+/// `stream_preamble` — a single streamed token of the fast-model *preamble*: a
+/// short "what I'm about to do" sentence generated in parallel with the main turn
+/// to cover the reasoning model's time-to-first-token. Shaped exactly like
+/// `stream_token`, but on a distinct `type` so clients render it as an *ephemeral*
+/// status line that the real answer replaces — never folded into the answer.
+/// Clients that don't know the type simply ignore it. Pearl th-9a5794.
+#[must_use]
+pub fn stream_preamble(request_id: &str, token: &str) -> Value {
+    json!({
+        "type": "stream_preamble",
+        "requestId": request_id,
+        "token": token,
+        "data": { "requestId": request_id, "token": token },
+        "timestamp": now_ms(),
+    })
+}
+
 /// `stream_chunk` — a per-node state snapshot. `node` is mirrored at the
 /// envelope level and inside `data` (per `stream-chunk.schema.json`). `state`
 /// only carries safe-to-expose fields.
@@ -411,6 +428,17 @@ mod tests {
         assert_eq!(ev["data"]["requestId"], "r1");
     }
 
+    #[test]
+    fn stream_preamble_is_distinct_type_but_mirrors_token() {
+        let ev = stream_preamble("r1", "Let me pull up your recent conversations.");
+        // Distinct type so clients render it as an ephemeral status line…
+        assert_eq!(ev["type"], "stream_preamble");
+        // …but shaped exactly like stream_token so they can reuse the render path.
+        assert_eq!(ev["token"], "Let me pull up your recent conversations.");
+        assert_eq!(ev["data"]["token"], "Let me pull up your recent conversations.");
+        assert_eq!(ev["data"]["requestId"], "r1");
+    }
+
     #[test]
     fn stream_chunk_mirrors_node() {
         let ev = stream_chunk("r1", "knowledge_search", json!({ "rawResponse": "x" }));

rust/smooth-operator-server/src/runner.rs

@@ -28,7 +28,8 @@ use serde_json::json;
 use smooth_operator_core::llm_provider::LlmProvider;
 use smooth_operator_core::{
     human_channel, Agent, AgentConfig, AgentEvent, ConfirmationHook, HumanRequest, HumanResponse,
-    KnowledgeBase, KnowledgeResult, LlmConfig, Message as EngineMessage, Role, ToolRegistry,
+    KnowledgeBase, KnowledgeResult, LlmClient, LlmConfig, Message as EngineMessage, Role,
+    ToolRegistry,
 };
 use tokio::sync::mpsc::{UnboundedReceiver, UnboundedSender};
 
@@ -83,6 +84,18 @@ const MAX_PRIOR_MESSAGES: usize = 50;
 /// task forever. Generous (5 min) because a human is in the loop.
 const CONFIRMATION_TIMEOUT: Duration = Duration::from_secs(300);
 
+/// `max_tokens` for the fast-model preamble — one short sentence. Pearl th-9a5794.
+const PREAMBLE_MAX_TOKENS: u32 = 64;
+
+/// System prompt for the fast-model preamble (see `SMOOTH_AGENT_PREAMBLE_MODEL`).
+/// One short present-tense sentence describing intent — no answer (it's generated
+/// WITHOUT the tool result), no greeting, no promises.
+const PREAMBLE_SYSTEM_PROMPT: &str = "You are the assistant's voice while it works. \
+In ONE short present-tense sentence (max ~12 words), tell the user what you're about to do to help with their message. \
+Do NOT answer the question, do NOT greet, do NOT promise a specific result or outcome. \
+Example: \"Let me pull up your recent conversations.\" \
+Reply with only that sentence — no quotes, no preamble, no markdown.";
+
 /// Registers a parked turn's [`HumanResponse`] sender under a session id (so a
 /// later `confirm_tool_action` can take it). Typically `AppState::register_confirmation`.
 pub type RegisterConfirmation = Arc<dyn Fn(&str, UnboundedSender<HumanResponse>) + Send + Sync>;
@@ -387,6 +400,25 @@ pub async fn run_streaming_turn(
     let model_for_span = llm.model.clone();
     let org_id_for_span = org_id.clone();
 
+    // Optional fast-model preamble. When `SMOOTH_AGENT_PREAMBLE_MODEL` is set, a
+    // small fast model (e.g. `groq-gpt-oss-20b`) runs in PARALLEL with the main
+    // turn and streams ONE ephemeral "what I'm about to do" sentence
+    // (`stream_preamble`), covering the reasoning model's time-to-first-token.
+    // Built from a clone of `llm` (same gateway/key) with the model + a tight
+    // token cap overridden. Unset/empty ⇒ off, so default behavior is unchanged.
+    // Skipped when a test provider is injected (no live gateway). Pearl th-9a5794.
+    let preamble_llm = (llm_provider.is_none())
+        .then(|| std::env::var("SMOOTH_AGENT_PREAMBLE_MODEL").ok())
+        .flatten()
+        .map(|m| m.trim().to_string())
+        .filter(|m| !m.is_empty())
+        .map(|model| {
+            let mut cfg = llm.clone();
+            cfg.model = model;
+            cfg.max_tokens = PREAMBLE_MAX_TOKENS;
+            cfg
+        });
+
     // The ONE ACL-enforcing knowledge handle both retrieval paths read through.
     // Built once from the requester's `AccessContext` so the auto-injected
     // context query, the agent's `knowledge_search` tool, and the citation
@@ -639,6 +671,42 @@ pub async fn run_streaming_turn(
     let request_id_owned = request_id.to_string();
     let sink_clone = sink.clone();
 
+    // First-answer-token guard for the optional preamble: the translator flips it
+    // on the first `TokenDelta` so a slow preamble never pops in AFTER the real
+    // answer has already begun streaming. Pearl th-9a5794.
+    let answer_started = Arc::new(std::sync::atomic::AtomicBool::new(false));
+    let answer_started_translator = Arc::clone(&answer_started);
+
+    // Fire the fast-model preamble in PARALLEL with the agent loop (no-op unless
+    // `SMOOTH_AGENT_PREAMBLE_MODEL` is set → `preamble_llm` is `Some`). Best-effort:
+    // any error/slowness is swallowed on its own task, and the guard drops it if the
+    // real answer already started — so it can never block or corrupt the turn.
+    // Pearl th-9a5794.
+    if let Some(pre_cfg) = preamble_llm {
+        let sink_preamble = sink.clone();
+        let request_id_preamble = request_id.to_string();
+        let user_preamble = user_message.to_string();
+        let answer_started_preamble = Arc::clone(&answer_started);
+        tokio::spawn(async move {
+            let client = LlmClient::new(pre_cfg);
+            let sys = EngineMessage::system(PREAMBLE_SYSTEM_PROMPT);
+            let user = EngineMessage::user(user_preamble);
+            match client.chat(&[&sys, &user], &[]).await {
+                Ok(resp) => {
+                    let text = resp.content.trim();
+                    if !text.is_empty()
+                        && !answer_started_preamble.load(std::sync::atomic::Ordering::Relaxed)
+                    {
+                        let _ = sink_preamble
+                            .send(crate::protocol::stream_preamble(&request_id_preamble, text));
+                    }
+                }
+                // Best-effort: a failed/slow preamble must never surface or block.
+                Err(e) => tracing::debug!(error = %e, "preamble generation failed (ignored)"),
+            }
+        });
+    }
+
     // Spawn the event translator so we forward tokens to the client in real
     // time while the agent loop runs concurrently.
     let translator = tokio::spawn(async move {
@@ -681,6 +749,9 @@ pub async fn run_streaming_turn(
         while let Some(event) = rx.recv().await {
             match event {
                 AgentEvent::TokenDelta { content } => {
+                    // Mark the answer as started so a still-in-flight preamble is
+                    // dropped rather than popping in after the reply. Pearl th-9a5794.
+                    answer_started_translator.store(true, std::sync::atomic::Ordering::Relaxed);
                     streamed_reply.push_str(&content);
                     let safe = suppressor.push(&content);
                     if !safe.is_empty() {

spec/events/stream-preamble.schema.json

@@ -0,0 +1,44 @@
+{
+  "$schema": "https://json-schema.org/draft/2020-12/schema",
+  "$id": "https://smooth-agent.dev/spec/events/stream-preamble.schema.json",
+  "title": "StreamPreamble",
+  "description": "Event: `stream_preamble`. A single token of a short, present-tense \"what I'm about to do\" sentence, generated by a small fast model IN PARALLEL with the main turn to cover the reasoning model's time-to-first-token. Emitted only when the server is configured with `SMOOTH_AGENT_PREAMBLE_MODEL`. Shaped identically to `stream_token` so clients can reuse the render path, but on a distinct `type` so it is shown as an EPHEMERAL status line that the real answer replaces — it is NEVER folded into the answer, and the final response (carried by `eventual_response`) never includes it. The server suppresses it if the real answer has already begun streaming. Clients that do not recognize this event MUST ignore it — the answer still streams via `stream_token`, so the preamble simply isn't shown.",
+  "type": "object",
+  "required": ["type", "data"],
+  "additionalProperties": false,
+  "properties": {
+    "type": {
+      "type": "string",
+      "const": "stream_preamble",
+      "description": "Event type discriminator."
+    },
+    "requestId": {
+      "type": "string",
+      "description": "Echoes the `requestId` from the originating `send_message` action."
+    },
+    "token": {
+      "type": "string",
+      "description": "The raw preamble token text. Also present inside `data.token` for consumers that only inspect `data`."
+    },
+    "data": {
+      "type": "object",
+      "description": "Preamble token event payload.",
+      "required": ["requestId", "token"],
+      "additionalProperties": false,
+      "properties": {
+        "requestId": {
+          "type": "string",
+          "description": "The request ID this preamble token belongs to."
+        },
+        "token": {
+          "type": "string",
+          "description": "The raw preamble token text."
+        }
+      }
+    },
+    "timestamp": {
+      "type": "integer",
+      "description": "Unix epoch milliseconds when the event was emitted."
+    }
+  }
+}

typescript/src/generated/types.ts

@@ -1474,6 +1474,42 @@ export interface StreamChunk {
     timestamp?: number;
 }
 
+// ── from events/stream-preamble.schema.json ──
+/**
+ * Event: `stream_preamble`. A single token of a short, present-tense "what I'm about to do" sentence, generated by a small fast model IN PARALLEL with the main turn to cover the reasoning model's time-to-first-token. Emitted only when the server is configured with `SMOOTH_AGENT_PREAMBLE_MODEL`. Shaped identically to `stream_token` so clients can reuse the render path, but on a distinct `type` so it is shown as an EPHEMERAL status line that the real answer replaces — it is NEVER folded into the answer, and the final response (carried by `eventual_response`) never includes it. The server suppresses it if the real answer has already begun streaming. Clients that do not recognize this event MUST ignore it — the answer still streams via `stream_token`, so the preamble simply isn't shown.
+ */
+export interface StreamPreamble {
+    /**
+     * Event type discriminator.
+     */
+    type: 'stream_preamble';
+    /**
+     * Echoes the `requestId` from the originating `send_message` action.
+     */
+    requestId?: string;
+    /**
+     * The raw preamble token text. Also present inside `data.token` for consumers that only inspect `data`.
+     */
+    token?: string;
+    /**
+     * Preamble token event payload.
+     */
+    data: {
+        /**
+         * The request ID this preamble token belongs to.
+         */
+        requestId: string;
+        /**
+         * The raw preamble token text.
+         */
+        token: string;
+    };
+    /**
+     * Unix epoch milliseconds when the event was emitted.
+     */
+    timestamp?: number;
+}
+
 // ── from events/stream-reasoning.schema.json ──
 /**
  * Event: `stream_reasoning`. A single *reasoning* token from a reasoning-model's separate thinking channel (`reasoning_content`/`reasoning` deltas — e.g. DeepSeek, gpt-oss/harmony, MiniMax, GLM), forwarded to the client in real time. Corresponds to smooth-operator's `AgentEvent::ReasoningDelta`. Shaped identically to `stream_token` so clients can render it the same way, but on a distinct `type` so reasoning is shown as collapsible "thinking" and is NEVER folded into the answer. The final response (carried by `eventual_response`) already excludes reasoning. Clients that do not recognize this event MUST ignore it — the answer still streams via `stream_token`, so reasoning simply isn't shown.

typescript/src/types.ts

@@ -34,6 +34,7 @@ import type {
     SendMessageRequest,
     SendMessageResponse,
     StreamChunk,
+    StreamPreamble,
     StreamToken,
     SubmitInteractionRequest,
     VerifyOtpRequest,
@@ -69,6 +70,7 @@ export const EVENT_TYPES = [
     'eventual_response',
     'stream_chunk',
     'stream_token',
+    'stream_preamble',
     'keepalive',
     'write_confirmation_required',
     'otp_verification_required',
@@ -109,6 +111,7 @@ export type ServerEvent =
     | EventualResponse
     | StreamChunk
     | StreamToken
+    | StreamPreamble
     | Keepalive
     | WriteConfirmationRequired
     | OtpVerificationRequired
@@ -128,6 +131,7 @@ export interface ServerEventByType {
     eventual_response: EventualResponse;
     stream_chunk: StreamChunk;
     stream_token: StreamToken;
+    stream_preamble: StreamPreamble;
     keepalive: Keepalive;
     write_confirmation_required: WriteConfirmationRequired;
     otp_verification_required: OtpVerificationRequired;

typescript/src/validate.ts

@@ -41,6 +41,7 @@ const EVENT_SCHEMA_FILE: Record<EventType, string> = {
     eventual_response: 'events/eventual-response.schema.json',
     stream_chunk: 'events/stream-chunk.schema.json',
     stream_token: 'events/stream-token.schema.json',
+    stream_preamble: 'events/stream-preamble.schema.json',
     keepalive: 'events/keepalive.schema.json',
     write_confirmation_required: 'events/write-confirmation-required.schema.json',
     otp_verification_required: 'events/otp-verification-required.schema.json',