diff --git a/.changeset/preamble-stream-event.md b/.changeset/preamble-stream-event.md new file mode 100644 index 00000000..bdbb04f0 --- /dev/null +++ b/.changeset/preamble-stream-event.md @@ -0,0 +1,5 @@ +--- +"@smooai/smooth-operator": patch +--- + +Add an optional fast-model **preamble** to streaming turns to cover the reasoning model's time-to-first-token. When the server is configured with `SMOOTH_AGENT_PREAMBLE_MODEL` (e.g. `groq-gpt-oss-20b`), a small fast model runs IN PARALLEL with the main turn and streams ONE short present-tense "what I'm about to do" sentence over a new `stream_preamble` wire event — an ephemeral status line the real answer replaces. It's best-effort (any error/slowness is swallowed on its own task) and guarded: it's dropped if the real answer has already begun streaming, so it can never block or corrupt a turn. Unset ⇒ no extra call and byte-for-byte unchanged behavior. Adds `stream-preamble.schema.json` to the SEP spec and `StreamPreamble` to the TypeScript SDK union. diff --git a/rust/smooth-operator-server/src/config.rs b/rust/smooth-operator-server/src/config.rs index 15a4a307..4728cebe 100644 --- a/rust/smooth-operator-server/src/config.rs +++ b/rust/smooth-operator-server/src/config.rs @@ -15,6 +15,7 @@ //! | `SMOOAI_GATEWAY_URL` | `https://llm.smoo.ai/v1` | OpenAI-compatible LLM gateway base URL. | //! | `SMOOAI_GATEWAY_KEY` | *(unset)* | Gateway API key. When unset, `send_message` errors cleanly. | //! | `SMOOTH_AGENT_MODEL` | `claude-haiku-4-5` | Model id requested from the gateway. | +//! | `SMOOTH_AGENT_PREAMBLE_MODEL` | *(unset → off)* | When set to a fast model id (e.g. `groq-gpt-oss-20b`), a small model runs in parallel with each streaming turn and emits ONE ephemeral `stream_preamble` sentence ("what I'm about to do") to cover the main model's time-to-first-token. Uses the same gateway/key as `SMOOTH_AGENT_MODEL`. Unset ⇒ no extra call, behavior unchanged. | //! | `SMOOTH_AGENT_SEED_KB` | *(unset)* | When `1`, seed a couple of distinctive demo docs on startup. | //! | `SMOOTH_AGENT_MAX_ITERATIONS` | `6` | Agent-loop iteration cap per turn. | //! | `SMOOTH_AGENT_MAX_TOKENS` | `512` | `max_tokens` sent to the gateway (kept low — paid endpoint). | diff --git a/rust/smooth-operator-server/src/protocol.rs b/rust/smooth-operator-server/src/protocol.rs index 9d4ef01b..15c38416 100644 --- a/rust/smooth-operator-server/src/protocol.rs +++ b/rust/smooth-operator-server/src/protocol.rs @@ -80,6 +80,23 @@ pub fn stream_reasoning(request_id: &str, token: &str) -> Value { }) } +/// `stream_preamble` — a single streamed token of the fast-model *preamble*: a +/// short "what I'm about to do" sentence generated in parallel with the main turn +/// to cover the reasoning model's time-to-first-token. Shaped exactly like +/// `stream_token`, but on a distinct `type` so clients render it as an *ephemeral* +/// status line that the real answer replaces — never folded into the answer. +/// Clients that don't know the type simply ignore it. Pearl th-9a5794. +#[must_use] +pub fn stream_preamble(request_id: &str, token: &str) -> Value { + json!({ + "type": "stream_preamble", + "requestId": request_id, + "token": token, + "data": { "requestId": request_id, "token": token }, + "timestamp": now_ms(), + }) +} + /// `stream_chunk` — a per-node state snapshot. `node` is mirrored at the /// envelope level and inside `data` (per `stream-chunk.schema.json`). `state` /// only carries safe-to-expose fields. @@ -411,6 +428,17 @@ mod tests { assert_eq!(ev["data"]["requestId"], "r1"); } + #[test] + fn stream_preamble_is_distinct_type_but_mirrors_token() { + let ev = stream_preamble("r1", "Let me pull up your recent conversations."); + // Distinct type so clients render it as an ephemeral status line… + assert_eq!(ev["type"], "stream_preamble"); + // …but shaped exactly like stream_token so they can reuse the render path. + assert_eq!(ev["token"], "Let me pull up your recent conversations."); + assert_eq!(ev["data"]["token"], "Let me pull up your recent conversations."); + assert_eq!(ev["data"]["requestId"], "r1"); + } + #[test] fn stream_chunk_mirrors_node() { let ev = stream_chunk("r1", "knowledge_search", json!({ "rawResponse": "x" })); diff --git a/rust/smooth-operator-server/src/runner.rs b/rust/smooth-operator-server/src/runner.rs index 1dec715c..b4190548 100644 --- a/rust/smooth-operator-server/src/runner.rs +++ b/rust/smooth-operator-server/src/runner.rs @@ -28,7 +28,8 @@ use serde_json::json; use smooth_operator_core::llm_provider::LlmProvider; use smooth_operator_core::{ human_channel, Agent, AgentConfig, AgentEvent, ConfirmationHook, HumanRequest, HumanResponse, - KnowledgeBase, KnowledgeResult, LlmConfig, Message as EngineMessage, Role, ToolRegistry, + KnowledgeBase, KnowledgeResult, LlmClient, LlmConfig, Message as EngineMessage, Role, + ToolRegistry, }; use tokio::sync::mpsc::{UnboundedReceiver, UnboundedSender}; @@ -83,6 +84,18 @@ const MAX_PRIOR_MESSAGES: usize = 50; /// task forever. Generous (5 min) because a human is in the loop. const CONFIRMATION_TIMEOUT: Duration = Duration::from_secs(300); +/// `max_tokens` for the fast-model preamble — one short sentence. Pearl th-9a5794. +const PREAMBLE_MAX_TOKENS: u32 = 64; + +/// System prompt for the fast-model preamble (see `SMOOTH_AGENT_PREAMBLE_MODEL`). +/// One short present-tense sentence describing intent — no answer (it's generated +/// WITHOUT the tool result), no greeting, no promises. +const PREAMBLE_SYSTEM_PROMPT: &str = "You are the assistant's voice while it works. \ +In ONE short present-tense sentence (max ~12 words), tell the user what you're about to do to help with their message. \ +Do NOT answer the question, do NOT greet, do NOT promise a specific result or outcome. \ +Example: \"Let me pull up your recent conversations.\" \ +Reply with only that sentence — no quotes, no preamble, no markdown."; + /// Registers a parked turn's [`HumanResponse`] sender under a session id (so a /// later `confirm_tool_action` can take it). Typically `AppState::register_confirmation`. pub type RegisterConfirmation = Arc) + Send + Sync>; @@ -387,6 +400,25 @@ pub async fn run_streaming_turn( let model_for_span = llm.model.clone(); let org_id_for_span = org_id.clone(); + // Optional fast-model preamble. When `SMOOTH_AGENT_PREAMBLE_MODEL` is set, a + // small fast model (e.g. `groq-gpt-oss-20b`) runs in PARALLEL with the main + // turn and streams ONE ephemeral "what I'm about to do" sentence + // (`stream_preamble`), covering the reasoning model's time-to-first-token. + // Built from a clone of `llm` (same gateway/key) with the model + a tight + // token cap overridden. Unset/empty ⇒ off, so default behavior is unchanged. + // Skipped when a test provider is injected (no live gateway). Pearl th-9a5794. + let preamble_llm = (llm_provider.is_none()) + .then(|| std::env::var("SMOOTH_AGENT_PREAMBLE_MODEL").ok()) + .flatten() + .map(|m| m.trim().to_string()) + .filter(|m| !m.is_empty()) + .map(|model| { + let mut cfg = llm.clone(); + cfg.model = model; + cfg.max_tokens = PREAMBLE_MAX_TOKENS; + cfg + }); + // The ONE ACL-enforcing knowledge handle both retrieval paths read through. // Built once from the requester's `AccessContext` so the auto-injected // context query, the agent's `knowledge_search` tool, and the citation @@ -639,6 +671,42 @@ pub async fn run_streaming_turn( let request_id_owned = request_id.to_string(); let sink_clone = sink.clone(); + // First-answer-token guard for the optional preamble: the translator flips it + // on the first `TokenDelta` so a slow preamble never pops in AFTER the real + // answer has already begun streaming. Pearl th-9a5794. + let answer_started = Arc::new(std::sync::atomic::AtomicBool::new(false)); + let answer_started_translator = Arc::clone(&answer_started); + + // Fire the fast-model preamble in PARALLEL with the agent loop (no-op unless + // `SMOOTH_AGENT_PREAMBLE_MODEL` is set → `preamble_llm` is `Some`). Best-effort: + // any error/slowness is swallowed on its own task, and the guard drops it if the + // real answer already started — so it can never block or corrupt the turn. + // Pearl th-9a5794. + if let Some(pre_cfg) = preamble_llm { + let sink_preamble = sink.clone(); + let request_id_preamble = request_id.to_string(); + let user_preamble = user_message.to_string(); + let answer_started_preamble = Arc::clone(&answer_started); + tokio::spawn(async move { + let client = LlmClient::new(pre_cfg); + let sys = EngineMessage::system(PREAMBLE_SYSTEM_PROMPT); + let user = EngineMessage::user(user_preamble); + match client.chat(&[&sys, &user], &[]).await { + Ok(resp) => { + let text = resp.content.trim(); + if !text.is_empty() + && !answer_started_preamble.load(std::sync::atomic::Ordering::Relaxed) + { + let _ = sink_preamble + .send(crate::protocol::stream_preamble(&request_id_preamble, text)); + } + } + // Best-effort: a failed/slow preamble must never surface or block. + Err(e) => tracing::debug!(error = %e, "preamble generation failed (ignored)"), + } + }); + } + // Spawn the event translator so we forward tokens to the client in real // time while the agent loop runs concurrently. let translator = tokio::spawn(async move { @@ -681,6 +749,9 @@ pub async fn run_streaming_turn( while let Some(event) = rx.recv().await { match event { AgentEvent::TokenDelta { content } => { + // Mark the answer as started so a still-in-flight preamble is + // dropped rather than popping in after the reply. Pearl th-9a5794. + answer_started_translator.store(true, std::sync::atomic::Ordering::Relaxed); streamed_reply.push_str(&content); let safe = suppressor.push(&content); if !safe.is_empty() { diff --git a/spec/events/stream-preamble.schema.json b/spec/events/stream-preamble.schema.json new file mode 100644 index 00000000..69a5c3dc --- /dev/null +++ b/spec/events/stream-preamble.schema.json @@ -0,0 +1,44 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://smooth-agent.dev/spec/events/stream-preamble.schema.json", + "title": "StreamPreamble", + "description": "Event: `stream_preamble`. A single token of a short, present-tense \"what I'm about to do\" sentence, generated by a small fast model IN PARALLEL with the main turn to cover the reasoning model's time-to-first-token. Emitted only when the server is configured with `SMOOTH_AGENT_PREAMBLE_MODEL`. Shaped identically to `stream_token` so clients can reuse the render path, but on a distinct `type` so it is shown as an EPHEMERAL status line that the real answer replaces — it is NEVER folded into the answer, and the final response (carried by `eventual_response`) never includes it. The server suppresses it if the real answer has already begun streaming. Clients that do not recognize this event MUST ignore it — the answer still streams via `stream_token`, so the preamble simply isn't shown.", + "type": "object", + "required": ["type", "data"], + "additionalProperties": false, + "properties": { + "type": { + "type": "string", + "const": "stream_preamble", + "description": "Event type discriminator." + }, + "requestId": { + "type": "string", + "description": "Echoes the `requestId` from the originating `send_message` action." + }, + "token": { + "type": "string", + "description": "The raw preamble token text. Also present inside `data.token` for consumers that only inspect `data`." + }, + "data": { + "type": "object", + "description": "Preamble token event payload.", + "required": ["requestId", "token"], + "additionalProperties": false, + "properties": { + "requestId": { + "type": "string", + "description": "The request ID this preamble token belongs to." + }, + "token": { + "type": "string", + "description": "The raw preamble token text." + } + } + }, + "timestamp": { + "type": "integer", + "description": "Unix epoch milliseconds when the event was emitted." + } + } +} diff --git a/typescript/src/generated/types.ts b/typescript/src/generated/types.ts index 357c45ad..356848a1 100644 --- a/typescript/src/generated/types.ts +++ b/typescript/src/generated/types.ts @@ -1474,6 +1474,42 @@ export interface StreamChunk { timestamp?: number; } +// ── from events/stream-preamble.schema.json ── +/** + * Event: `stream_preamble`. A single token of a short, present-tense "what I'm about to do" sentence, generated by a small fast model IN PARALLEL with the main turn to cover the reasoning model's time-to-first-token. Emitted only when the server is configured with `SMOOTH_AGENT_PREAMBLE_MODEL`. Shaped identically to `stream_token` so clients can reuse the render path, but on a distinct `type` so it is shown as an EPHEMERAL status line that the real answer replaces — it is NEVER folded into the answer, and the final response (carried by `eventual_response`) never includes it. The server suppresses it if the real answer has already begun streaming. Clients that do not recognize this event MUST ignore it — the answer still streams via `stream_token`, so the preamble simply isn't shown. + */ +export interface StreamPreamble { + /** + * Event type discriminator. + */ + type: 'stream_preamble'; + /** + * Echoes the `requestId` from the originating `send_message` action. + */ + requestId?: string; + /** + * The raw preamble token text. Also present inside `data.token` for consumers that only inspect `data`. + */ + token?: string; + /** + * Preamble token event payload. + */ + data: { + /** + * The request ID this preamble token belongs to. + */ + requestId: string; + /** + * The raw preamble token text. + */ + token: string; + }; + /** + * Unix epoch milliseconds when the event was emitted. + */ + timestamp?: number; +} + // ── from events/stream-reasoning.schema.json ── /** * Event: `stream_reasoning`. A single *reasoning* token from a reasoning-model's separate thinking channel (`reasoning_content`/`reasoning` deltas — e.g. DeepSeek, gpt-oss/harmony, MiniMax, GLM), forwarded to the client in real time. Corresponds to smooth-operator's `AgentEvent::ReasoningDelta`. Shaped identically to `stream_token` so clients can render it the same way, but on a distinct `type` so reasoning is shown as collapsible "thinking" and is NEVER folded into the answer. The final response (carried by `eventual_response`) already excludes reasoning. Clients that do not recognize this event MUST ignore it — the answer still streams via `stream_token`, so reasoning simply isn't shown. diff --git a/typescript/src/types.ts b/typescript/src/types.ts index e751dfa4..00a8ca07 100644 --- a/typescript/src/types.ts +++ b/typescript/src/types.ts @@ -34,6 +34,7 @@ import type { SendMessageRequest, SendMessageResponse, StreamChunk, + StreamPreamble, StreamToken, SubmitInteractionRequest, VerifyOtpRequest, @@ -69,6 +70,7 @@ export const EVENT_TYPES = [ 'eventual_response', 'stream_chunk', 'stream_token', + 'stream_preamble', 'keepalive', 'write_confirmation_required', 'otp_verification_required', @@ -109,6 +111,7 @@ export type ServerEvent = | EventualResponse | StreamChunk | StreamToken + | StreamPreamble | Keepalive | WriteConfirmationRequired | OtpVerificationRequired @@ -128,6 +131,7 @@ export interface ServerEventByType { eventual_response: EventualResponse; stream_chunk: StreamChunk; stream_token: StreamToken; + stream_preamble: StreamPreamble; keepalive: Keepalive; write_confirmation_required: WriteConfirmationRequired; otp_verification_required: OtpVerificationRequired; diff --git a/typescript/src/validate.ts b/typescript/src/validate.ts index 80bacb24..043d0818 100644 --- a/typescript/src/validate.ts +++ b/typescript/src/validate.ts @@ -41,6 +41,7 @@ const EVENT_SCHEMA_FILE: Record = { eventual_response: 'events/eventual-response.schema.json', stream_chunk: 'events/stream-chunk.schema.json', stream_token: 'events/stream-token.schema.json', + stream_preamble: 'events/stream-preamble.schema.json', keepalive: 'events/keepalive.schema.json', write_confirmation_required: 'events/write-confirmation-required.schema.json', otp_verification_required: 'events/otp-verification-required.schema.json',