diff --git a/docs/content/docs/framework/frontend.en.mdx b/docs/content/docs/framework/frontend.en.mdx index f649e755..74d292f7 100644 --- a/docs/content/docs/framework/frontend.en.mdx +++ b/docs/content/docs/framework/frontend.en.mdx @@ -15,7 +15,12 @@ VeADK ships a React frontend that renders [A2UI](/en/docs/framework/a2ui) (agent ### What it can do - **Chat**: multi-turn conversations that render streamed A2UI cards, plus thinking, tool calls, token usage and timing. -- **Agent picker**: switch agents from the top-left; hover an agent to see its model and mounted tools. +- **Multimodal messages**: upload images, TXT/Markdown, PDF, and video; preview + and replay both user and model media from history. Chat images use compact + thumbnails and open in a zoomable full-screen viewer. +- **Skill and sub-agent invocation**: type `/` to select a mounted skill or `@` to route the turn to an eligible sub-agent. +- **Agent picker**: switch agents from the top-left; long lists scroll within + the viewport, and hovering an agent shows its model and mounted tools. - **Session history**: auto-saved, time-sorted, reopen or delete. - **Smart search**: the *Session* source full-text-searches the current agent's history; the *Web* source calls the agent's mounted web-search tool live (using credentials from the server's environment variables). - **Add an AgentKit agent**: paste a URL + API key to connect a remote agent over the ADK protocol; it then appears in the picker. @@ -64,6 +69,62 @@ cd frontend && npm run dev # http://localhost:5173 (proxies A Outside dev mode, if the built frontend directory is not found, the command fails and asks you to run `npm run build` first (or use `--dev` with the Vite dev server). +### Multimodal attachments and storage + +The composer accepts PNG, JPEG, WebP, GIF, TXT, Markdown, PDF, MP4, WebM, and QuickTime files. The default per-file limit is 20 MB. The browser uploads binary form data and never writes attachment base64 into the session. + +Media bytes are stored separately from the ADK Session: + +- Local mode writes to `/tmp/veadk-media/apps/.../sessions/.../media//` by default. `content` contains the bytes and `metadata.json` contains the name, MIME type, size, and hash. +- TOS mode writes the same two objects below `veadk-media/users//apps//sessions//media//` by default. The username is the outer tenant prefix, isolating each user's objects; username, app, and session segments are URL-encoded. +- Session Events contain only a stable Google GenAI `FileData` reference such as `veadk-media://apps/.../media/`; reopening history fetches the original object through the media API. + +Before a model call, TXT/Markdown is decoded into `Part.text`; images and video +are read from the configured backend and converted to `Part.inline_data`, while +PDFs are automatically rendered to page PNGs. The PDF rendering runtime is +included in VeADK's default dependencies, with no extra installation or Agent +callback required. Model-returned `inline_data` is persisted first, then +replaced with a stable reference before the Event is saved or emitted over SSE. +A 15-minute TOS signed URL is used only for browser download and preview, never +as the model's `FileData` URI. + +When a cloud Agent Runtime is selected, upload, preview, and deletion still use +Studio's own `/web/media` API instead of being forwarded to the Runtime. Studio +resolves stable references into model-ready Parts only while proxying +`/run_sse`. It preserves the original `veadkMedia` metadata so history reloads +the original file type and name. This works with both `/tmp` and TOS and does +not require the remote Runtime to mount media HTTP routes. + +| Environment variable | Default | Description | +| :-- | :-- | :-- | +| `VEADK_MEDIA_STORAGE` | `local` | Select `local` or `tos`. | +| `VEADK_MEDIA_LOCAL_DIR` | `/tmp/veadk-media` | Local media root. | +| `VEADK_MEDIA_MAX_FILE_BYTES` | `20971520` | Byte limit for one upload or model output. | +| `VEADK_MEDIA_TOS_PREFIX` | `veadk-media` | TOS object-key prefix. | +| `DATABASE_TOS_BUCKET` | — | TOS bucket. | +| `DATABASE_TOS_REGION` / `DATABASE_TOS_ENDPOINT` | cloud-aware defaults | TOS region and endpoint. | +| `VOLCENGINE_ACCESS_KEY` / `VOLCENGINE_SECRET_KEY` | — | TOS credentials. | +| `VOLCENGINE_SESSION_TOKEN` | — | Optional temporary credential token. | + +Removing an unsent attachment deletes its object; deleting a Session cleans up all media scoped to that Session. Local mode needs no additional configuration, but `/tmp` may be cleared when the process or host is replaced. Use TOS when attachments must remain durable. Minimal TOS configuration: + +```bash +export VEADK_MEDIA_STORAGE=tos +export DATABASE_TOS_BUCKET= +export DATABASE_TOS_REGION=cn-beijing +export VOLCENGINE_ACCESS_KEY= +export VOLCENGINE_SECRET_KEY= +veadk frontend --agents-dir examples +``` + +### Using skills and sub-agents + +Type `/` in the composer to search skills mounted on the current agent. Type `@` to search eligible descendants in its agent tree. Use the arrow keys to move, Enter or Tab to select, and Escape to close the menu. A selection becomes a removable chip instead of remaining ordinary message text. + +After selecting a sub-agent, the `/` menu shows skills mounted on that target. Changing or removing the target clears its selected skills, preventing a skill from being sent to an agent that does not own it. `task` and `single_turn` workflow nodes remain visible in the topology but cannot be selected with `@`. + +The frontend sends structured `veadkInvocation` metadata instead of inferring invocation intent from the message string. The backend plugin directs ADK to call the skill tool or invoke `transfer_to_agent` one tree edge at a time until it reaches the target. The same metadata is written to the first Google GenAI `Part`, so reopening session history restores the `/skill` and `@agent` chips. + ### IAM permissions for `veadk studio deploy` When `--iam-role` is omitted, the deploy command creates or reuses `VeADKFrontendServiceRole` and ensures that the role has these Volcengine system policies: @@ -148,9 +209,9 @@ username (letters + digits, ≤16), stored locally and used as the `user_id`. Login state is cached: SSO via the `veadk_session` cookie, local mode via - `localStorage`. The session itself is created lazily on the first message (not - on page load). Logout is a local logout (clears the session and returns to the - login page). + `localStorage`. The session itself is created lazily on the first message or + attachment upload (not on page load). Logout is a local logout (clears the + session and returns to the login page). ### How it works @@ -161,6 +222,14 @@ username (letters + digits, ≤16), stored locally and used as the `user_id`. `adk/client.ts` calls `/list-apps`, creates a session, and streams `/run_sse`. + +`veadk.multimodal` validates uploads, manages local/TOS storage, resolves media references before model calls, and persists model-returned media before history is saved. + + + +`veadk.cli.frontend_invocation` exposes mounted skills and translates structured `/skill` and `@agent` selections into ADK skill-tool and agent-transfer directives. + + `a2ui/extract.ts` pulls A2UI messages out of the `send_a2ui_json_to_client` tool response (`validated_a2ui_json`). diff --git a/docs/content/docs/framework/frontend.mdx b/docs/content/docs/framework/frontend.mdx index 0b0640f2..b5037a1d 100644 --- a/docs/content/docs/framework/frontend.mdx +++ b/docs/content/docs/framework/frontend.mdx @@ -15,7 +15,10 @@ VeADK 自带一个 React 前端，用于渲染智能体经由 Google ADK API Ser ### 功能一览 - **对话**：与智能体多轮对话，渲染流式返回的 A2UI 卡片，并展示思考过程、工具调用、Token 与用时。 -- **Agent 选择器**：左上角切换 agent；悬停可查看该 agent 的模型与挂载的工具。 +- **多模态消息**：上传图片、TXT/Markdown、PDF 与视频；用户附件和模型返回的媒体均支持预览与历史回放。 + 对话中的图片默认以紧凑尺寸展示，点击后可全屏缩放预览。 +- **技能与子 Agent 调用**：在输入框输入 `/` 选择挂载技能，输入 `@` 将本轮交给可选的子 Agent。 +- **Agent 选择器**：左上角切换 agent；长列表在视口内独立滚动，悬停可查看该 agent 的模型与挂载的工具。 - **历史会话**：自动保存、按时间排序，可重新打开或删除。 - **智能搜索**：「会话」源在当前 agent 的历史消息中做全文检索；「网页」源调用该 agent 挂载的联网搜索工具实时检索（使用服务端环境变量里的凭据）。 - **添加 AgentKit 智能体**：填入访问地址 + API Key，按 ADK 协议接入远程 agent，接入后出现在选择器中。 @@ -64,6 +67,57 @@ cd frontend && npm run dev # http://localhost:5173（代理 A 非开发模式下，若未找到已构建的前端目录，命令会报错提示先执行 `npm run build`（或改用 `--dev` 配合 Vite 开发服务器）。 +### 多模态附件与存储 + +输入框支持 PNG、JPEG、WebP、GIF、TXT、Markdown、PDF、MP4、WebM 和 QuickTime。默认单文件上限为 20 MB。浏览器使用二进制表单上传，不会把附件转成 base64 写入会话。 + +媒体本体与 ADK Session 分开保存： + +- 本地模式默认写入 `/tmp/veadk-media/apps/.../sessions/.../media//`，其中 `content` 是文件本体，`metadata.json` 是文件名、MIME、大小和哈希等元数据。 +- TOS 模式默认写入 `veadk-media/users//apps//sessions//media//` 下的同名两个对象。用户名位于最外层租户 prefix，不同用户的数据相互隔离；username、app 和 session 路径段都会进行 URL 编码。 +- Session Event 只保存 Google GenAI `FileData` 稳定引用，例如 `veadk-media://apps/.../media/`；重新打开历史会话时再通过媒体 API 读取本体。 + +调用模型前，TXT/Markdown 会解码为 `Part.text`，图片和视频会从当前存储读取并转换为 +`Part.inline_data`，PDF 则自动渲染为逐页 PNG 图片。PDF 渲染运行时包含在 VeADK +默认依赖中，不需要额外安装或配置 Agent callback。模型返回的 `inline_data` 会先落到 +媒体存储，再在 Event 持久化和 SSE 输出前改写为稳定引用。TOS 的 15 分钟签名 URL +只用于浏览器下载/预览，不作为模型的 `FileData` URI。 + +选择云端 Agent Runtime 时，上传、预览和删除仍由 Studio 自身的 `/web/media` +接口处理，不会转发到 Runtime。Studio 仅在代理 `/run_sse` 时读取稳定引用，将内容转换为 +远端模型可直接消费的 Part；原始 `veadkMedia` 元数据会保留，因此历史消息仍按原文件类型 +和名称加载。这个流程同时支持默认 `/tmp` 与 TOS，无需远端 Runtime 挂载媒体 HTTP 路由。 + +| 环境变量 | 默认值 | 说明 | +| :-- | :-- | :-- | +| `VEADK_MEDIA_STORAGE` | `local` | 选择 `local` 或 `tos`。 | +| `VEADK_MEDIA_LOCAL_DIR` | `/tmp/veadk-media` | 本地媒体根目录。 | +| `VEADK_MEDIA_MAX_FILE_BYTES` | `20971520` | 单个上传文件或模型输出的字节上限。 | +| `VEADK_MEDIA_TOS_PREFIX` | `veadk-media` | TOS 对象 Key 前缀。 | +| `DATABASE_TOS_BUCKET` | — | TOS Bucket。 | +| `DATABASE_TOS_REGION` / `DATABASE_TOS_ENDPOINT` | 按云环境推导 | TOS Region 与 Endpoint。 | +| `VOLCENGINE_ACCESS_KEY` / `VOLCENGINE_SECRET_KEY` | — | TOS 访问凭据。 | +| `VOLCENGINE_SESSION_TOKEN` | — | 可选的临时凭据 Token。 | + +移除尚未发送的附件会立即删除对应对象；删除 Session 会清理该 Session 下的全部媒体。本地模式无需额外配置，但 `/tmp` 可能随进程或宿主机回收而被清理；需要持久保存附件时应使用 TOS。启用 TOS 的最小配置如下： + +```bash +export VEADK_MEDIA_STORAGE=tos +export DATABASE_TOS_BUCKET= +export DATABASE_TOS_REGION=cn-beijing +export VOLCENGINE_ACCESS_KEY= +export VOLCENGINE_SECRET_KEY= +veadk frontend --agents-dir examples +``` + +### 使用技能与子 Agent + +在输入框中输入 `/` 可搜索当前 Agent 挂载的技能，输入 `@` 可搜索其 Agent 树中允许转移的后代节点。使用上下方向键移动，按 Enter 或 Tab 选择，按 Escape 关闭菜单。选中项会变成可移除的 chip，不会作为普通文本留在消息中。 + +选中子 Agent 后，`/` 菜单会改为展示目标 Agent 自己挂载的技能。切换或移除目标时会清空原有技能，避免把技能发送给没有挂载它的 Agent。`task` 和 `single_turn` 工作流节点仍会显示在拓扑中，但不能通过 `@` 选择。 + +前端不会从消息字符串中猜测调用意图，而是发送结构化的 `veadkInvocation` metadata。后端插件据此要求 ADK 调用技能工具，或沿 Agent 树逐级调用 `transfer_to_agent`，直到到达目标节点。同一份 metadata 也会写入首个 Google GenAI `Part`，因此重新加载历史会话后仍能恢复 `/skill` 与 `@agent` chip。 + ### `veadk studio deploy` 的 IAM 权限 未传入 `--iam-role` 时，部署命令会创建或复用 `VeADKFrontendServiceRole`，并确保该 Role 绑定以下火山引擎系统策略： @@ -128,7 +182,7 @@ Google 同理，把 `OAUTH2_PROVIDER` 换成 `google`；Keycloak / Auth0 / Okta **无 SSO（本地用户名）** —— 不传上述参数时，登录页会让用户输入一个用户名（字母 + 数字，≤16 位），保存在本地并作为 `user_id`。 - 登录态会被缓存：SSO 走 `veadk_session` Cookie，本地模式走 `localStorage`。会话本身**在发送第一条消息时才创建**（而非打开页面时）。退出登录为本地登出（清除会话并回到登录页）。 + 登录态会被缓存：SSO 走 `veadk_session` Cookie，本地模式走 `localStorage`。会话本身**在发送第一条消息或上传第一个附件时才创建**（而非打开页面时）。退出登录为本地登出（清除会话并回到登录页）。 ### 工作原理 @@ -139,6 +193,14 @@ Google 同理，把 `OAUTH2_PROVIDER` 换成 `google`；Keycloak / Auth0 / Okta `adk/client.ts` 调用 `/list-apps`，创建会话，并通过 `/run_sse` 进行流式接收。 + +`veadk.multimodal` 校验上传，统一管理本地/TOS 存储，在模型调用前解析媒体引用，并在历史持久化前保存模型返回的媒体。 + + + +`veadk.cli.frontend_invocation` 暴露 Agent 挂载的技能，并把结构化的 `/skill`、`@agent` 选择转换为 ADK 技能工具与 Agent 转移指令。 + + `a2ui/extract.ts` 从 `send_a2ui_json_to_client` 工具的响应（`validated_a2ui_json`）中提取 A2UI 消息。 diff --git a/examples/README.md b/examples/README.md index 25751aa1..de6f0253 100644 --- a/examples/README.md +++ b/examples/README.md @@ -22,8 +22,12 @@ and a bilingual README (English + 中文). | 10 | [Agent routing](./10_agent_routing/) | Complex | A coordinator that delegates to specialists dynamically | | 11 | [Tracing](./11_tracing/) | Complex | Observe LLM/tool calls; dump or export spans | -There is also [`a2ui_agent/`](./a2ui_agent/) — a demo of agent-driven UI, run with -`veadk frontend --agents-dir examples`. +There are also frontend-focused demos that run with +`veadk frontend --agents-dir examples`: + +- [`a2ui_agent/`](./a2ui_agent/) demonstrates agent-driven UI. +- [`multimodal_agent/`](./multimodal_agent/) analyzes images, TXT/Markdown, + PDFs, and videos uploaded from the chat composer. For a deployable **full app** (web UI + agent API in one container, shipped to Volcengine AgentKit via `veadk agentkit`), see [`basic-app/`](./basic-app/). diff --git a/examples/README.zh.md b/examples/README.zh.md index 22b8f4c1..6dc6e8c9 100644 --- a/examples/README.zh.md +++ b/examples/README.zh.md @@ -21,8 +21,11 @@ | 10 | [智能体路由](./10_agent_routing/) | 复杂 | 协调者动态委派给专家智能体 | | 11 | [链路追踪](./11_tracing/) | 复杂 | 观测大模型/工具调用；导出 span | -另外还有 [`a2ui_agent/`](./a2ui_agent/) —— 一个由智能体驱动 UI 的示例， -可通过 `veadk frontend --agents-dir examples` 运行。 +另外还有可通过 `veadk frontend --agents-dir examples` 运行的 frontend 示例： + +- [`a2ui_agent/`](./a2ui_agent/) 展示由智能体驱动的 UI。 +- [`multimodal_agent/`](./multimodal_agent/) 分析从聊天输入框上传的图片、 + TXT/Markdown、PDF 和视频。 如需一个可部署的**完整应用**（Web 前端 + Agent API 同处一个容器，通过 `veadk agentkit` 部署到火山引擎 AgentKit），参见 [`basic-app/`](./basic-app/)。 diff --git a/examples/multimodal_agent/.env.example b/examples/multimodal_agent/.env.example new file mode 100644 index 00000000..0968158e --- /dev/null +++ b/examples/multimodal_agent/.env.example @@ -0,0 +1,13 @@ +# Copy this file to the repository-root `.env` before starting the frontend. + +MODEL_AGENT_PROVIDER=openai +MODEL_AGENT_NAME=doubao-seed-2-1-pro-260628 +MODEL_AGENT_API_BASE=https://ark.cn-beijing.volces.com/api/v3/ +MODEL_AGENT_API_KEY=your-ark-api-key-here + +# Optional generation-tool overrides. The API keys fall back to +# MODEL_AGENT_API_KEY when omitted. +MODEL_IMAGE_NAME=doubao-seedream-5-0-260128 +MODEL_IMAGE_API_BASE=https://ark.cn-beijing.volces.com/api/v3/ +MODEL_VIDEO_NAME=doubao-seedance-2-0-260128 +MODEL_VIDEO_API_BASE=https://ark.cn-beijing.volces.com/api/v3/ diff --git a/examples/multimodal_agent/README.md b/examples/multimodal_agent/README.md new file mode 100644 index 00000000..d8da8e3e --- /dev/null +++ b/examples/multimodal_agent/README.md @@ -0,0 +1,36 @@ +# Multimodal Agent + +This demo uses the VeADK frontend's multimodal pipeline to analyze images, +TXT/Markdown documents, PDFs, and videos, and mounts the `image_generate` and +`video_generate` tools. The frontend uploads files, `MultimodalMediaPlugin` +resolves them into Google GenAI Parts, and `doubao-seed-2-1-pro-260628` analyzes +them. Image or video creation requests call the corresponding tool. + +## Run + +From the repository root: + +```bash +cp examples/multimodal_agent/.env.example .env +uv run veadk frontend --agents-dir examples --dev +``` + +Open , select `multimodal_agent`, then use the `+` button +in the composer to upload one or more files. + +Example prompts: + +- Image: `Describe the scene and extract all visible text.` +- TXT/Markdown: `Summarize this document into five actionable points.` +- PDF: `Explain the main argument and list the supporting evidence.` +- Video: `Give me a timeline of the important events in this video.` +- Mixed files: `Compare these files and identify any contradictions.` +- Image generation: `Create a cinematic image of Shanghai on a rainy night.` +- Video generation: `Create a time-lapse video of sunrise over the ocean.` + +The Agent explicitly uses VeADK's default 2.1 Pro model. The generation tools +fall back to `MODEL_AGENT_API_KEY`; set `MODEL_IMAGE_API_KEY` and +`MODEL_VIDEO_API_KEY` to override them independently. Local uploads are stored +under `/tmp/veadk-media` by default; configure TOS for durable storage. PDFs +are automatically rendered to page images before the model call, with no extra +dependency or Agent callback required. diff --git a/examples/multimodal_agent/README.zh.md b/examples/multimodal_agent/README.zh.md new file mode 100644 index 00000000..383f53ff --- /dev/null +++ b/examples/multimodal_agent/README.zh.md @@ -0,0 +1,34 @@ +# 多模态 Agent + +这个 Demo 使用 VeADK frontend 的多模态链路分析图片、TXT/Markdown、PDF +和视频，并挂载 `image_generate` 与 `video_generate` 工具。frontend 负责上传 +文件，`MultimodalMediaPlugin` 将文件还原为 Google GenAI Parts，再交给 +`doubao-seed-2-1-pro-260628` 分析；生成图片或视频时则调用对应工具。 + +## 运行 + +在仓库根目录执行： + +```bash +cp examples/multimodal_agent/.env.example .env +uv run veadk frontend --agents-dir examples --dev +``` + +打开 ，选择 `multimodal_agent`，然后通过输入框中的 +`+` 按钮上传一个或多个文件。 + +示例问题： + +- 图片：`描述画面，并提取所有可见文字。` +- TXT/Markdown：`把这份文档总结成五条可执行建议。` +- PDF：`解释核心观点，并列出支撑证据。` +- 视频：`按时间线总结视频中的重要事件。` +- 混合文件：`比较这些文件，并找出相互矛盾的地方。` +- 图片生成：`生成一张雨夜上海街头的电影感图片。` +- 视频生成：`生成一段海边日出的延时摄影视频。` + +Agent 明确使用 VeADK 默认的 2.1 Pro 模型。图片与视频生成工具默认复用 +`MODEL_AGENT_API_KEY`；也可以分别设置 `MODEL_IMAGE_API_KEY` 和 +`MODEL_VIDEO_API_KEY`。本地上传默认保存在 `/tmp/veadk-media`；需要持久化时 +请配置 TOS。PDF 会在调用模型前自动渲染为页面图片，无需安装额外依赖或配置 +Agent callback。 diff --git a/examples/multimodal_agent/agent.py b/examples/multimodal_agent/agent.py new file mode 100644 index 00000000..4a453944 --- /dev/null +++ b/examples/multimodal_agent/agent.py @@ -0,0 +1,50 @@ +# Copyright (c) 2025 Beijing Volcano Engine Technology Co., Ltd. and/or its affiliates. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""A frontend-ready Agent that understands common multimodal attachments.""" + +from veadk import Agent +from veadk.consts import DEFAULT_MODEL_AGENT_NAME +from veadk.tools.builtin_tools.image_generate import image_generate +from veadk.tools.builtin_tools.video_generate import video_generate + +INSTRUCTION = """You are a multimodal analysis assistant. + +The user may attach images, TXT or Markdown documents, PDFs, and videos. Inspect +every attachment that is available in the request and answer in the user's +language. + +- For images, describe relevant visual details and read visible text when useful. +- For TXT or Markdown, summarize or answer from the document content. +- For PDFs, explain the structure and key findings and cite page numbers when the + input exposes them. +- For videos, describe the timeline, important scenes, actions, and visible text. +- When several attachments are present, compare and connect their information. +- When the user asks to create an image, call `image_generate`. +- When the user asks to create a video, call `video_generate`. + +Distinguish observations from inferences. Never claim to have inspected content +that is unavailable or unreadable; explain the limitation instead. +""" + +agent = Agent( + name="multimodal_agent", + description="Analyzes multimodal files and generates images or videos.", + instruction=INSTRUCTION, + model_name=DEFAULT_MODEL_AGENT_NAME, + tools=[image_generate, video_generate], +) + +# Required by the Google ADK agent loader. +root_agent = agent diff --git a/frontend/README.md b/frontend/README.md index c7e7d531..10107e57 100644 --- a/frontend/README.md +++ b/frontend/README.md @@ -7,9 +7,14 @@ server that `veadk frontend` launches — no separate backend. - **Streaming chat** over the ADK `/run_sse` event stream. - **Markdown** rendering for user and assistant messages (GFM + code highlight). +- **Multimodal messages** with images, TXT/Markdown, PDF, and video attachments, + including previews and history replay for both user and model media. Chat + images use compact thumbnails and open in a zoomable full-screen viewer. +- **Composer invocations**: type `/` to select a mounted skill or `@` to route + the turn to a mentionable sub-agent. - **Reasoning & tool calls** shown inline (collapsible "thinking", tool blocks). - **Sessions**: pick an agent, browse history, new chat, delete — per signed-in - user. + user. Long Agent lists stay within the viewport and scroll independently. - **Tracing viewer**: a span tree + detail panel from the ADK debug trace. - **Custom-agent workbench**: configure and debug an agent, then review generated source with line numbers and syntax highlighting before setting @@ -70,12 +75,84 @@ exposed at `GET /web/auth-config`. (letters + digits, ≤16), stored locally and used as the `user_id`. Login state is cached: SSO via the `veadk_session` cookie, local mode via -`localStorage`. The session itself is created lazily on the first message. +`localStorage`. The session itself is created lazily on the first message or +attachment upload. + +## Multimodal media + +The composer accepts PNG, JPEG, WebP, GIF, TXT, Markdown, PDF, MP4, WebM, and +QuickTime files. The default per-file limit is 20 MB. Files are uploaded as +binary form data; the browser does not put base64 payloads into chat events. + +Media bytes live outside the ADK session store: + +- Local mode stores `content` and `metadata.json` below + `/tmp/veadk-media/apps/.../sessions/.../media//` by default. +- TOS mode stores the same two objects below + `veadk-media/users//apps//sessions//media//` + by default. The user-first prefix keeps each tenant's objects separate; + username, app, and session segments are URL-encoded. +- Session events contain only a stable Google GenAI `FileData` reference such + as `veadk-media://apps/.../media/`, so history stays small and can + load the original attachment later. + +Immediately before a model call, TXT and Markdown are decoded into `Part.text`; +images and video are loaded from the selected backend into `Part.inline_data`, +and PDF pages are rendered to PNG images. PDF support and its rendering runtime +are included in the default VeADK installation. Model-returned `inline_data` is +persisted first and replaced with the same stable reference before the event is +saved or streamed. TOS uses a 15-minute signed URL only for browser delivery, +not as a model `FileData` URI. + +For cloud AgentKit runtimes, media HTTP operations remain on the Studio server; +they are not sent to `/web/runtime-proxy/.../web/media`. The Studio proxy +resolves stored references into model-ready Parts only for `/run_sse` and keeps +the original `veadkMedia` metadata so history still renders the original +attachment. Both the default `/tmp` backend and TOS work without adding media +routes to the remote runtime. + +| Environment variable | Default | Purpose | +| :-- | :-- | :-- | +| `VEADK_MEDIA_STORAGE` | `local` | Select `local` or `tos`. | +| `VEADK_MEDIA_LOCAL_DIR` | `/tmp/veadk-media` | Local media root. | +| `VEADK_MEDIA_MAX_FILE_BYTES` | `20971520` | Upload/model-output limit. | +| `VEADK_MEDIA_TOS_PREFIX` | `veadk-media` | TOS object-key prefix. | +| `DATABASE_TOS_BUCKET` | — | TOS bucket name. | +| `DATABASE_TOS_REGION` | cloud-aware | TOS region. | +| `DATABASE_TOS_ENDPOINT` | region-aware | TOS endpoint. | +| `VOLCENGINE_ACCESS_KEY` / `VOLCENGINE_SECRET_KEY` | — | TOS credentials. | +| `VOLCENGINE_SESSION_TOKEN` | — | Optional temporary credential token. | + +Deleting a draft attachment deletes its object. Deleting a session deletes all +media scoped to that session from either backend. Because `/tmp` may be cleared +at any time, use TOS when attachments must survive process or host replacement. + +## Skills and sub-agents + +Type `/` in the composer to search skills mounted on the selected agent. Type +`@` to search any mentionable descendant in its sub-agent tree. Use the arrow +keys to move, Enter or Tab to select, and Escape to close the menu. A selected +item becomes a removable chip instead of remaining plain message text. + +After selecting a sub-agent, the `/` menu shows that target's skills. Changing +or removing the target clears its selected skills, so a skill is never sent to +an agent that does not own it. Task and single-turn workflow nodes are shown in +the topology but cannot be selected with `@`. + +Selections are sent as structured `veadkInvocation` metadata, not parsed from +the message string. The invocation plugin directs ADK to call the mounted skill +tool or transfer one tree edge at a time until it reaches the selected agent. +The same metadata is attached to the first Google GenAI `Part`, so session +history restores the `/skill` and `@agent` chips after a reload. ## How it works - `adk/client.ts` calls `/list-apps`, creates a session, and streams `/run_sse`; events are normalised into ordered blocks (`blocks.ts`). +- `veadk.multimodal` validates uploads, abstracts local/TOS storage, resolves + stable references for model calls, and persists model-returned media. +- `veadk.cli.frontend_invocation` exposes mounted skills and translates + structured composer selections into ADK skill and transfer tool directives. - `ui/` holds the chat shell: sidebar, composer, message blocks, trace drawer. - `adk/identity.ts` resolves the user (SSO `userinfo` or local username). diff --git a/frontend/package-lock.json b/frontend/package-lock.json index fd2c24c7..a2d438a3 100644 --- a/frontend/package-lock.json +++ b/frontend/package-lock.json @@ -21,6 +21,7 @@ "react": "^18.3.1", "react-dom": "^18.3.1", "react-markdown": "^9.1.0", + "react-photo-view": "^1.2.7", "rehype-highlight": "^7.0.2", "remark-gfm": "^4.0.1", "yaml": "^2.9.0" @@ -3466,6 +3467,16 @@ "react": ">=18" } }, + "node_modules/react-photo-view": { + "version": "1.2.7", + "resolved": "https://registry.npmmirror.com/react-photo-view/-/react-photo-view-1.2.7.tgz", + "integrity": "sha512-MfOWVPxuibncRLaycZUNxqYU8D9IA+rbGDDaq6GM8RIoGJal592hEJoRAyRSI7ZxyyJNJTLMUWWL3UIXHJJOpw==", + "license": "Apache-2.0", + "peerDependencies": { + "react": ">=16.8.0", + "react-dom": ">=16.8.0" + } + }, "node_modules/react-refresh": { "version": "0.17.0", "resolved": "https://registry.npmmirror.com/react-refresh/-/react-refresh-0.17.0.tgz", diff --git a/frontend/package.json b/frontend/package.json index a647a772..76e79173 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -23,6 +23,7 @@ "react": "^18.3.1", "react-dom": "^18.3.1", "react-markdown": "^9.1.0", + "react-photo-view": "^1.2.7", "rehype-highlight": "^7.0.2", "remark-gfm": "^4.0.1", "yaml": "^2.9.0" diff --git a/frontend/src/App.tsx b/frontend/src/App.tsx index 82c64d89..e74910dc 100644 --- a/frontend/src/App.tsx +++ b/frontend/src/App.tsx @@ -6,7 +6,6 @@ import { CircleCheck, CircleX, Copy, - FileText, ListTodo, Loader2, } from "lucide-react"; @@ -14,15 +13,23 @@ import { motion } from "motion/react"; import { cancelAgentkitDeployment, createSession, + deleteMedia, + deleteSessionMedia, deleteSession, + getAgentInfo, getSession, listApps, listSessions, runSSE, + uploadMedia, getUiConfig, type AdkEvent, + type AgentInfo, + type AgentNode, + type AgentTarget, type AdkSession, type Attachment, + type FrontendInvocation, type UiFeatures, } from "./adk/client"; import { applyEvent, emptyAcc, eventsToTurns, type Block, type Turn } from "./blocks"; @@ -42,6 +49,8 @@ import { } from "./adk/connections"; import { Blocks, ThinkingPlaceholder } from "./ui/Blocks"; import { Composer } from "./ui/Composer"; +import { InvocationChips } from "./ui/InvocationChips"; +import { MediaGroup } from "./ui/Media"; import { QuickCreate, type QuickCreateKind } from "./ui/QuickCreate"; import { StackCards } from "./ui/AddAgentMenu"; import { IntelligentCreate } from "./create/IntelligentCreate"; @@ -67,6 +76,35 @@ type CreateView = "menu" | QuickCreateKind | null; const LS = { app: "veadk.appName", view: "veadk.view", session: "veadk.sessionId" } as const; const EMPTY_STRING_SET: Set = new Set(); const EMPTY_STRING_ARR: string[] = []; + +function emptyInvocation(): FrontendInvocation { + return { skills: [] }; +} + +function findAgentNode(node: AgentNode, name: string): AgentNode | undefined { + if (node.name === name) return node; + for (const child of node.children) { + const found = findAgentNode(child, name); + if (found) return found; + } + return undefined; +} + +function mentionableDescendants(node: AgentNode): AgentTarget[] { + const targets: AgentTarget[] = []; + for (const child of node.children) { + if (!child.mentionable) continue; + targets.push({ + name: child.name, + description: child.description, + type: child.type, + path: child.path, + }); + targets.push(...mentionableDescendants(child)); + } + return targets; +} + function loadView(): CreateView { const v = typeof localStorage !== "undefined" ? localStorage.getItem(LS.view) : null; return v === "menu" || v === "intelligent" || v === "custom" || v === "template" || v === "workflow" @@ -141,12 +179,13 @@ function turnText(turn: Turn): string { const A2UI_TOOL_NAME = "send_a2ui_json_to_client"; /** Whether a finalized assistant turn has anything visible to render — non-empty - * text, a renderable A2UI surface, or a non-A2UI tool. Thinking, the hidden + * text, media, a renderable A2UI surface, or a non-A2UI tool. Thinking, the hidden * (done) A2UI tool, and empty A2UI surfaces don't count, so a reply that was * ONLY thinking + an empty surface returns false (→ we show a fallback). */ function turnHasVisibleContent(turn: Turn): boolean { return turn.blocks.some((b) => { if (b.kind === "text") return b.text.trim().length > 0; + if (b.kind === "attachment") return b.files.length > 0; if (b.kind === "tool") return !(b.name === A2UI_TOOL_NAME && b.done); if (b.kind === "a2ui") return buildSurfaces(b.messages).some((s) => s.components[s.rootId]); if (b.kind === "auth") return true; // the OAuth card counts as content @@ -427,20 +466,22 @@ const GREETINGS = [ ]; const pickGreeting = () => GREETINGS[Math.floor(Math.random() * GREETINGS.length)]; -const MAX_FILE_BYTES = 20 * 1024 * 1024; // 20 MB/file (base64 inflates ~33%) +function releaseAttachmentPreviews(items: Attachment[]) { + for (const item of items) { + if (item.previewUrl?.startsWith("blob:")) URL.revokeObjectURL(item.previewUrl); + } +} -/** Read a File as base64 (without the `data:...;base64,` prefix). */ -function fileToBase64(file: File): Promise { - return new Promise((resolve, reject) => { - const reader = new FileReader(); - reader.onload = () => { - const res = String(reader.result); - const comma = res.indexOf(","); - resolve(comma >= 0 ? res.slice(comma + 1) : res); - }; - reader.onerror = () => reject(reader.error); - reader.readAsDataURL(file); - }); +function attachmentDraftId() { + return `draft-${Date.now()}-${Math.random().toString(36).slice(2)}`; +} + +function browserMimeType(file: File) { + if (file.type) return file.type; + const extension = file.name.split(".").pop()?.toLowerCase(); + if (extension === "md" || extension === "markdown") return "text/markdown"; + if (extension === "txt") return "text/plain"; + return "application/octet-stream"; } export default function App() { @@ -448,6 +489,7 @@ export default function App() { const [appName, setAppName] = useState(""); const [sessions, setSessions] = useState([]); const [sessionId, setSessionId] = useState(""); + const creatingSessionRef = useRef | null>(null); // Turns are stored PER SESSION, so a background stream can keep updating its // own session's transcript while you view another one — no cross-session // leak, no data loss, and no re-fetch when you switch back (its entry is @@ -466,6 +508,10 @@ export default function App() { })); const [input, setInput] = useState(""); const [attachments, setAttachments] = useState([]); + const [invocation, setInvocation] = useState(emptyInvocation); + const [agentInfo, setAgentInfo] = useState(null); + const [capabilitiesLoading, setCapabilitiesLoading] = useState(false); + const removedAttachmentIdsRef = useRef>(new Set()); // Streaming state is PER SESSION so multiple sessions can stream at once // (each /run_sse is an independent request). `streamingSids` = which sessions // are currently streaming; the AbortControllers let unmount / delete cancel @@ -525,6 +571,61 @@ export default function App() { const activeAgent = activeAgentBySession[sessionId] ?? ""; const seenAgents = seenAgentsBySession[sessionId] ?? EMPTY_STRING_SET; const execPath = execPathBySession[sessionId] ?? EMPTY_STRING_ARR; + const rootCapabilityNode = agentInfo?.graph; + const skillCapabilityNode = invocation.targetAgent && rootCapabilityNode + ? findAgentNode(rootCapabilityNode, invocation.targetAgent.name) + : rootCapabilityNode; + const availableSkills = skillCapabilityNode?.skills ?? + (invocation.targetAgent ? [] : agentInfo?.skills ?? []); + const availableAgents = rootCapabilityNode + ? mentionableDescendants(rootCapabilityNode) + : []; + + function discardDraftAttachments(items: Attachment[]) { + releaseAttachmentPreviews(items); + for (const item of items) { + if (item.status === "uploading") { + removedAttachmentIdsRef.current.add(item.id); + } else if (item.uri) { + void deleteMedia(appName, item.uri).catch((e) => setError(String(e))); + } + } + } + + async function abandonDraftSession(sid: string) { + try { + await deleteSessionMedia(appName, userId, sid); + await deleteSession(appName, userId, sid); + setSessions((current) => current.filter((session) => session.id !== sid)); + setTurnsBySession((current) => { + const { [sid]: _drop, ...rest } = current; + return rest; + }); + } catch (e) { + setError(String(e)); + } + } + + function removeDraftAttachment(id: string) { + const removed = attachments.find((item) => item.id === id); + if (!removed) return; + const remaining = attachments.filter((item) => item.id !== id); + releaseAttachmentPreviews([removed]); + if (removed.status === "uploading") { + removedAttachmentIdsRef.current.add(id); + } + setAttachments(remaining); + + const shouldAbandonSession = + remaining.length === 0 && !input.trim() && !!sessionId && turns.length === 0; + if (shouldAbandonSession) { + viewSidRef.current = ""; + setSessionId(""); + void abandonDraftSession(sessionId); + } else if (removed.uri) { + void deleteMedia(appName, removed.uri).catch((e) => setError(String(e))); + } + } // Apply a stream event's control-flow signals to a session's live state: // author = who's executing now; transfer_to_agent pushes the delegation @@ -737,6 +838,29 @@ export default function App() { useEffect(() => { if (appName) localStorage.setItem(LS.app, appName); }, [appName]); + useEffect(() => { + let cancelled = false; + setAgentInfo(null); + setInvocation(emptyInvocation()); + if (!appName) { + setCapabilitiesLoading(false); + return; + } + setCapabilitiesLoading(true); + getAgentInfo(appName) + .then((info) => { + if (!cancelled) setAgentInfo(info); + }) + .catch(() => { + if (!cancelled) setAgentInfo(null); + }) + .finally(() => { + if (!cancelled) setCapabilitiesLoading(false); + }); + return () => { + cancelled = true; + }; + }, [appName]); useEffect(() => { localStorage.setItem(LS.view, createView ?? "chat"); }, [createView]); @@ -817,14 +941,22 @@ export default function App() { function startNewChat() { setError(""); setGreeting(pickGreeting()); + const abandonedSession = sessionId && turns.length === 0 && attachments.length > 0 + ? sessionId + : ""; viewSidRef.current = ""; setSessionId(""); + setInvocation(emptyInvocation()); + discardDraftAttachments(attachments); + setAttachments([]); + if (abandonedSession) void abandonDraftSession(abandonedSession); } async function removeSession(id: string) { try { // Deleting a session with a running stream — abort just that one. streamAbortsRef.current.get(id)?.abort(); + await deleteSessionMedia(appName, userId, id); await deleteSession(appName, userId, id); setTurnsBySession((m) => { const { [id]: _drop, ...rest } = m; @@ -841,6 +973,7 @@ export default function App() { if (id === sessionId) return; viewSidRef.current = id; setError(""); + setInvocation(emptyInvocation()); setSessionId(id); // Already have this session's turns (it's cached, or streaming in the // background)? Show them instantly and let any live stream keep updating — @@ -857,49 +990,85 @@ export default function App() { } } + async function ensureSession(): Promise { + if (sessionId) return sessionId; + if (!creatingSessionRef.current) { + creatingSessionRef.current = createSession(appName, userId); + } + const pending = creatingSessionRef.current; + try { + const sid = await pending; + setSessionId(sid); + const now = Date.now() / 1000; + const optimistic: AdkSession = { id: sid, lastUpdateTime: now, events: [] }; + setSessions((prev) => [optimistic, ...prev.filter((s) => s.id !== sid)]); + return sid; + } finally { + if (creatingSessionRef.current === pending) creatingSessionRef.current = null; + } + } + async function addFiles(files: FileList | File[]) { - const picked: Attachment[] = []; - for (const f of Array.from(files)) { - if (f.size > MAX_FILE_BYTES) { - setError(`文件过大（>20MB）：${f.name}`); - continue; - } - const data = await fileToBase64(f); - picked.push({ - mimeType: f.type || "application/octet-stream", - data, - name: f.name, - }); + setError(""); + let sid: string; + try { + sid = await ensureSession(); + } catch (e) { + setError(String(e)); + return; } - if (picked.length) setAttachments((a) => [...a, ...picked]); + const drafts = Array.from(files).map((file) => ({ + file, + attachment: { + id: attachmentDraftId(), + mimeType: browserMimeType(file), + name: file.name, + sizeBytes: file.size, + status: "uploading" as const, + }, + })); + setAttachments((current) => [...current, ...drafts.map((draft) => draft.attachment)]); + await Promise.all( + drafts.map(async ({ file, attachment }) => { + try { + const uploaded = await uploadMedia(appName, userId, sid, file); + if (removedAttachmentIdsRef.current.delete(attachment.id)) { + if (uploaded.uri) await deleteMedia(appName, uploaded.uri); + return; + } + setAttachments((current) => current.map((item) => + item.id === attachment.id + ? uploaded + : item, + )); + } catch (e) { + if (removedAttachmentIdsRef.current.delete(attachment.id)) return; + const message = e instanceof Error ? e.message : String(e); + setAttachments((current) => current.map((item) => + item.id === attachment.id ? { ...item, status: "error", error: message } : item, + )); + setError(message); + } + }), + ); } - async function send(text: string, atts: Attachment[] = []) { + async function send( + text: string, + atts: Attachment[] = [], + selectedInvocation: FrontendInvocation = emptyInvocation(), + ) { // `busy` here = the CURRENT session is already streaming (can't double-send // to it). Other sessions can stream concurrently. if ((!text.trim() && atts.length === 0) || busy || !appName || !userId) return; setError(""); - // Lazily create the backend session on the first message. - let sid = sessionId; - if (!sid) { - try { - sid = await createSession(appName, userId); - setSessionId(sid); - // Show the session in the sidebar immediately (titled with this first - // message) instead of waiting for the reply to finish; the post-stream - // refreshSessions() below reconciles it with the server. - const now = Date.now() / 1000; - const optimistic: AdkSession = { - id: sid, - lastUpdateTime: now, - events: [{ author: "user", timestamp: now, content: { role: "user", parts: [{ text }] } }], - }; - setSessions((prev) => [optimistic, ...prev.filter((s) => s.id !== sid)]); - } catch (e) { - setError(String(e)); - return; - } + let sid: string; + try { + sid = await ensureSession(); + } catch (e) { + setError(String(e)); + return; } // Register this session's own stream (concurrent with other sessions'). @@ -909,10 +1078,20 @@ export default function App() { viewSidRef.current = sid; const userBlocks: Turn["blocks"] = []; + if (selectedInvocation.skills.length > 0 || selectedInvocation.targetAgent) { + userBlocks.push({ kind: "invocation", value: selectedInvocation }); + } if (atts.length) userBlocks.push({ kind: "attachment", - files: atts.map((a) => ({ mimeType: a.mimeType, data: a.data, name: a.name })), + files: atts.map((a) => ({ + id: a.id, + mimeType: a.mimeType, + data: a.data, + uri: a.uri, + name: a.name, + sizeBytes: a.sizeBytes, + })), }); if (text.trim()) userBlocks.push({ kind: "text", text }); setTurnsFor(sid, (t) => [ @@ -934,6 +1113,7 @@ export default function App() { sessionId: sid, text, attachments: atts, + invocation: selectedInvocation, signal: ctrl.signal, })) { if (ctrl.signal.aborted) break; @@ -1194,22 +1374,29 @@ export default function App() { const composer = ( { const text = input; const atts = attachments; + const selectedInvocation = invocation; setInput(""); setAttachments([]); - send(text, atts); + setInvocation(emptyInvocation()); + send(text, atts, selectedInvocation); + releaseAttachmentPreviews(atts); }} disabled={!appName || !userId} busy={busy} attachments={attachments} + skills={availableSkills} + agents={availableAgents} + invocation={invocation} + capabilitiesLoading={capabilitiesLoading} + onInvocationChange={setInvocation} onAddFiles={addFiles} - onRemoveAttachment={(i) => - setAttachments((a) => a.filter((_, j) => j !== i)) - } + onRemoveAttachment={removeDraftAttachment} /> ); return ( @@ -1391,6 +1578,7 @@ export default function App() { const atts = turn.blocks.flatMap((b) => b.kind === "attachment" ? b.files : [], ); + const turnInvocation = turn.blocks.find((b) => b.kind === "invocation"); return ( + {turnInvocation?.kind === "invocation" && ( + + )} {atts.length > 0 && ( -
- {atts.map((f, j) => - f.mimeType?.startsWith("image/") && f.data ? ( - {f.name - ) : ( -
- - {f.name ?? "文件"} -
- ), - )} -
+ )} {text && (
@@ -1443,7 +1618,7 @@ export default function App() { isLast && busy ? : null ) : ( <> - + {/* Finalized turn that produced no visible answer (e.g. only thinking + an empty A2UI surface) — show a fallback note. */} {!(isLast && busy) && !turnHasVisibleContent(turn) && ( diff --git a/frontend/src/adk/client.ts b/frontend/src/adk/client.ts index c20d0684..edd6aa18 100644 --- a/frontend/src/adk/client.ts +++ b/frontend/src/adk/client.ts @@ -40,6 +40,8 @@ export interface AdkEvent { endOfAgent?: boolean; end_of_agent?: boolean; escalate?: boolean; + artifactDelta?: Record; + artifact_delta?: Record; }; [k: string]: unknown; } @@ -71,11 +73,24 @@ export interface AdkInlineData { display_name?: string; } +export interface AdkFileData { + fileUri?: string; + mimeType?: string; + displayName?: string; + file_uri?: string; + mime_type?: string; + display_name?: string; +} + export interface AdkPart { text?: string; thought?: boolean; inlineData?: AdkInlineData; inline_data?: AdkInlineData; // snake_case fallback (defensive) + fileData?: AdkFileData; + file_data?: AdkFileData; + partMetadata?: Record; + part_metadata?: Record; functionCall?: { id?: string; name?: string; args?: Record }; functionResponse?: { id?: string; name?: string; response?: Record }; // snake_case fallbacks (defensive) @@ -83,11 +98,17 @@ export interface AdkPart { function_response?: { id?: string; name?: string; response?: Record }; } -/** A file the user attached in the composer, encoded for `/run_sse`. */ +/** A file attached in the composer or reconstructed from message history. */ export interface Attachment { + id: string; mimeType: string; - data: string; // base64 (no data: prefix) + uri?: string; + data?: string; // legacy inline base64 (no data: prefix) name?: string; + sizeBytes?: number; + status?: "uploading" | "ready" | "error"; + error?: string; + previewUrl?: string; } const API_BASE = ""; // same origin (prod) / proxied (dev) @@ -263,6 +284,94 @@ export async function deleteSession( if (!res.ok && res.status !== 404) throw new Error(`delete session failed: ${res.status}`); } +export interface MediaCapabilities { + maxFileBytes: number; + mimeTypes: string[]; + storage: "local" | "tos" | string; +} + +export async function getMediaCapabilities(appName: string): Promise { + void appName; + const res = await apiFetch("/web/media/capabilities"); + if (!res.ok) throw new Error(await httpErrorMessage(res, "media capabilities failed")); + return res.json(); +} + +export async function uploadMedia( + appName: string, + userId: string, + sessionId: string, + file: File, +): Promise { + const { app } = resolve(appName); + const body = new FormData(); + body.set("app_name", app); + body.set("user_id", userId); + body.set("session_id", sessionId); + body.set("file", file); + const res = await apiFetch("/web/media", { method: "POST", body }); + if (!res.ok) throw new Error(await httpErrorMessage(res, "文件上传失败")); + const media = (await res.json()) as { + id: string; + uri: string; + name: string; + mimeType: string; + sizeBytes: number; + }; + return { ...media, status: "ready" }; +} + +export async function deleteSessionMedia( + appName: string, + userId: string, + sessionId: string, +): Promise { + const { app } = resolve(appName); + const path = `/web/media/${encodeURIComponent(app)}/${encodeURIComponent(userId)}/${encodeURIComponent(sessionId)}`; + const res = await apiFetch(path, { method: "DELETE" }); + if (!res.ok && res.status !== 404) { + throw new Error(await httpErrorMessage(res, "media cleanup failed")); + } +} + +function mediaApiPath(uri: string): string | undefined { + try { + const parsed = new URL(uri); + if (parsed.protocol !== "veadk-media:" || parsed.hostname !== "apps") return undefined; + const segments = parsed.pathname.split("/").filter(Boolean).map(decodeURIComponent); + if ( + segments.length !== 7 || + segments[1] !== "users" || + segments[3] !== "sessions" || + segments[5] !== "media" + ) return undefined; + return `/web/media/${segments.map(encodeURIComponent).filter((_, i) => ![1, 3, 5].includes(i)).join("/")}`; + } catch { + return undefined; + } +} + +/** Delete one uploaded media object that has not been sent in a message. */ +export async function deleteMedia(appName: string, uri: string): Promise { + const path = mediaApiPath(uri); + if (!path) throw new Error("Invalid VeADK media URI"); + void appName; + const res = await apiFetch(path, { method: "DELETE" }); + if (!res.ok && res.status !== 404) { + throw new Error(await httpErrorMessage(res, "media cleanup failed")); + } +} + +/** Resolve a stable media URI to an authenticated same-origin delivery URL. */ +export function mediaContentUrl(appName: string, uri: string): string { + if (uri.startsWith("data:") || uri.startsWith("blob:") || /^https?:/.test(uri)) return uri; + const basePath = mediaApiPath(uri); + if (!basePath) return uri; + const path = `${basePath}/content`; + void appName; + return withAuth(`${API_BASE}${path}`); +} + export async function getSessionTrace( appName: string, sessionId: string, @@ -295,9 +404,29 @@ export interface AgentNode { type: AgentNodeType; model: string; tools: string[]; + skills: AgentSkill[]; + path: string[]; + mentionable: boolean; children: AgentNode[]; } +export interface AgentSkill { + name: string; + description: string; +} + +export interface AgentTarget { + name: string; + description: string; + type: AgentNodeType; + path: string[]; +} + +export interface FrontendInvocation { + skills: AgentSkill[]; + targetAgent?: AgentTarget; +} + /** Introspected metadata for an agent app (model, tools), for the picker. * Only the local server implements `/web/agent-info`; remote AgentKit apps * will reject this and the caller falls back to a basic flyout. */ @@ -306,6 +435,7 @@ export interface AgentInfo { description: string; model: string; tools: string[]; + skills: AgentSkill[]; subAgents: string[]; /** Recursive typed tree; only the local server provides it. */ graph?: AgentNode; @@ -347,6 +477,7 @@ export interface RunArgs { sessionId: string; text: string; attachments?: Attachment[]; + invocation?: FrontendInvocation; /** Function responses to send instead of/alongside text — used to resume a * long-running call (e.g. answering ADK's `adk_request_credential`). */ functionResponses?: { id: string; name: string; response: unknown }[]; @@ -361,19 +492,53 @@ export async function* runSSE({ sessionId, text, attachments = [], + invocation, functionResponses = [], signal, }: RunArgs): AsyncGenerator { const { app, ep } = resolve(appName); + const attachmentParts = attachments.flatMap>((a) => { + if (a.status && a.status !== "ready") return []; + if (a.uri) { + return [{ + fileData: { mimeType: a.mimeType, fileUri: a.uri, displayName: a.name }, + partMetadata: { + veadkMedia: { + id: a.id, + uri: a.uri, + name: a.name, + mimeType: a.mimeType, + sizeBytes: a.sizeBytes, + }, + }, + }]; + } + return a.data ? [{ + inlineData: { mimeType: a.mimeType, data: a.data, displayName: a.name }, + }] : []; + }); + const invocationMetadata = invocation && + (invocation.skills.length > 0 || invocation.targetAgent) + ? invocation + : undefined; const parts: Record[] = [ - ...attachments.map((a) => ({ - inlineData: { mimeType: a.mimeType, data: a.data, displayName: a.name }, - })), + ...attachmentParts, ...functionResponses.map((fr) => ({ functionResponse: { id: fr.id, name: fr.name, response: fr.response }, })), ...(text.trim() ? [{ text }] : []), ]; + if (invocationMetadata && parts.length > 0) { + const firstPart = parts[0]; + const partMetadata = firstPart.partMetadata as Record | undefined; + parts[0] = { + ...firstPart, + partMetadata: { + ...partMetadata, + veadkInvocation: invocationMetadata, + }, + }; + } const res = await apiFetch( `/run_sse`, { @@ -385,6 +550,9 @@ export async function* runSSE({ session_id: sessionId, new_message: { role: "user", parts }, streaming: true, + custom_metadata: invocationMetadata + ? { veadkInvocation: invocationMetadata } + : undefined, }), signal, }, diff --git a/frontend/src/blocks.ts b/frontend/src/blocks.ts index 741f106f..82e0557b 100644 --- a/frontend/src/blocks.ts +++ b/frontend/src/blocks.ts @@ -8,7 +8,14 @@ // arrives we discard that preview and append the authoritative content. Stored // history is all consolidated (partial falsey), which this same logic handles. -import type { AdkEvent, AdkPart } from "./adk/client"; +import type { + AdkEvent, + AdkPart, + AgentNodeType, + AgentSkill, + AgentTarget, + FrontendInvocation, +} from "./adk/client"; import type { A2uiMessage } from "./a2ui/types"; const A2UI_TOOL = "send_a2ui_json_to_client"; @@ -29,9 +36,12 @@ export function authUriOf(authConfig: unknown): string | undefined { } export interface AttachmentView { + id: string; mimeType?: string; data?: string; // base64 (no data: prefix) + uri?: string; name?: string; + sizeBytes?: number; } export type Block = @@ -40,6 +50,7 @@ export type Block = | { kind: "tool"; name: string; args?: unknown; response?: unknown; done: boolean } | { kind: "a2ui"; messages: A2uiMessage[] } | { kind: "attachment"; files: AttachmentView[] } + | { kind: "invocation"; value: FrontendInvocation } | { kind: "auth"; callId: string; @@ -85,19 +96,113 @@ function toStdBase64(b64: string): string { /** Pull file attachments (inline_data) out of a message's parts. */ export function attachmentsFromParts(parts: AdkPart[]): AttachmentView[] { const files: AttachmentView[] = []; - for (const p of parts) { + for (const [index, p] of parts.entries()) { + const metadata = (p.partMetadata ?? p.part_metadata) as + | Record + | undefined; + const transport = metadata?.veadkTransport as Record | undefined; + if (transport?.hidden === true) continue; + const stored = metadata?.veadkMedia as Record | undefined; + if (typeof stored?.uri === "string") { + files.push({ + id: String(stored.id ?? stored.uri), + mimeType: typeof stored.mimeType === "string" ? stored.mimeType : undefined, + uri: stored.uri, + name: typeof stored.name === "string" ? stored.name : undefined, + sizeBytes: typeof stored.sizeBytes === "number" ? stored.sizeBytes : undefined, + }); + continue; + } const d = p.inlineData ?? p.inline_data; if (d && d.data) { files.push({ + id: `inline-${index}-${d.displayName ?? d.display_name ?? "media"}`, mimeType: d.mimeType ?? d.mime_type, data: toStdBase64(d.data), name: d.displayName ?? d.display_name, }); + continue; + } + const f = p.fileData ?? p.file_data; + const uri = f?.fileUri ?? f?.file_uri; + if (f && uri) { + files.push({ + id: uri, + mimeType: f.mimeType ?? f.mime_type, + uri, + name: f.displayName ?? f.display_name, + }); } } return files; } +function visiblePartText(part: AdkPart): string | undefined { + const metadata = (part.partMetadata ?? part.part_metadata) as + | Record + | undefined; + const transport = metadata?.veadkTransport as Record | undefined; + return transport?.hideText === true ? undefined : part.text; +} + +const AGENT_NODE_TYPES = new Set([ + "llm", + "sequential", + "parallel", + "loop", + "a2a", +]); + +/** Restore slash-skill and @agent selections persisted in part metadata. */ +export function invocationFromParts(parts: AdkPart[]): FrontendInvocation | undefined { + for (const part of parts) { + const raw = (part.partMetadata ?? part.part_metadata)?.veadkInvocation; + if (!raw || typeof raw !== "object") continue; + const metadata = raw as Record; + const skills = Array.isArray(metadata.skills) + ? metadata.skills.flatMap((item) => { + if (!item || typeof item !== "object") return []; + const skill = item as Record; + return typeof skill.name === "string" + ? [{ + name: skill.name, + description: typeof skill.description === "string" ? skill.description : "", + }] + : []; + }) + : []; + + let targetAgent: AgentTarget | undefined; + const rawTarget = metadata.targetAgent; + if (rawTarget && typeof rawTarget === "object") { + const target = rawTarget as Record; + const type = target.type; + if ( + typeof target.name === "string" && + typeof type === "string" && + AGENT_NODE_TYPES.has(type as AgentNodeType) && + Array.isArray(target.path) + ) { + targetAgent = { + name: target.name, + description: typeof target.description === "string" ? target.description : "", + type: type as AgentNodeType, + path: target.path.filter((item): item is string => typeof item === "string"), + }; + } + } + if (skills.length > 0 || targetAgent) return { skills, targetAgent }; + } + return undefined; +} + +function appendAttachments(blocks: Block[], files: AttachmentView[]) { + if (!files.length) return; + const last = blocks[blocks.length - 1]; + if (last?.kind === "attachment") last.files.push(...files); + else blocks.push({ kind: "attachment", files }); +} + function appendText(blocks: Block[], kind: "thinking" | "text", text: string) { const last = blocks[blocks.length - 1]; if (last && last.kind === kind) last.text += text; @@ -118,8 +223,9 @@ export function applyEvent(acc: Acc, ev: AdkEvent): Acc { if (ev.partial && !hasFn) { // Streaming delta: append into the live-preview region. for (const p of parts) { - if (typeof p.text === "string" && p.text) - appendText(blocks, p.thought ? "thinking" : "text", p.text); + const text = visiblePartText(p); + if (typeof text === "string" && text) + appendText(blocks, p.thought ? "thinking" : "text", text); } return { blocks, liveStart }; } @@ -130,8 +236,13 @@ export function applyEvent(acc: Acc, ev: AdkEvent): Acc { for (const p of parts) { const fc = fnCall(p); const fr = fnResp(p); - if (typeof p.text === "string" && p.text) { - appendText(blocks, p.thought ? "thinking" : "text", p.text); + const files = attachmentsFromParts([p]); + const text = visiblePartText(p); + if (typeof text === "string" && text) { + appendText(blocks, p.thought ? "thinking" : "text", text); + } else if (files.length) { + closeThinking(blocks); + appendAttachments(blocks, files); } else if (fc) { closeThinking(blocks); if (fc.name === REQUEST_EUC) { @@ -212,16 +323,18 @@ export function eventsToTurns(events: AdkEvent[]): Turn[] { } } const text = parts - .map((p) => p.text) + .map(visiblePartText) .filter((t): t is string => !!t) .join(""); const files = attachmentsFromParts(parts); + const invocation = invocationFromParts(parts); // Skip pure function-response turns (no text/files) — they're internal. - if (!text && !files.length) { + if (!text && !files.length && !invocation) { acc = emptyAcc(); continue; } const blocks: Block[] = []; + if (invocation) blocks.push({ kind: "invocation", value: invocation }); if (files.length) blocks.push({ kind: "attachment", files }); if (text) blocks.push({ kind: "text", text }); turns.push({ role: "user", blocks, meta: { ts: ev.timestamp } }); diff --git a/frontend/src/main.tsx b/frontend/src/main.tsx index 6e519d21..3891f848 100644 --- a/frontend/src/main.tsx +++ b/frontend/src/main.tsx @@ -1,7 +1,9 @@ import React from "react"; import ReactDOM from "react-dom/client"; import { MotionConfig } from "motion/react"; +import { PhotoProvider } from "react-photo-view"; import App from "./App"; +import "react-photo-view/dist/react-photo-view.css"; import "./styles.css"; // OAuth popup callback landing. When an OAuth authorize flow (see runOAuthPopup @@ -31,7 +33,9 @@ ReactDOM.createRoot(document.getElementById("root")!).render( {/* reducedMotion="user" makes all motion components honor the OS prefers-reduced-motion setting (transforms/opacity are stilled). */} - + + + , ); diff --git a/frontend/src/styles.css b/frontend/src/styles.css index 7f697cbf..e8617a03 100644 --- a/frontend/src/styles.css +++ b/frontend/src/styles.css @@ -168,8 +168,6 @@ body { left: 100%; top: 8px; z-index: 32; - height: auto; /* size to content — no full-height empty space below */ - max-height: calc(100vh - 16px); display: flex; flex-direction: row; align-items: stretch; @@ -197,7 +195,14 @@ body { border-left: 1px solid hsl(var(--border)); background: hsl(var(--canvas) / 0.4); } -.agentsel-detail-body { flex: 1; min-height: 0; overflow-y: auto; padding: 12px 14px; } +.agentsel-detail-body { + flex: 1; + min-height: 0; + overflow-y: auto; + overscroll-behavior-y: contain; + scrollbar-gutter: stable; + padding: 12px 14px; +} .agentsel-detail-id { font-size: 11.5px; color: hsl(var(--muted-foreground)); word-break: break-all; margin-bottom: 12px; } .agentsel-kv { margin: 0; display: flex; flex-direction: column; gap: 8px; } .agentsel-kv-row { display: grid; grid-template-columns: 52px 1fr; gap: 8px; font-size: 12.5px; } @@ -243,7 +248,14 @@ body { } .agentsel-refresh:hover { background: hsl(var(--accent)); color: hsl(var(--foreground)); } .agentsel-refresh .icon { width: 16px; height: 16px; } -.agentsel-body { flex: 1; min-height: 0; overflow-y: auto; padding: 10px; } +.agentsel-body { + flex: 1; + min-height: 0; + overflow-y: auto; + overscroll-behavior-y: contain; + scrollbar-gutter: stable; + padding: 10px; +} .agentsel-tools { display: flex; flex-direction: column; gap: 8px; margin-bottom: 10px; } .agentsel-search { display: flex; align-items: center; gap: 8px; @@ -591,7 +603,53 @@ body { text-align: left; } .md th { background: hsl(var(--muted)); font-weight: 600; } -.md img { max-width: 100%; border-radius: 8px; } +.md .image-preview-trigger { + position: relative; + display: block; + width: fit-content; + max-width: 40%; + margin: 0 0 0.7em; + padding: 0; + overflow: hidden; + border: 0; + border-radius: 10px; + background: hsl(var(--muted)); + box-shadow: 0 0 0 1px hsl(var(--border)); + cursor: zoom-in; + line-height: 0; +} +.md .image-preview-trigger:focus-visible { + outline: 2px solid hsl(var(--ring)); + outline-offset: 3px; +} +.md .image-preview-trigger img { + display: block; + width: auto; + max-width: 100%; + height: auto; + border-radius: inherit; + transition: filter 0.18s ease, transform 0.18s ease; +} +.md .image-preview-trigger:hover img { filter: brightness(0.92); transform: scale(1.01); } +.image-preview-hint { + position: absolute; + right: 8px; + bottom: 8px; + display: grid; + width: 28px; + height: 28px; + place-items: center; + border: 1px solid hsl(0 0% 100% / 0.2); + border-radius: 8px; + background: hsl(240 6% 8% / 0.68); + color: white; + opacity: 0; + transform: translateY(3px); + transition: opacity 0.16s ease, transform 0.16s ease; +} +.image-preview-hint svg { width: 14px; height: 14px; } +.image-preview-trigger:hover .image-preview-hint, +.image-preview-trigger:focus-visible .image-preview-hint { opacity: 1; transform: translateY(0); } /* user bubble: keep markdown compact and on-tone inside the grey pill */ .turn--user .md code { background: hsl(var(--background) / 0.55); } .turn--user .md pre { background: hsl(var(--background) / 0.55); } @@ -984,6 +1042,7 @@ body { padding: 6px 16px 18px; } .composer-box { + position: relative; display: flex; align-items: flex-end; gap: 6px; @@ -1055,6 +1114,178 @@ body { .comp-send:active:not(:disabled) { transform: scale(0.94); } .comp-send:disabled { opacity: 0.3; cursor: default; } +/* ---------- explicit /skill and @agent invocation context ---------- */ +.invocation-chips { + display: flex; + flex-wrap: wrap; + gap: 6px; + min-width: 0; +} +.composer > .invocation-chips { padding: 0 8px 8px; } +.turn--user > .invocation-chips { + justify-content: flex-end; + margin-bottom: 6px; +} +.invocation-chip { + display: inline-flex; + align-items: center; + gap: 5px; + max-width: 260px; + min-height: 28px; + padding: 4px 8px; + border: 1px solid hsl(var(--border)); + border-radius: 8px; + background: hsl(var(--background)); + color: hsl(var(--foreground)); + box-shadow: 0 1px 2px hsl(var(--foreground) / 0.025); + font-size: 12px; + font-weight: 560; + line-height: 1.2; +} +.invocation-chip--skill { color: hsl(145 52% 31%); } +.invocation-chip--agent { color: hsl(214 64% 42%); } +.invocation-chip > svg { width: 13px; height: 13px; flex: 0 0 auto; } +.invocation-chip > span { + overflow: hidden; + text-overflow: ellipsis; + white-space: nowrap; +} +.invocation-chip button { + display: inline-flex; + align-items: center; + justify-content: center; + width: 17px; + height: 17px; + margin: -1px -3px -1px 1px; + padding: 0; + border: none; + border-radius: 5px; + background: transparent; + color: currentColor; + cursor: pointer; + opacity: 0.55; +} +.invocation-chip button:hover { background: hsl(var(--accent)); opacity: 1; } +.invocation-chip button svg { width: 11px; height: 11px; } + +.composer-command-menu { + position: absolute; + z-index: 34; + bottom: calc(100% + 10px); + left: 0; + width: min(500px, calc(100vw - 48px)); + overflow: hidden; + border: 1px solid hsl(var(--border)); + border-radius: 14px; + background: hsl(var(--background)); + box-shadow: + 0 2px 7px hsl(var(--foreground) / 0.08), + 0 22px 60px -24px hsl(var(--foreground) / 0.28); + transform-origin: bottom left; + animation: command-menu-in 0.13s ease-out; +} +@keyframes command-menu-in { + from { opacity: 0; transform: translateY(5px) scale(0.985); } + to { opacity: 1; transform: translateY(0) scale(1); } +} +.composer-command-head { + display: flex; + align-items: center; + gap: 7px; + height: 38px; + padding: 0 10px 0 12px; + border-bottom: 1px solid hsl(var(--border)); + color: hsl(var(--muted-foreground)); + font-size: 11px; + font-weight: 650; + letter-spacing: 0.02em; +} +.composer-command-head > svg { width: 13px; height: 13px; } +.composer-command-head > span { flex: 1; } +.composer-command-menu kbd { + min-width: 22px; + padding: 2px 5px; + border: 1px solid hsl(var(--border)); + border-radius: 5px; + background: hsl(var(--canvas)); + color: hsl(var(--muted-foreground)); + font: 10px/1.2 ui-monospace, SFMono-Regular, Menlo, monospace; + text-align: center; +} +.composer-command-list { + display: grid; + max-height: min(330px, 42vh); + overflow-y: auto; + padding: 5px; +} +.composer-command-item { + display: grid; + grid-template-columns: 34px minmax(0, 1fr) auto; + align-items: center; + gap: 9px; + width: 100%; + min-height: 52px; + padding: 6px 8px; + border: none; + border-radius: 9px; + background: transparent; + color: hsl(var(--foreground)); + font: inherit; + text-align: left; + cursor: pointer; +} +.composer-command-item.is-active { background: hsl(var(--accent)); } +.composer-command-icon { + display: inline-flex; + align-items: center; + justify-content: center; + width: 34px; + height: 34px; + border-radius: 9px; +} +.composer-command-icon--skill { + background: hsl(145 55% 94%); + color: hsl(145 60% 32%); +} +.composer-command-icon--agent { + background: hsl(214 75% 95%); + color: hsl(214 65% 43%); +} +.composer-command-icon svg { width: 16px; height: 16px; } +.composer-command-copy { + display: grid; + min-width: 0; + gap: 3px; +} +.composer-command-copy strong { + overflow: hidden; + color: hsl(var(--foreground)); + font-size: 13px; + font-weight: 620; + line-height: 1.2; + text-overflow: ellipsis; + white-space: nowrap; +} +.composer-command-copy > span { + overflow: hidden; + color: hsl(var(--muted-foreground)); + font-size: 11px; + line-height: 1.3; + text-overflow: ellipsis; + white-space: nowrap; +} +.composer-command-empty { + display: flex; + align-items: center; + justify-content: center; + gap: 7px; + min-height: 68px; + padding: 14px; + color: hsl(var(--muted-foreground)); + font-size: 12px; +} +.composer-command-empty svg { width: 14px; height: 14px; } + /* ---------- composer "+" upload menu ---------- */ .composer-menu-wrap { position: relative; flex-shrink: 0; } .composer-menu { @@ -1071,69 +1302,209 @@ body { box-shadow: 0 6px 20px hsl(var(--foreground) / 0.12); } -/* ---------- pending attachments (composer) ---------- */ -.attachment-row { +/* ---------- multimodal media (compact cards + focused viewer) ---------- */ +.media-grid { display: flex; flex-wrap: wrap; gap: 8px; - padding: 0 8px 8px; + max-width: min(620px, 100%); } -.attachment-thumb-wrap { position: relative; } -.attachment-thumb { - width: 56px; - height: 56px; - object-fit: cover; - border-radius: 12px; - border: 1px solid hsl(var(--border)); - display: block; +.turn--user .media-grid { justify-content: flex-end; } +.composer > .media-grid { + padding: 0 8px 9px; + justify-content: flex-start; } -.attachment-file { +.media-card { position: relative; + width: 272px; + min-width: 0; + overflow: visible; + border: 1px solid hsl(var(--border)); + border-radius: 14px; + background: hsl(var(--background)); + box-shadow: 0 1px 2px hsl(var(--foreground) / 0.025); + transition: border-color 0.16s, box-shadow 0.16s, transform 0.16s; +} +.media-card:hover { + border-color: hsl(var(--foreground) / 0.2); + box-shadow: 0 8px 28px -20px hsl(var(--foreground) / 0.28); + transform: translateY(-1px); +} +.media-card--image { width: 176px; } +.media-grid--compact .media-card { width: 224px; } +.media-grid--compact .media-card--image { width: 92px; } +.media-card-main { display: flex; align-items: center; - gap: 6px; - max-width: 200px; - padding: 8px 10px; - border: 1px solid hsl(var(--border)); - border-radius: 12px; - background: hsl(var(--secondary)); - color: hsl(var(--foreground)); - font-size: 13px; + gap: 11px; + width: 100%; + min-width: 0; + height: 68px; + padding: 10px 12px; + border: none; + border-radius: inherit; + background: transparent; + color: inherit; + font: inherit; + text-align: left; + cursor: pointer; +} +.media-card-main:disabled { cursor: default; } +.media-card--image .media-card-main { + display: block; + height: 132px; + padding: 4px; +} +.media-grid--compact .media-card-main { height: 58px; padding: 8px 10px; } +.media-grid--compact .media-card--image .media-card-main { height: 72px; padding: 3px; } +.media-card-image { + width: 100%; + height: 100%; + object-fit: cover; + border-radius: 10px; + display: block; + background: hsl(var(--muted)); } -.attachment-file .icon { width: 16px; height: 16px; flex-shrink: 0; } -.attachment-file-name { +.media-card--image .media-card-copy, +.media-card--image .media-card-open { display: none; } +.media-card-icon { + display: inline-flex; + align-items: center; + justify-content: center; + width: 40px; + height: 44px; + flex: 0 0 auto; + border-radius: 9px; + color: hsl(var(--muted-foreground)); + background: hsl(var(--muted)); +} +.media-card--pdf .media-card-icon { color: hsl(2 72% 50%); background: hsl(2 80% 96%); } +.media-card--video .media-card-icon { color: hsl(215 72% 48%); background: hsl(215 80% 96%); } +.media-card--markdown .media-card-icon { color: hsl(145 60% 36%); background: hsl(145 60% 95%); } +.media-card-icon svg { width: 21px; height: 21px; } +.media-card-copy { min-width: 0; flex: 1; display: grid; gap: 5px; } +.media-card-name { overflow: hidden; text-overflow: ellipsis; white-space: nowrap; + font-size: 13px; + line-height: 1.2; + font-weight: 560; } -.attachment-remove { - position: absolute; - top: -6px; - right: -6px; +.media-card-meta { display: flex; align-items: center; + gap: 5px; + min-width: 0; + color: hsl(var(--muted-foreground)); + font-size: 11px; + line-height: 1.2; +} +.media-card-type { + font-size: 9px; + font-weight: 700; + letter-spacing: 0.06em; +} +.media-card-open { width: 14px; height: 14px; flex: 0 0 auto; color: hsl(var(--muted-foreground)); opacity: 0; transition: opacity 0.15s; } +.media-card:hover .media-card-open { opacity: 1; } +.media-card-spinner { width: 12px; height: 12px; animation: spin 0.85s linear infinite; } +.media-card--error { border-color: hsl(var(--destructive) / 0.42); } +.media-card--error .media-card-meta { color: hsl(var(--destructive)); } +.media-card-remove { + position: absolute; + z-index: 2; + top: -7px; + right: -7px; + display: inline-flex; + align-items: center; justify-content: center; - width: 18px; - height: 18px; + width: 21px; + height: 21px; padding: 0; border: 1px solid hsl(var(--border)); - border-radius: 50%; + border-radius: 999px; background: hsl(var(--background)); color: hsl(var(--muted-foreground)); + box-shadow: 0 2px 8px hsl(var(--foreground) / 0.12); cursor: pointer; } -.attachment-remove:hover { color: hsl(var(--foreground)); } -.attachment-remove .icon { width: 11px; height: 11px; } +.media-card-remove:hover { color: hsl(var(--foreground)); } +.media-card-remove svg { width: 12px; height: 12px; } -/* ---------- attachments on a sent user turn ---------- */ -.msg-attachments { +.media-viewer-backdrop { + position: fixed; + inset: 0; + z-index: 90; + display: grid; + place-items: center; + padding: 28px; + background: hsl(240 6% 8% / 0.72); + backdrop-filter: blur(12px) saturate(0.8); +} +.media-viewer { display: flex; - flex-wrap: wrap; - justify-content: flex-end; - gap: 8px; - margin-bottom: 6px; + flex-direction: column; + width: min(1080px, 94vw); + height: min(820px, 90vh); + overflow: hidden; + border: 1px solid hsl(var(--foreground) / 0.13); + border-radius: 18px; + background: hsl(var(--background)); + box-shadow: 0 32px 100px hsl(240 10% 3% / 0.46); +} +.media-viewer-header { + display: flex; + align-items: center; + justify-content: space-between; + gap: 18px; + min-height: 58px; + padding: 9px 12px 9px 18px; + border-bottom: 1px solid hsl(var(--border)); + background: hsl(var(--background) / 0.94); +} +.media-viewer-header > div { min-width: 0; display: grid; gap: 2px; } +.media-viewer-header strong { overflow: hidden; text-overflow: ellipsis; white-space: nowrap; font-size: 13px; font-weight: 620; } +.media-viewer-header span { color: hsl(var(--muted-foreground)); font-size: 11px; } +.media-viewer-header nav { display: flex; gap: 4px; } +.media-viewer-header a, +.media-viewer-header button { + display: inline-flex; + align-items: center; + justify-content: center; + width: 36px; + height: 36px; + padding: 0; + border: none; + border-radius: 9px; + background: transparent; + color: hsl(var(--muted-foreground)); + cursor: pointer; +} +.media-viewer-header a:hover, +.media-viewer-header button:hover { background: hsl(var(--accent)); color: hsl(var(--foreground)); } +.media-viewer-header svg { width: 17px; height: 17px; } +.media-viewer-body { flex: 1; min-height: 0; overflow: auto; background: hsl(var(--canvas)); } +.media-viewer-body--image, +.media-viewer-body--video { display: grid; place-items: center; padding: 24px; background: hsl(240 6% 9%); } +.media-viewer-body--image img, +.media-viewer-body--video video { max-width: 100%; max-height: 100%; object-fit: contain; border-radius: 8px; } +.media-viewer-body--pdf iframe { display: block; width: 100%; height: 100%; border: none; background: white; } +.media-document { width: min(820px, calc(100% - 48px)); margin: 24px auto; padding: 34px 40px; border: 1px solid hsl(var(--border)); border-radius: 12px; background: hsl(var(--background)); box-shadow: 0 12px 38px -30px hsl(var(--foreground) / 0.3); } +.media-document--plain { min-height: calc(100% - 48px); white-space: pre-wrap; word-break: break-word; font: 13px/1.65 ui-monospace, SFMono-Regular, Menlo, monospace; } +.media-viewer-loading { display: flex; align-items: center; justify-content: center; gap: 8px; height: 100%; color: hsl(var(--muted-foreground)); font-size: 13px; } +.media-viewer-loading svg { width: 17px; height: 17px; animation: spin 0.85s linear infinite; } + +@media (max-width: 640px) { + .composer-command-menu { + right: 0; + width: auto; + } + .media-card { width: min(272px, 82vw); } + .media-viewer-backdrop { padding: 0; } + .media-viewer { width: 100vw; height: 100vh; border: none; border-radius: 0; } + .media-document { width: calc(100% - 24px); margin: 12px auto; padding: 22px 18px; } + .md .image-preview-trigger { max-width: 100%; } } -.msg-attachments .attachment-thumb { width: 120px; height: 120px; } /* ---------- A2UI rendered surfaces ---------- */ .a2ui-surface { max-width: 360px; width: 100%; font-size: 14px; } @@ -1780,7 +2151,7 @@ body { z-index: 31; min-width: 240px; max-width: 360px; - max-height: 70vh; + max-height: min(560px, calc(100dvh - 84px)); padding: 6px; background: hsl(var(--panel)); border: 1px solid hsl(var(--border)); @@ -1790,6 +2161,8 @@ body { flex-direction: column; gap: 2px; overflow-y: auto; + overscroll-behavior-y: contain; + scrollbar-gutter: stable; animation: ddpop 0.14s ease; } @keyframes ddpop { from { opacity: 0; transform: translateY(-4px); } to { opacity: 1; transform: translateY(0); } } diff --git a/frontend/src/ui/AgentSelector.tsx b/frontend/src/ui/AgentSelector.tsx index 70a005ea..f8fa1f93 100644 --- a/frontend/src/ui/AgentSelector.tsx +++ b/frontend/src/ui/AgentSelector.tsx @@ -266,7 +266,10 @@ export function AgentSelector({ className="agentsel" role="dialog" aria-label="选择 Agent" - style={{ top: anchorTop }} + style={{ + top: anchorTop, + height: `min(640px, calc(100dvh - ${anchorTop}px - 10px))`, + }} >
diff --git a/frontend/src/ui/Blocks.tsx b/frontend/src/ui/Blocks.tsx index 27c39f12..bb1fb2e8 100644 --- a/frontend/src/ui/Blocks.tsx +++ b/frontend/src/ui/Blocks.tsx @@ -5,6 +5,8 @@ import type { Block } from "../blocks"; import { buildSurfaces, SurfaceView } from "../a2ui/Surface"; import { useStickToBottom } from "./useStickToBottom"; import { Markdown } from "./Markdown"; +import { InvocationChips } from "./InvocationChips"; +import { MediaGroup } from "./Media"; import type { A2uiAction, A2uiComponent } from "../a2ui/types"; const A2UI_TOOL = "send_a2ui_json_to_client"; @@ -218,12 +220,13 @@ function AuthCard({ export interface BlocksProps { blocks: Block[]; + appName?: string; onAction: (action: A2uiAction | undefined, node: A2uiComponent) => void; /** Handle an MCP/tool OAuth request (opens auth URL, resumes the run). */ onAuth?: (block: AuthBlock) => Promise; } -export function Blocks({ blocks, onAction, onAuth }: BlocksProps) { +export function Blocks({ blocks, appName = "", onAction, onAuth }: BlocksProps) { return ( <> {blocks.map((b, i) => { @@ -238,6 +241,10 @@ export function Blocks({ blocks, onAction, onAuth }: BlocksProps) {
) : null; } + case "attachment": + return ; + case "invocation": + return ; case "tool": if (b.name === A2UI_TOOL && b.done) return null; return ( diff --git a/frontend/src/ui/Composer.tsx b/frontend/src/ui/Composer.tsx index 4ab85a19..443c7ca6 100644 --- a/frontend/src/ui/Composer.tsx +++ b/frontend/src/ui/Composer.tsx @@ -1,35 +1,78 @@ import { useLayoutEffect, useRef, useState } from "react"; -import { ArrowUp, FileText, ImageIcon, Loader2, Plus, X } from "lucide-react"; +import { + ArrowUp, + AtSign, + Bot, + FileText, + FileVideo2, + ImageIcon, + Loader2, + Plus, + Sparkles, +} from "lucide-react"; import { motion } from "motion/react"; -import type { Attachment } from "../adk/client"; +import type { + AgentSkill, + AgentTarget, + Attachment, + FrontendInvocation, +} from "../adk/client"; +import { InvocationChips } from "./InvocationChips"; +import { MediaGroup } from "./Media"; + +interface CompletionTrigger { + kind: "skill" | "agent"; + query: string; + start: number; + end: number; +} + +type CompletionItem = + | { kind: "skill"; value: AgentSkill } + | { kind: "agent"; value: AgentTarget }; export interface ComposerProps { sessionId: string; + appName: string; value: string; onChange: (v: string) => void; onSubmit: () => void; disabled: boolean; // not connected yet busy: boolean; // a turn is streaming attachments: Attachment[]; + skills: AgentSkill[]; + agents: AgentTarget[]; + invocation: FrontendInvocation; + capabilitiesLoading?: boolean; + onInvocationChange: (value: FrontendInvocation) => void; onAddFiles: (files: FileList | File[]) => void; - onRemoveAttachment: (index: number) => void; + onRemoveAttachment: (id: string) => void; } export function Composer({ sessionId, + appName, value, onChange, onSubmit, disabled, busy, attachments, + skills, + agents, + invocation, + capabilitiesLoading = false, + onInvocationChange, onAddFiles, onRemoveAttachment, }: ComposerProps) { const ref = useRef(null); const imageInput = useRef(null); - const fileInput = useRef(null); + const documentInput = useRef(null); + const videoInput = useRef(null); const [menuOpen, setMenuOpen] = useState(false); + const [trigger, setTrigger] = useState(null); + const [activeIndex, setActiveIndex] = useState(0); // Auto-grow the textarea up to a max height, then scroll. useLayoutEffect(() => { @@ -39,36 +82,150 @@ export function Composer({ el.style.height = `${Math.min(el.scrollHeight, 200)}px`; }, [value]); - const canSend = - !disabled && !busy && (value.trim().length > 0 || attachments.length > 0); + const uploadPending = attachments.some((attachment) => attachment.status !== "ready"); + const canSend = !disabled && !busy && !uploadPending && + (value.trim().length > 0 || attachments.length > 0); - function pick(input: React.RefObject) { + const query = trigger?.query.toLocaleLowerCase() ?? ""; + const suggestions: CompletionItem[] = trigger?.kind === "skill" + ? skills + .filter((skill) => !invocation.skills.some((selected) => selected.name === skill.name)) + .filter((skill) => `${skill.name} ${skill.description}`.toLocaleLowerCase().includes(query)) + .map((value) => ({ kind: "skill" as const, value })) + : trigger?.kind === "agent" + ? agents + .filter((agent) => `${agent.name} ${agent.description}`.toLocaleLowerCase().includes(query)) + .map((value) => ({ kind: "agent" as const, value })) + : []; + + function pick(input: React.RefObject) { setMenuOpen(false); + setTrigger(null); input.current?.click(); } + function updateCompletion(nextValue: string, cursor: number) { + const prefix = nextValue.slice(0, cursor); + const match = /(^|\s)([/@])([^\s/@]*)$/.exec(prefix); + if (!match) { + setTrigger(null); + return; + } + const tokenLength = match[2].length + match[3].length; + const nextTrigger: CompletionTrigger = { + kind: match[2] === "/" ? "skill" : "agent", + query: match[3], + start: cursor - tokenLength, + end: cursor, + }; + const completionChanged = !trigger || + trigger.kind !== nextTrigger.kind || + trigger.query !== nextTrigger.query || + trigger.start !== nextTrigger.start || + trigger.end !== nextTrigger.end; + setTrigger(nextTrigger); + if (completionChanged) setActiveIndex(0); + setMenuOpen(false); + } + + function choose(item: CompletionItem) { + if (!trigger) return; + const nextValue = value.slice(0, trigger.start) + value.slice(trigger.end); + onChange(nextValue); + if (item.kind === "skill") { + onInvocationChange({ + ...invocation, + skills: [...invocation.skills, item.value], + }); + } else { + onInvocationChange({ skills: [], targetAgent: item.value }); + } + const cursor = trigger.start; + setTrigger(null); + requestAnimationFrame(() => { + ref.current?.focus(); + ref.current?.setSelectionRange(cursor, cursor); + }); + } + + function removeLastInvocation() { + if (invocation.targetAgent) { + onInvocationChange({ skills: [] }); + return; + } + if (invocation.skills.length > 0) { + onInvocationChange({ ...invocation, skills: invocation.skills.slice(0, -1) }); + } + } + function onInputChange(e: React.ChangeEvent) { - if (e.target.files && e.target.files.length) onAddFiles(e.target.files); + const selected = e.target.files ? Array.from(e.target.files) : []; + if (selected.length) onAddFiles(selected); e.target.value = ""; // allow re-picking the same file } return (
+ onInvocationChange({ + ...invocation, + skills: invocation.skills.filter((skill) => skill.name !== name), + })} + onRemoveAgent={() => onInvocationChange({ skills: [] })} + /> {attachments.length > 0 && ( -
- {attachments.map((a, i) => ( - onRemoveAttachment(i)} - /> - ))} -
+ )}
+ {trigger ? ( +
+
+ {trigger.kind === "skill" ? : } + {trigger.kind === "skill" ? "调用技能" : "使用子 Agent"} + {trigger.kind === "skill" ? "/" : "@"} +
+ {capabilitiesLoading ? ( +
正在读取 Agent 能力…
+ ) : suggestions.length === 0 ? ( +
+ {trigger.kind === "skill" ? "当前 Agent 没有匹配技能" : "当前 Agent 没有匹配子 Agent"} +
+ ) : ( +
+ {suggestions.map((item, index) => ( + + ))} +
+ )} +
+ ) : null}
@@ -95,10 +255,18 @@ export function Composer({ +
@@ -112,8 +280,45 @@ export function Composer({ value={value} disabled={disabled} placeholder={disabled ? "请选择 Agent" : "给智能体发消息…"} - onChange={(e) => onChange(e.target.value)} + aria-expanded={Boolean(trigger)} + onChange={(e) => { + onChange(e.target.value); + updateCompletion(e.target.value, e.target.selectionStart); + }} + onSelect={(e) => updateCompletion(e.currentTarget.value, e.currentTarget.selectionStart)} + onBlur={() => setTimeout(() => setTrigger(null), 0)} onKeyDown={(e) => { + if (trigger) { + if (e.key === "ArrowDown" && suggestions.length > 0) { + e.preventDefault(); + setActiveIndex((index) => (index + 1) % suggestions.length); + return; + } + if (e.key === "ArrowUp" && suggestions.length > 0) { + e.preventDefault(); + setActiveIndex((index) => (index - 1 + suggestions.length) % suggestions.length); + return; + } + if ((e.key === "Enter" || e.key === "Tab") && suggestions[activeIndex]) { + e.preventDefault(); + choose(suggestions[activeIndex]); + return; + } + if (e.key === "Escape") { + e.preventDefault(); + setTrigger(null); + return; + } + } + if ( + e.key === "Backspace" && + !value && + e.currentTarget.selectionStart === 0 && + e.currentTarget.selectionEnd === 0 + ) { + removeLastInvocation(); + return; + } if (e.key === "Enter" && !e.shiftKey) { e.preventDefault(); if (canSend) onSubmit(); @@ -156,51 +361,21 @@ export function Composer({ onChange={onInputChange} /> + -
- ); -} - -function AttachmentChip({ - mimeType, - data, - name, - onRemove, -}: { - mimeType: string; - data: string; - name?: string; - onRemove: () => void; -}) { - const isImage = mimeType.startsWith("image/"); - return ( -
- {isImage ? ( - {name - ) : ( - <> - - {name ?? "file.pdf"} - - )} -
); } diff --git a/frontend/src/ui/InvocationChips.tsx b/frontend/src/ui/InvocationChips.tsx new file mode 100644 index 00000000..7121d6a2 --- /dev/null +++ b/frontend/src/ui/InvocationChips.tsx @@ -0,0 +1,46 @@ +import { AtSign, Sparkles, X } from "lucide-react"; +import type { FrontendInvocation } from "../adk/client"; + +export interface InvocationChipsProps { + value: FrontendInvocation; + onRemoveSkill?: (name: string) => void; + onRemoveAgent?: () => void; +} + +export function InvocationChips({ + value, + onRemoveSkill, + onRemoveAgent, +}: InvocationChipsProps) { + if (value.skills.length === 0 && !value.targetAgent) return null; + + return ( +
+ {value.skills.map((skill) => ( + + + /{skill.name} + {onRemoveSkill ? ( + + ) : null} + + ))} + {value.targetAgent ? ( + + + {value.targetAgent.name} + {onRemoveAgent ? ( + + ) : null} + + ) : null} +
+ ); +} diff --git a/frontend/src/ui/ManageAgents.tsx b/frontend/src/ui/ManageAgents.tsx index 99cf03f9..648a091f 100644 --- a/frontend/src/ui/ManageAgents.tsx +++ b/frontend/src/ui/ManageAgents.tsx @@ -71,6 +71,9 @@ export function ManageAgentsView({ author }: ManageAgentsViewProps) { type: "llm", model: next.detail.model, tools: [], + skills: [], + path: [next.detail.name], + mentionable: false, children: [], }, ]; diff --git a/frontend/src/ui/Markdown.tsx b/frontend/src/ui/Markdown.tsx index f9718b3d..c42c5714 100644 --- a/frontend/src/ui/Markdown.tsx +++ b/frontend/src/ui/Markdown.tsx @@ -1,5 +1,7 @@ import { memo } from "react"; +import { Maximize2 } from "lucide-react"; import ReactMarkdown from "react-markdown"; +import { PhotoView } from "react-photo-view"; import remarkGfm from "remark-gfm"; import rehypeHighlight from "rehype-highlight"; import "highlight.js/styles/github.css"; @@ -20,6 +22,26 @@ function MarkdownImpl({ text, className }: { text: string; className?: string }) a: ({ node, ...props }) => ( ), + img: ({ node, src, alt, ...props }) => { + const image = ( + {alt + ); + if (!src) return image; + return ( + + + + ); + }, }} > {text} diff --git a/frontend/src/ui/Media.tsx b/frontend/src/ui/Media.tsx new file mode 100644 index 00000000..374fe6ef --- /dev/null +++ b/frontend/src/ui/Media.tsx @@ -0,0 +1,228 @@ +import { useEffect, useMemo, useState } from "react"; +import { + Download, + FileText, + FileType2, + FileVideo2, + ImageIcon, + LoaderCircle, + Maximize2, + X, +} from "lucide-react"; +import { AnimatePresence, motion } from "motion/react"; +import { PhotoView } from "react-photo-view"; +import { mediaContentUrl } from "../adk/client"; +import { Markdown } from "./Markdown"; + +export interface MediaItem { + id: string; + mimeType?: string; + data?: string; + uri?: string; + name?: string; + sizeBytes?: number; + previewUrl?: string; + status?: "uploading" | "ready" | "error"; + error?: string; +} + +interface MediaGroupProps { + appName: string; + items: MediaItem[]; + compact?: boolean; + onRemove?: (id: string) => void; +} + +function mediaKind(mimeType = "") { + if (mimeType.startsWith("image/")) return "image"; + if (mimeType.startsWith("video/")) return "video"; + if (mimeType === "application/pdf") return "pdf"; + if (mimeType === "text/markdown") return "markdown"; + return "text"; +} + +function labelFor(item: MediaItem) { + const kind = mediaKind(item.mimeType); + if (kind === "pdf") return "PDF"; + if (kind === "markdown") return "MD"; + if (kind === "video") return item.mimeType?.split("/")[1]?.toUpperCase() ?? "VIDEO"; + if (kind === "image") return item.mimeType?.split("/")[1]?.toUpperCase() ?? "IMAGE"; + return "TXT"; +} + +function formatBytes(bytes?: number) { + if (!bytes) return ""; + if (bytes < 1024) return `${bytes} B`; + if (bytes < 1024 * 1024) return `${Math.round(bytes / 1024)} KB`; + return `${(bytes / (1024 * 1024)).toFixed(1)} MB`; +} + +function sourceFor(item: MediaItem, appName: string) { + if (item.previewUrl) return item.previewUrl; + if (item.data) return `data:${item.mimeType ?? "application/octet-stream"};base64,${item.data}`; + if (item.uri) return mediaContentUrl(appName, item.uri); + return ""; +} + +function KindIcon({ kind }: { kind: ReturnType }) { + if (kind === "image") return ; + if (kind === "video") return ; + if (kind === "pdf") return ; + return ; +} + +export function MediaGroup({ appName, items, compact = false, onRemove }: MediaGroupProps) { + const [open, setOpen] = useState(null); + return ( + <> +
+ {items.map((item) => { + const kind = mediaKind(item.mimeType); + const source = sourceFor(item, appName); + const disabled = item.status === "uploading" || item.status === "error" || !source; + const previewButton = ( + + ); + return ( + + {kind === "image" && !disabled ? ( + {previewButton} + ) : previewButton} + {onRemove ? ( + + ) : null} + + ); + })} +
+ + {open ? ( + setOpen(null)} /> + ) : null} + + + ); +} + +function MediaViewer({ appName, item, onClose }: { appName: string; item: MediaItem; onClose: () => void }) { + const source = useMemo(() => sourceFor(item, appName), [appName, item]); + const kind = mediaKind(item.mimeType); + const [text, setText] = useState(""); + const [loading, setLoading] = useState(kind === "text" || kind === "markdown"); + const [loadError, setLoadError] = useState(""); + + useEffect(() => { + const onKey = (event: KeyboardEvent) => { + if (event.key === "Escape") onClose(); + }; + window.addEventListener("keydown", onKey); + return () => window.removeEventListener("keydown", onKey); + }, [onClose]); + + useEffect(() => { + if (kind !== "text" && kind !== "markdown") return; + const controller = new AbortController(); + setLoading(true); + setLoadError(""); + fetch(source, { signal: controller.signal }) + .then((response) => { + if (!response.ok) throw new Error(`HTTP ${response.status}`); + return response.text(); + }) + .then(setText) + .catch((error: unknown) => { + if (!controller.signal.aborted) { + setLoadError(error instanceof Error ? error.message : String(error)); + } + }) + .finally(() => { + if (!controller.signal.aborted) setLoading(false); + }); + return () => controller.abort(); + }, [kind, source]); + + return ( + { + if (event.target === event.currentTarget) onClose(); + }} + > + +
+
+ {item.name ?? "附件"} + {labelFor(item)}{item.sizeBytes ? ` · ${formatBytes(item.sizeBytes)}` : ""} +
+ +
+
+ {kind === "image" ? {item.name : null} + {kind === "video" ?