Fix chat name generation

Author: Roman Snapko

ai-prompts/chat-name.txt

@@ -1,9 +1,13 @@
 You are an AI assistant on the OpenOps platform, where users interact about FinOps, cloud providers (AWS, Azure, GCP), OpenOps features, and workflow automation.
 
-Your task:
-Given the following conversation, suggest a short, descriptive name (max five words) that best summarizes the main topic, question, or action discussed in this chat. 
+Task:
+Analyze the provided conversation and attempt to produce a concise chat name describing the main topic, question, or action.
 
-Guidelines:
-- The name should be specific (not generic like "Chat" or "Conversation"), and reflect the user's intent (e.g., "AWS Cost Optimization", "Create Budget Workflow", "OpenOps Integration Help").
-- Limit the name to five words or less.
-- Respond with only the chat name.
+Rules:
+- If you can confidently produce a specific, helpful name (not generic like "Chat" or "Conversation"), set `isGenerated` to true and provide `name`.
+- The `name` must be five words or fewer
+- If there is insufficient information, the content is unclear, or you cannot determine a good name, set `isGenerated` to false.
+
+Notes:
+- Keep the name short and specific.
+- Avoid quotes, punctuation-heavy outputs, or trailing spaces in the name.

packages/react-ui/src/app/features/ai/lib/ai-assistant-chat-history-api.ts

@@ -1,5 +1,5 @@
 import { api } from '@/app/lib/api';
-import { ListChatsResponse } from '@openops/shared';
+import { GeneratedChatName, ListChatsResponse } from '@openops/shared';
 
 export const aiAssistantChatHistoryApi = {
   list() {
@@ -9,7 +9,7 @@ export const aiAssistantChatHistoryApi = {
     return api.delete<void>(`/v1/ai/conversation/${chatId}`);
   },
   generateName(chatId: string) {
-    return api.post<{ chatName: string }>('/v1/ai/conversation/chat-name', {
+    return api.post<GeneratedChatName>('/v1/ai/conversation/chat-name', {
       chatId,
     });
   },

packages/react-ui/src/app/features/ai/lib/assistant-ui-chat-hook.ts

@@ -243,7 +243,10 @@ export const useAssistantChat = ({
       if (messagesRef.current.length >= MIN_MESSAGES_BEFORE_NAME_GENERATION) {
         try {
           hasAttemptedNameGenerationRef.current[chatId] = true;
-          await aiAssistantChatHistoryApi.generateName(chatId);
+          const result = await aiAssistantChatHistoryApi.generateName(chatId);
+          if (!result.isGenerated) {
+            hasAttemptedNameGenerationRef.current[chatId] = false;
+          }
           qc.invalidateQueries({ queryKey: [QueryKeys.assistantHistory] });
         } catch (error) {
           console.error('Failed to generate chat name', error);

packages/server/api/src/app/ai/chat/ai-chat.service.ts

@@ -2,6 +2,7 @@ import {
   AiAuth,
   getAiModelFromConnection,
   getAiProviderLanguageModel,
+  isLLMTelemetryEnabled,
 } from '@openops/common';
 import {
   AppSystemProp,
@@ -14,14 +15,19 @@ import {
   ApplicationError,
   CustomAuthConnectionValue,
   ErrorCode,
+  GeneratedChatName,
   removeConnectionBrackets,
 } from '@openops/shared';
-import { LanguageModel, ModelMessage, UIMessage, generateText } from 'ai';
+import { generateObject, LanguageModel, ModelMessage, UIMessage } from 'ai';
+import { z } from 'zod';
 import { appConnectionService } from '../../app-connection/app-connection-service/app-connection-service';
 import { aiConfigService } from '../config/ai-config.service';
 import { loadPrompt } from './prompts.service';
 import { Conversation } from './types';
-import { mergeToolResultsIntoMessages } from './utils';
+import {
+  mergeToolResultsIntoMessages,
+  sanitizeMessagesForChatName,
+} from './utils';
 
 const chatContextKey = (
   chatId: string,
@@ -84,28 +90,38 @@ export const generateChatIdForMCP = (params: {
   });
 };
 
+const generatedChatNameSchema = z.object({
+  name: z
+    .string()
+    .max(100)
+    .nullable()
+    .describe('Conversation name or null if it was not generated'),
+  isGenerated: z.boolean().describe('Whether the name was generated or not'),
+});
+
 export async function generateChatName(
   messages: ModelMessage[],
   projectId: string,
-): Promise<string> {
+): Promise<GeneratedChatName> {
   const { languageModel } = await getLLMConfig(projectId);
   const systemPrompt = await loadPrompt('chat-name.txt');
   if (!systemPrompt.trim()) {
     throw new Error('Failed to load prompt to generate the chat name.');
   }
-  const prompt: ModelMessage[] = [
-    {
-      role: 'system',
-      content: systemPrompt,
-    } as const,
-    ...messages,
-  ];
-  const response = await generateText({
+
+  const sanitizedMessages: ModelMessage[] =
+    sanitizeMessagesForChatName(messages);
+
+  const result = await generateObject({
     model: languageModel,
-    messages: prompt,
+    system: systemPrompt,
+    messages: sanitizedMessages,
+    schema: generatedChatNameSchema,
+    experimental_telemetry: { isEnabled: isLLMTelemetryEnabled() },
     maxRetries: 2,
   });
-  return response.text.trim();
+
+  return result.object;
 }
 
 export const updateChatName = async (

packages/server/api/src/app/ai/chat/ai-mcp-chat.controller.ts

@@ -22,7 +22,7 @@ import {
   UpdateChatModelRequest,
   UpdateChatModelResponse,
 } from '@openops/shared';
-import { ModelMessage, UserModelMessage } from 'ai';
+import { ModelMessage } from 'ai';
 import { FastifyReply } from 'fastify';
 import { StatusCodes } from 'http-status-codes';
 import {
@@ -247,27 +247,24 @@ export const aiMCPChatController: FastifyPluginAsyncTypebox = async (app) => {
       const { chatHistory } = await getConversation(chatId, userId, projectId);
 
       if (chatHistory.length === 0) {
-        return await reply.code(200).send({ chatName: DEFAULT_CHAT_NAME });
+        return await reply
+          .code(200)
+          .send({ name: DEFAULT_CHAT_NAME, isGenerated: false });
       }
 
-      const userMessages = chatHistory.filter(
-        (msg): msg is UserModelMessage =>
-          msg &&
-          typeof msg === 'object' &&
-          'role' in msg &&
-          msg.role === 'user',
-      );
+      const generated = await generateChatName(chatHistory, projectId);
 
-      if (userMessages.length === 0) {
-        return await reply.code(200).send({ chatName: DEFAULT_CHAT_NAME });
+      if (!generated.isGenerated) {
+        return await reply
+          .code(200)
+          .send({ name: DEFAULT_CHAT_NAME, isGenerated: false });
       }
 
-      const rawChatName = await generateChatName(userMessages, projectId);
-      const chatName = rawChatName.trim() || DEFAULT_CHAT_NAME;
-
-      await updateChatName(chatId, userId, projectId, chatName);
+      if (generated.isGenerated && generated.name) {
+        await updateChatName(chatId, userId, projectId, generated.name);
+      }
 
-      return await reply.code(200).send({ chatName });
+      return await reply.code(200).send(generated);
     } catch (error) {
       return handleError(error, reply, 'generate chat name');
     }

packages/server/api/src/app/ai/chat/utils.ts

@@ -52,6 +52,46 @@ export function mergeToolResultsIntoMessages(
   return uiMessages;
 }
 
+/**
+ * Sanitize chat history for secondary tasks like naming/summarization.
+ * - keeps only 'user' and 'assistant' roles
+ * - strips tool calls and non-text parts
+ * - merges multiple text parts into a single string with newlines
+ */
+export function sanitizeMessagesForChatName(
+  messages: ModelMessage[],
+): ModelMessage[] {
+  return messages
+    .filter((m) => m.role === 'user' || m.role === 'assistant')
+    .map((m) => {
+      if (typeof m.content === 'string') {
+        const text = m.content.trim();
+        return text ? { role: m.role, content: text } : null;
+      }
+      if (Array.isArray(m.content)) {
+        const textParts: string[] = [];
+        for (const part of m.content as Array<unknown>) {
+          if (
+            part &&
+            typeof part === 'object' &&
+            'type' in (part as Record<string, unknown>)
+          ) {
+            const p = part as { type?: string; text?: string };
+            if (p.type === 'text' && typeof p.text === 'string') {
+              textParts.push(p.text);
+            }
+          }
+        }
+        const merged = textParts.join('\n').trim();
+        return merged
+          ? ({ role: m.role, content: merged } as ModelMessage)
+          : null;
+      }
+      return null;
+    })
+    .filter((m) => m !== null);
+}
+
 function isToolMessage(msg: ModelMessage): boolean {
   return (
     msg.role === 'tool' && Array.isArray(msg.content) && msg.content.length > 0

packages/server/api/test/unit/ai/utils.test.ts

@@ -1,6 +1,9 @@
 /* eslint-disable @typescript-eslint/no-explicit-any */
 import { ModelMessage } from 'ai';
-import { mergeToolResultsIntoMessages } from '../../../src/app/ai/chat/utils';
+import {
+  mergeToolResultsIntoMessages,
+  sanitizeMessagesForChatName,
+} from '../../../src/app/ai/chat/utils';
 
 describe('mergeToolResultsIntoMessages', () => {
   describe('basic message handling', () => {
@@ -801,3 +804,80 @@ describe('mergeToolResultsIntoMessages', () => {
     });
   });
 });
+
+describe('sanitizeMessagesForChatName', () => {
+  it('keeps only user and assistant roles, dropping tool messages', () => {
+    const messages: ModelMessage[] = [
+      { role: 'system', content: 'sys' },
+      { role: 'user', content: 'hello ' },
+      {
+        role: 'assistant',
+        content: [{ type: 'text', text: 'hi' } as any],
+      },
+      {
+        role: 'tool',
+        content: [{ toolCallId: 'x', type: 'tool_result', content: 'ok' } as any],
+      },
+    ];
+
+    const result = sanitizeMessagesForChatName(messages);
+    expect(result).toEqual([
+      { role: 'user', content: 'hello' },
+      { role: 'assistant', content: 'hi' },
+    ]);
+  });
+
+  it('trims string content and removes empty messages after trim', () => {
+    const messages: ModelMessage[] = [
+      { role: 'user', content: '   ' },
+      { role: 'assistant', content: '  answer  ' },
+    ];
+
+    const result = sanitizeMessagesForChatName(messages);
+    expect(result).toEqual([{ role: 'assistant', content: 'answer' }]);
+  });
+
+  it('merges multiple text parts with newlines and trims overall', () => {
+    const messages: ModelMessage[] = [
+      {
+        role: 'user',
+        content: [
+          { type: 'text', text: 'Line 1' } as any,
+          { type: 'text', text: 'Line 2' } as any,
+        ],
+      },
+    ];
+
+    const result = sanitizeMessagesForChatName(messages);
+    expect(result).toEqual([{ role: 'user', content: 'Line 1\nLine 2' }]);
+  });
+
+  it('ignores non-text parts (e.g., tool_use, image) and keeps only text', () => {
+    const messages: ModelMessage[] = [
+      {
+        role: 'assistant',
+        content: [
+          { type: 'tool_use', id: 't1', name: 'X', input: {} } as any,
+          { type: 'text', text: 'Only this is kept' } as any,
+          { type: 'image', image: 'raw' } as any,
+        ],
+      },
+    ];
+
+    const result = sanitizeMessagesForChatName(messages);
+    expect(result).toEqual([
+      { role: 'assistant', content: 'Only this is kept' },
+    ]);
+  });
+
+  it('returns empty array when nothing useful remains', () => {
+    const messages: ModelMessage[] = [
+      { role: 'system', content: 'not included' },
+      { role: 'tool', content: [] as any },
+      { role: 'user', content: [] as any },
+    ];
+
+    const result = sanitizeMessagesForChatName(messages);
+    expect(result).toEqual([]);
+  });
+});

packages/shared/src/lib/ai/chat/index.ts

@@ -129,3 +129,9 @@ export const UpdateChatModelResponse = Type.Object({
 export type UpdateChatModelResponse = Static<typeof UpdateChatModelResponse>;
 
 export * from './code-output-structure';
+
+export const GeneratedChatName = Type.Object({
+  name: Type.Union([Type.String(), Type.Null()]),
+  isGenerated: Type.Boolean(),
+});
+export type GeneratedChatName = Static<typeof GeneratedChatName>;