Fix chat name generation (#1719)

<!--
Ensure the title clearly reflects what was changed.
Provide a clear and concise description of the changes made. The PR
should only contain the changes related to the issue, and no other
unrelated changes.
-->

Fixes OPS-3173.

Deployed on UX


The idea is to let LLM decided whether it has enough context or no. In
case AI decides that name is generated we set it as chat name. In case
there is no enough data, we wait for the next LLM response and try to
generate the name once again



<!-- This is an auto-generated comment: release notes by coderabbit.ai
-->

## Summary by CodeRabbit

* **New Features**
* Chat name generation now includes an `isGenerated` flag to indicate
whether a name was successfully generated or a default was used.

* **Bug Fixes**
* Improved chat name generation reliability with enhanced validation and
structured output handling.

* **Chores**
* Updated chat name API response format from `chatName` to include
`name` and `isGenerated` fields for better clarity on generation status.

<sub>✏️ Tip: You can customize this high-level summary in your review
settings.</sub>

<!-- end of auto-generated comment: release notes by coderabbit.ai -->

---------

Co-authored-by: greptile-apps[bot] <165735046+greptile-apps[bot]@users.noreply.github.com>

Author: Roman Snapko

ai-prompts/chat-name.txt

@@ -1,9 +1,13 @@
 You are an AI assistant on the OpenOps platform, where users interact about FinOps, cloud providers (AWS, Azure, GCP), OpenOps features, and workflow automation.
 
-Your task:
-Given the following conversation, suggest a short, descriptive name (max five words) that best summarizes the main topic, question, or action discussed in this chat. 
+Task:
+Analyze the provided conversation and attempt to produce a concise chat name describing the main topic, question, or action.
 
-Guidelines:
-- The name should be specific (not generic like "Chat" or "Conversation"), and reflect the user's intent (e.g., "AWS Cost Optimization", "Create Budget Workflow", "OpenOps Integration Help").
-- Limit the name to five words or less.
-- Respond with only the chat name.
+Rules:
+- If you can confidently produce a specific, helpful name (not generic like "Chat" or "Conversation"), set `isGenerated` to true and provide `name`.
+- The `name` must be five words or fewer
+- If there is insufficient information, the content is unclear, or you cannot determine a good name, set `isGenerated` to false.
+
+Notes:
+- Keep the name short and specific.
+- Avoid quotes, punctuation-heavy outputs, or trailing spaces in the name.

packages/react-ui/src/app/features/ai/lib/ai-assistant-chat-history-api.ts

@@ -1,5 +1,5 @@
 import { api } from '@/app/lib/api';
-import { ListChatsResponse } from '@openops/shared';
+import { GeneratedChatName, ListChatsResponse } from '@openops/shared';
 
 export const aiAssistantChatHistoryApi = {
   list() {
@@ -9,7 +9,7 @@ export const aiAssistantChatHistoryApi = {
     return api.delete<void>(`/v1/ai/conversation/${chatId}`);
   },
   generateName(chatId: string) {
-    return api.post<{ chatName: string }>('/v1/ai/conversation/chat-name', {
+    return api.post<GeneratedChatName>('/v1/ai/conversation/chat-name', {
       chatId,
     });
   },

packages/react-ui/src/app/features/ai/lib/assistant-ui-chat-hook.ts

@@ -243,7 +243,10 @@ export const useAssistantChat = ({
       if (messagesRef.current.length >= MIN_MESSAGES_BEFORE_NAME_GENERATION) {
         try {
           hasAttemptedNameGenerationRef.current[chatId] = true;
-          await aiAssistantChatHistoryApi.generateName(chatId);
+          const result = await aiAssistantChatHistoryApi.generateName(chatId);
+          if (!result.isGenerated) {
+            hasAttemptedNameGenerationRef.current[chatId] = false;
+          }
           qc.invalidateQueries({ queryKey: [QueryKeys.assistantHistory] });
         } catch (error) {
           console.error('Failed to generate chat name', error);

packages/server/api/src/app/ai/chat/ai-chat.service.ts

@@ -2,26 +2,35 @@ import {
   AiAuth,
   getAiModelFromConnection,
   getAiProviderLanguageModel,
+  isLLMTelemetryEnabled,
 } from '@openops/common';
 import {
   AppSystemProp,
   cacheWrapper,
   hashUtils,
+  logger,
   system,
 } from '@openops/server-shared';
 import {
   AiConfigParsed,
   ApplicationError,
   CustomAuthConnectionValue,
   ErrorCode,
+  GeneratedChatName,
+  isEmpty,
   removeConnectionBrackets,
 } from '@openops/shared';
-import { LanguageModel, ModelMessage, UIMessage, generateText } from 'ai';
+import { generateObject, LanguageModel, ModelMessage, UIMessage } from 'ai';
+import { z } from 'zod';
 import { appConnectionService } from '../../app-connection/app-connection-service/app-connection-service';
 import { aiConfigService } from '../config/ai-config.service';
+import { findFirstKeyInObject } from '../mcp/llm-query-router';
 import { loadPrompt } from './prompts.service';
 import { Conversation } from './types';
-import { mergeToolResultsIntoMessages } from './utils';
+import {
+  mergeToolResultsIntoMessages,
+  sanitizeMessagesForChatName,
+} from './utils';
 
 const chatContextKey = (
   chatId: string,
@@ -84,28 +93,74 @@ export const generateChatIdForMCP = (params: {
   });
 };
 
+const generatedChatNameSchema = z.object({
+  name: z
+    .string()
+    .max(100)
+    .nullable()
+    .describe('Conversation name or null if it was not generated'),
+  isGenerated: z.boolean().describe('Whether the name was generated or not'),
+});
+
+/**
+ * Attempts to repair a malformed JSON string produced by the model for chat name generation.
+ * It extracts only the expected fields according to generatedChatNameSchema.
+ * Returns null if the input cannot be parsed or repaired (so the AI SDK can retry/throw).
+ */
+const repairText = (text: string): string | null => {
+  try {
+    const parsed = JSON.parse(text);
+
+    const nameRaw = findFirstKeyInObject(parsed, 'name');
+    let name: string | null = null;
+    if (typeof nameRaw === 'string') {
+      const trimmed = nameRaw.trim();
+      name = trimmed.length > 0 ? trimmed.slice(0, 100) : null;
+    }
+
+    const isGeneratedRaw = findFirstKeyInObject(parsed, 'isGenerated');
+    const isGenerated =
+      typeof isGeneratedRaw === 'boolean' ? isGeneratedRaw : Boolean(name);
+
+    return JSON.stringify({ name, isGenerated });
+  } catch {
+    return null;
+  }
+};
+
 export async function generateChatName(
   messages: ModelMessage[],
   projectId: string,
-): Promise<string> {
-  const { languageModel } = await getLLMConfig(projectId);
+): Promise<GeneratedChatName> {
+  const { languageModel, aiConfig } = await getLLMConfig(projectId);
   const systemPrompt = await loadPrompt('chat-name.txt');
   if (!systemPrompt.trim()) {
     throw new Error('Failed to load prompt to generate the chat name.');
   }
-  const prompt: ModelMessage[] = [
-    {
-      role: 'system',
-      content: systemPrompt,
-    } as const,
-    ...messages,
-  ];
-  const response = await generateText({
-    model: languageModel,
-    messages: prompt,
-    maxRetries: 2,
-  });
-  return response.text.trim();
+
+  const sanitizedMessages: ModelMessage[] =
+    sanitizeMessagesForChatName(messages);
+
+  if (isEmpty(sanitizedMessages)) {
+    return { name: null, isGenerated: false };
+  }
+
+  try {
+    const result = await generateObject({
+      model: languageModel,
+      system: systemPrompt,
+      messages: sanitizedMessages,
+      schema: generatedChatNameSchema,
+      ...aiConfig.modelSettings,
+      experimental_telemetry: { isEnabled: isLLMTelemetryEnabled() },
+      experimental_repairText: async ({ text }) => repairText(text),
+      maxRetries: 2,
+    });
+    return result.object;
+  } catch (error) {
+    logger.error('Failed to generate chat name', { error });
+    return { name: null, isGenerated: false };
+  }
 }
 
 export const updateChatName = async (

packages/server/api/src/app/ai/chat/ai-mcp-chat.controller.ts

@@ -22,7 +22,7 @@ import {
   UpdateChatModelRequest,
   UpdateChatModelResponse,
 } from '@openops/shared';
-import { ModelMessage, UserModelMessage } from 'ai';
+import { ModelMessage } from 'ai';
 import { FastifyReply } from 'fastify';
 import { StatusCodes } from 'http-status-codes';
 import removeMarkdown from 'markdown-to-text';
@@ -248,27 +248,30 @@ export const aiMCPChatController: FastifyPluginAsyncTypebox = async (app) => {
       const { chatHistory } = await getConversation(chatId, userId, projectId);
 
       if (chatHistory.length === 0) {
-        return await reply.code(200).send({ chatName: DEFAULT_CHAT_NAME });
+        return await reply
+          .code(200)
+          .send({ name: DEFAULT_CHAT_NAME, isGenerated: false });
       }
 
-      const userMessages = chatHistory.filter(
-        (msg): msg is UserModelMessage =>
-          msg &&
-          typeof msg === 'object' &&
-          'role' in msg &&
-          msg.role === 'user',
-      );
+      const generated = await generateChatName(chatHistory, projectId);
 
-      if (userMessages.length === 0) {
-        return await reply.code(200).send({ chatName: DEFAULT_CHAT_NAME });
+      if (!generated.isGenerated) {
+        return await reply
+          .code(200)
+          .send({ name: DEFAULT_CHAT_NAME, isGenerated: false });
       }
 
-      const rawChatName = await generateChatName(userMessages, projectId);
-      const chatName = removeMarkdown(rawChatName).trim() || DEFAULT_CHAT_NAME;
+      const chatName = generated?.name
+        ? removeMarkdown(generated.name).trim()
+        : DEFAULT_CHAT_NAME;
 
-      await updateChatName(chatId, userId, projectId, chatName);
+      if (generated.isGenerated && generated.name) {
+        await updateChatName(chatId, userId, projectId, chatName);
+      }
 
-      return await reply.code(200).send({ chatName });
+      return await reply
+        .code(200)
+        .send({ name: chatName, isGenerated: generated.isGenerated });
     } catch (error) {
       return handleError(error, reply, 'generate chat name');
     }

packages/server/api/src/app/ai/chat/utils.ts

@@ -52,6 +52,57 @@ export function mergeToolResultsIntoMessages(
   return uiMessages;
 }
 
+/**
+ * Sanitize chat history for secondary tasks like naming/summarization.
+ * - keeps only 'user' and 'assistant' roles
+ * - strips tool calls and non-text parts
+ * - merges multiple text parts into a single string with newlines
+ */
+export function sanitizeMessagesForChatName(
+  messages: ModelMessage[],
+): ModelMessage[] {
+  const isSupportedRole = (m: ModelMessage) =>
+    m.role === 'user' || m.role === 'assistant';
+
+  const extractText = (content: ModelMessage['content']): string | null => {
+    if (typeof content === 'string') {
+      const text = content.trim();
+      return text ?? null;
+    }
+
+    if (Array.isArray(content)) {
+      const merged = (content as Array<unknown>)
+        .reduce<string[]>((acc, part) => {
+          if (
+            part &&
+            typeof part === 'object' &&
+            'type' in (part as Record<string, unknown>)
+          ) {
+            const p = part as { type?: string; text?: string };
+            if (p.type === 'text' && typeof p.text === 'string') {
+              acc.push(p.text);
+            }
+          }
+          return acc;
+        }, [])
+        .join('\n')
+        .trim();
+
+      return merged ?? null;
+    }
+
+    return null;
+  };
+
+  return messages
+    .filter(isSupportedRole)
+    .map((m) => {
+      const text = extractText(m.content);
+      return text ? ({ role: m.role, content: text } as ModelMessage) : null;
+    })
+    .filter((m): m is ModelMessage => m !== null);
+}
+
 function isToolMessage(msg: ModelMessage): boolean {
   return (
     msg.role === 'tool' && Array.isArray(msg.content) && msg.content.length > 0

packages/server/api/src/app/ai/mcp/llm-query-router.ts

@@ -222,7 +222,7 @@ const getSystemPrompt = async (
  * @param targetKey - The key to search for.
  * @returns The value of the first key in the object.
  */
-function findFirstKeyInObject(
+export function findFirstKeyInObject(
   obj: Record<string, unknown>,
   targetKey: string,
 ): unknown {