fix: handle LLM responses wrapped in markdown code fences (#3972)

Signed-off-by: Yeganathan S <63534555+skwowet@users.noreply.github.com>

Author: skwowet

services/apps/organizations_enrichment_worker/src/activities/llm.ts

@@ -1,3 +1,4 @@
+import { parseLlmJson } from '@crowd/common'
 import { LlmService } from '@crowd/common_services'
 import {
   OrganizationField,
@@ -132,7 +133,7 @@ export async function selectMostRelevantDomainWithLLM(
       organizationId,
     )
     if (!response) throw new Error('LLM returned no response')
-    return JSON.parse(response.answer) as LlmDomainSelection
+    return parseLlmJson<LlmDomainSelection>(response.answer)
   }
 
   const MAX_RETRIES = 1

services/apps/profiles_worker/src/workflows/member/processMemberBotAnalysisWithLLM.ts

@@ -1,5 +1,6 @@
 import { proxyActivities } from '@temporalio/workflow'
 
+import { parseLlmJson } from '@crowd/common'
 import { LlmQueryType } from '@crowd/types'
 
 import * as activities from '../../activities'
@@ -72,7 +73,7 @@ export async function processMemberBotAnalysisWithLLM(
 
   const llm = await getLLMResult(LlmQueryType.MEMBER_BOT_VALIDATION, PROMPT, memberId)
 
-  const { isBot, signals } = JSON.parse(llm.answer) as MemberBotSuggestionResult
+  const { isBot, signals } = parseLlmJson<MemberBotSuggestionResult>(llm.answer)
 
   if (!isBot) {
     await createMemberNoBot(memberId)

services/libs/common/src/index.ts

@@ -36,6 +36,7 @@ export * from './displayName'
 export * from './jira'
 export * from './email'
 export * from './bot'
+export * from './llm'
 
 export * from './i18n'
 export * from './member'

services/libs/common/src/llm.ts

@@ -0,0 +1,78 @@
+export function parseLlmJson<T>(answer: string): T {
+  const raw = answer.trim()
+
+  let lastError: string | undefined
+
+  const tryParse = (str: string): T | undefined => {
+    try {
+      return JSON.parse(str) as T
+    } catch (e) {
+      lastError = (e as Error).message
+      return undefined
+    }
+  }
+
+  // 1. Direct parse
+  const direct = tryParse(raw)
+  if (direct !== undefined) return direct
+
+  // 2. Fenced ```json``` block
+  const fenced = raw.match(/^```(?:json)?\n([\s\S]*?)\n```$/i)?.[1]
+  if (fenced) {
+    const parsed = tryParse(fenced.trim())
+    if (parsed !== undefined) return parsed
+  }
+
+  // 3. Balanced extraction — try each delimiter type, scanning forward on failed parse
+  const delimiters = [
+    ...([
+      ['{', '}'],
+      ['[', ']'],
+    ] as const),
+  ].sort((a, b) => {
+    const ia = raw.indexOf(a[0])
+    const ib = raw.indexOf(b[0])
+    if (ia === -1) return 1
+    if (ib === -1) return -1
+    return ia - ib
+  })
+
+  for (const [open, close] of delimiters) {
+    let start = raw.indexOf(open)
+
+    while (start >= 0) {
+      let depth = 0,
+        inString = false,
+        escaped = false
+
+      for (let i = start; i < raw.length; i++) {
+        const char = raw[i]
+
+        if (inString) {
+          if (escaped) escaped = false
+          else if (char === '\\') escaped = true
+          else if (char === '"') inString = false
+          continue
+        }
+
+        if (char === '"') {
+          inString = true
+        } else if (char === open) {
+          depth++
+        } else if (char === close && --depth === 0) {
+          const parsed = tryParse(raw.slice(start, i + 1))
+          if (parsed !== undefined) return parsed
+          start = raw.indexOf(open, i + 1)
+          break
+        }
+      }
+
+      if (depth !== 0) break
+    }
+  }
+
+  const preview = raw.length > 200 ? `${raw.slice(0, 200)}…` : raw
+  throw new SyntaxError(
+    `LLM response does not contain valid JSON: ${lastError} | Input: ${preview}`,
+  )
+}

services/libs/common_services/src/services/llm.service.ts

@@ -5,7 +5,7 @@ import {
 } from '@aws-sdk/client-bedrock-runtime'
 import { performance } from 'perf_hooks'
 
-import { IS_LLM_ENABLED } from '@crowd/common'
+import { IS_LLM_ENABLED, parseLlmJson } from '@crowd/common'
 import { insertPromptHistoryEntry } from '@crowd/data-access-layer'
 import { QueryExecutor } from '@crowd/data-access-layer'
 import { Logger, LoggerBase } from '@crowd/logging'
@@ -177,7 +177,7 @@ export class LlmService extends LoggerBase {
       } as ILlmResult<LlmMemberEnrichmentResult>
     }
 
-    const result = JSON.parse(response.answer)
+    const result = parseLlmJson<LlmMemberEnrichmentResult>(response.answer)
 
     return {
       result,
@@ -200,7 +200,7 @@ export class LlmService extends LoggerBase {
       } as ILlmResult<{ profileIndex: number }>
     }
 
-    const result = JSON.parse(response.answer)
+    const result = parseLlmJson<{ profileIndex: number }>(response.answer)
 
     return {
       result,
@@ -224,7 +224,7 @@ export class LlmService extends LoggerBase {
       } as ILlmResult<T>
     }
 
-    const result = JSON.parse(response.answer)
+    const result = parseLlmJson<T>(response.answer)
 
     return {
       result,
@@ -248,7 +248,7 @@ export class LlmService extends LoggerBase {
       } as ILlmResult<T>
     }
 
-    const result = JSON.parse(response.answer)
+    const result = parseLlmJson<T>(response.answer)
 
     return {
       result,
@@ -268,7 +268,7 @@ export class LlmService extends LoggerBase {
       } as ILlmResult<T>
     }
 
-    const result = JSON.parse(response.answer)
+    const result = parseLlmJson<T>(response.answer)
 
     return {
       result,
@@ -285,7 +285,7 @@ export class LlmService extends LoggerBase {
       } as ILlmResult<T>
     }
 
-    const result = JSON.parse(response.answer)
+    const result = parseLlmJson<T>(response.answer)
 
     return {
       result,
@@ -302,7 +302,7 @@ export class LlmService extends LoggerBase {
       } as ILlmResult<T>
     }
 
-    const result = JSON.parse(response.answer)
+    const result = parseLlmJson<T>(response.answer)
 
     return {
       result,