fix: Wheat classification labels and data quality cleanup

Root cause fixes for wheat and similar HS code classifications:

Database:
- Cleaned 20 dirty descriptions with DGFT notification dates
- Fixed malformed date pattern in code 2942

Backend (llm-navigator.service.ts):
- Rewrote generateUniqueLabel to parse ": --" format correctly
- Now extracts "Seed" vs "Other" qualifiers from descriptions like "Durum wheat :-- Seed"
- Skip generic "Other" base labels to force meaningful qualifiers
- Fixed filterOptionsByKeywordsSimple to add ALL relevant Other options (was only adding first)
- Added seed pattern to generateSmartQuestion for sowing-related products

Results:
- Wheat now shows 4 distinct options: Durum wheat, Durum wheat - Other, Other - Seed, Other - Other
- No more meaningless "(19)" or "(99)" code suffixes in labels
- Durum wheat seeds correctly classifies to 1001.11.00

🤖 Generated with [Claude Code](https://claude.com/claude-code)

Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>

Author: AryanBV

backend/src/services/llm-conversational-classifier.service.ts

@@ -39,6 +39,9 @@ export interface LLMClassifyResponse {
   conversationId: string;
   responseType: 'questions' | 'classification' | 'error';
 
+  // Explicit round number for frontend deduplication
+  roundNumber?: number;
+
   // If responseType === 'questions' (matches frontend format)
   questions?: Array<{
     id: string;
@@ -436,6 +439,7 @@ async function processNavigationResult(
       success: true,
       conversationId,
       responseType: 'questions',  // Frontend expects plural
+      roundNumber: turnNumber,  // Explicit round number for frontend deduplication
       questions: [{
         id: result.question!.id,
         text: result.question!.text,

backend/src/services/llm-navigator.service.ts

@@ -274,17 +274,16 @@ function filterOptionsByKeywordsSimple(
     const matchedChapters = new Set(matchingOptions.map(o => o.code.substring(0, 2)));
 
     // Only add "Other" if all matched options are from the same chapter
-    // AND there's an "Other" option from that same chapter
+    // AND there are "Other" options from that same chapter
     if (matchedChapters.size === 1) {
       const chapter = [...matchedChapters][0];
-      const relevantOther = filteredOptions.find(o =>
+      const relevantOthers = filteredOptions.filter(o =>
         o.isOther &&
         o.code.substring(0, 2) === chapter &&
         !matchingOptions.includes(o)
       );
-      if (relevantOther) {
-        matchingOptions.push(relevantOther);
-      }
+      // Add ALL relevant "Other" options (e.g., "Other - Seed" and "Other - Other" for wheat)
+      matchingOptions.push(...relevantOthers);
     }
 
     logger.info(`[FILTER] Keyword filter: ${filteredOptions.length} -> ${matchingOptions.length} options`);
@@ -437,6 +436,7 @@ function generateSmartQuestion(options: HierarchyOption[]): string {
   // Check for common distinguishing patterns - ordered by specificity
   const patterns = [
     // Most specific patterns first
+    { keywords: ['seed', 'sowing', 'for seed'], question: "Is this for sowing (seed) or other purposes?" },
     { keywords: ['decaffeinated', 'not decaffeinated'], question: "Is the product decaffeinated?" },
     { keywords: ['roasted', 'not roasted'], question: "Is the product roasted?" },
     { keywords: ['crushed', 'ground', 'whole', 'neither crushed nor ground'], question: "What is the processing state?" },
@@ -786,84 +786,111 @@ async function getCodeDescription(code: string): Promise<string | null> {
  * ROOT CAUSE FIX: Generate UNIQUE labels that distinguish between similar codes
  *
  * Problem it solves:
- *   "Coffee, not roasted : --Not decaffeinated" → "Coffee, not roasted"
- *   "Coffee, not roasted : --Decaffeinated"     → "Coffee, not roasted"  // DUPLICATE!
+ *   "Durum wheat : -- Seed" → Should extract "Seed"
+ *   "Durum wheat : -- Other" → Should extract "Other"
+ *   "Other : -- Seed" → Should extract "Seed"
  *
  * Fixed behavior:
- *   Step 1: Try base label (before first colon)
- *   Step 2: If duplicate, add distinguishing part (after colon, before ----)
- *   Step 3: If still duplicate, add code suffix
+ *   Step 1: Clean the description (remove DGFT dates)
+ *   Step 2: Parse the description format (": --" or ":--" or ": ----")
+ *   Step 3: Try base label first
+ *   Step 4: Add qualifier if we have one and it's meaningful
+ *   Step 5: Handle "Other" or empty qualifiers with seed/other distinction
+ *   Step 6: Last resort - add meaningful code part
  */
-function generateUniqueLabel(code: string, description: string, existingLabels: Set<string>): string {
-  // Clean up the description - remove leading dashes and extra spaces
-  const cleanDesc = description.replace(/^[-:\s]+/, '').trim();
-
-  // Split by colon to separate main part from qualifier
-  const colonParts = cleanDesc.split(/\s*:\s*/);
-  const mainPart = (colonParts[0] || '').trim();
-
-  // Step 1: Try base label (just the part before colon)
-  let label = mainPart;
-
-  // Truncate if too long
-  if (label.length > 50) {
-    label = label.substring(0, 47) + '...';
+function generateUniqueLabel(
+  code: string,
+  description: string,
+  existingLabels: Set<string>
+): string {
+  // Step 1: Clean the description
+  let cleanDesc = description
+    .replace(/\s*\d{2}\/\d{4}-\d{2,4}\s+\d{2}\.\d{2}\.\d{4}\s*/g, '') // Remove DGFT dates
+    .trim();
+
+  // Step 2: Parse the description format
+  // Common formats:
+  // "Category : -- Qualifier" (e.g., "Durum wheat : -- Seed")
+  // "Category :-- Qualifier" (e.g., "Durum wheat :-- Seed")
+  // "Category: ---- Detail" (e.g., "Coffee, not roasted: ----Not decaffeinated")
+
+  let baseLabel = '';
+  let qualifier = '';
+
+  // Try to split by ": --" or ":--" or ": ----"
+  const colonDashMatch = cleanDesc.match(/^(.+?)\s*:\s*-{1,4}\s*(.+)$/);
+  if (colonDashMatch) {
+    baseLabel = colonDashMatch[1]!.trim();
+    qualifier = colonDashMatch[2]!.trim();
+  } else if (cleanDesc.includes(':')) {
+    // Simple colon split
+    const parts = cleanDesc.split(':');
+    baseLabel = parts[0]!.trim();
+    qualifier = parts.slice(1).join(':').replace(/^[\s-]+/, '').trim();
+  } else {
+    baseLabel = cleanDesc;
   }
 
-  if (!existingLabels.has(label)) {
-    existingLabels.add(label);
-    return label;
+  // Step 3: Try base label first (but NOT if it's a generic "Other" - those need qualifiers)
+  const isGenericBase = baseLabel.toLowerCase() === 'other';
+  if (!isGenericBase && !existingLabels.has(baseLabel) && baseLabel.length > 0) {
+    existingLabels.add(baseLabel);
+    return baseLabel;
   }
 
-  // Step 2: Add the distinguishing part (after colon, before ----)
-  // This handles: "Coffee, not roasted : --Not decaffeinated" → extracts "Not decaffeinated"
-  if (colonParts.length > 1) {
-    // Get qualifier from the part after colon
-    let qualifierPart = colonParts.slice(1).join(' : '); // Rejoin in case multiple colons
-
-    // Remove leading dashes (e.g., "--Not decaffeinated" → "Not decaffeinated")
-    let qualifier = qualifierPart.replace(/^[-\s]+/, '').trim();
-
-    // If there's a "----" separator, take the part before it
-    if (qualifier.includes('----')) {
-      qualifier = qualifier.split('----')[0]?.trim() || qualifier;
+  // Step 4: Add qualifier if we have one and it's meaningful
+  if (qualifier && qualifier.length > 0 && qualifier.toLowerCase() !== 'other') {
+    const labelWithQualifier = `${baseLabel} - ${qualifier}`;
+    if (!existingLabels.has(labelWithQualifier)) {
+      existingLabels.add(labelWithQualifier);
+      return labelWithQualifier;
     }
+  }
 
-    // Also handle "--" separator (common in HS descriptions)
-    if (qualifier.includes('--')) {
-      qualifier = qualifier.split('--')[0]?.trim() || qualifier;
-    }
-
-    // Clean up the qualifier
-    qualifier = qualifier.replace(/^[-\s]+/, '').trim();
-
-    if (qualifier && qualifier.toLowerCase() !== 'other') {
-      label = `${mainPart} - ${qualifier}`;
-
-      // Truncate if too long
-      if (label.length > 60) {
-        label = label.substring(0, 57) + '...';
+  // Step 5: If qualifier is "Other" or empty, try more specific label
+  if (qualifier.toLowerCase() === 'other' || !qualifier) {
+    // Check if this is a seed vs non-seed distinction
+    if (cleanDesc.toLowerCase().includes('seed')) {
+      const seedLabel = `${baseLabel} - Seed`;
+      if (!existingLabels.has(seedLabel)) {
+        existingLabels.add(seedLabel);
+        return seedLabel;
       }
-
-      if (!existingLabels.has(label)) {
-        existingLabels.add(label);
-        return label;
+    } else {
+      const otherLabel = `${baseLabel} - Other`;
+      if (!existingLabels.has(otherLabel)) {
+        existingLabels.add(otherLabel);
+        return otherLabel;
       }
     }
   }
 
-  // Step 3: Add code suffix as last resort
-  const codeParts = code.split('.');
-  const codeEnd = codeParts[codeParts.length - 1] || code;
-  label = `${mainPart} (${codeEnd})`;
+  // Step 6: Last resort - add meaningful code part
+  // Extract the last meaningful segment (e.g., "11" from "1001.11")
+  const codeParts = code.replace(/\.00$/, '').split('.');
+  const lastPart = codeParts[codeParts.length - 1]!;
+
+  // Try to make it meaningful
+  let suffix = '';
+  if (lastPart === '11' || lastPart === '91') {
+    suffix = 'Seed';
+  } else if (lastPart === '19' || lastPart === '99') {
+    suffix = 'Other';
+  } else if (lastPart === '00') {
+    suffix = 'General';
+  } else {
+    suffix = lastPart;
+  }
 
-  // Final truncation check
-  if (label.length > 60) {
-    label = label.substring(0, 57) + '...';
+  const finalLabel = `${baseLabel} - ${suffix}`;
+  if (!existingLabels.has(finalLabel)) {
+    existingLabels.add(finalLabel);
+    return finalLabel;
   }
 
-  existingLabels.add(label);
-  return label;
+  // Absolute last resort - add full code
+  existingLabels.add(`${baseLabel} (${code})`);
+  return `${baseLabel} (${code})`;
 }
 
 /**

frontend/src/app/classify/page.tsx

@@ -182,9 +182,19 @@ export default function ClassifyPage() {
       // Add AI question message with DATA (not JSX)
       setChatMessages(prev => {
         // Check if we already added this round's questions
-        const existingQuestion = prev.find(m =>
-          m.id.startsWith(`ai-questions-round${conversation.roundNumber}`)
-        )
+        const searchId = `ai-questions-round${conversation.roundNumber}`
+        const existingQuestion = prev.find(m => m.id.startsWith(searchId))
+
+        // Debug logging for conversation continuation
+        if (process.env.NODE_ENV === 'development') {
+          console.log('[Dedup Check]', {
+            roundNumber: conversation.roundNumber,
+            searchingFor: searchId,
+            found: !!existingQuestion,
+            existingIds: prev.filter(m => m.id.startsWith('ai-questions')).map(m => m.id)
+          })
+        }
+
         if (existingQuestion) return prev
 
         return [

frontend/src/lib/hooks/use-conversation.ts

@@ -1,6 +1,6 @@
 'use client'
 
-import { useState, useCallback } from 'react'
+import { useState, useCallback, useRef, useEffect } from 'react'
 import {
   classifyConversational,
   skipToClassification,
@@ -56,6 +56,14 @@ const initialState: ConversationState = {
 export function useConversation() {
   const [state, setState] = useState<ConversationState>(initialState)
 
+  // Use ref to prevent stale closure issues with conversationId
+  const conversationIdRef = useRef<string | null>(null)
+
+  // Keep the ref in sync with state
+  useEffect(() => {
+    conversationIdRef.current = state.conversationId
+  }, [state.conversationId])
+
   /**
    * Start a new classification conversation
    */
@@ -89,7 +97,7 @@ export function useConversation() {
           conversationId: response.conversationId,
           currentQuestions: response.questions || null,
           questionContext: response.questionContext || null,
-          roundNumber: response.roundNumber || 1,
+          roundNumber: response.roundNumber ?? 1,
           totalQuestionsAsked: response.totalQuestionsAsked || 0
         }))
       } else if (response.responseType === 'classification' && response.result) {
@@ -115,7 +123,9 @@ export function useConversation() {
    * Submit answers to current questions
    */
   const submitAnswers = useCallback(async (answers: Record<string, string>) => {
-    if (!state.conversationId || !state.currentQuestions) return
+    // Use ref for conversationId to prevent stale closure issues
+    const currentConversationId = conversationIdRef.current
+    if (!currentConversationId || !state.currentQuestions) return
 
     // Add Q&A to history
     const newHistoryItems: ConversationHistoryItem[] = []
@@ -144,9 +154,10 @@ export function useConversation() {
 
     try {
       const sessionId = getSessionId()
+      // Use ref value for conversationId to avoid stale closure
       const response = await classifyConversational({
         productDescription: state.productDescription,
-        conversationId: state.conversationId,
+        conversationId: currentConversationId,
         sessionId,
         answers
       })
@@ -166,7 +177,7 @@ export function useConversation() {
           status: 'asking',
           currentQuestions: response.questions || null,
           questionContext: response.questionContext || null,
-          roundNumber: response.roundNumber || prev.roundNumber + 1,
+          roundNumber: response.roundNumber ?? (prev.roundNumber + 1),
           totalQuestionsAsked: response.totalQuestionsAsked || prev.totalQuestionsAsked
         }))
       } else if (response.responseType === 'classification' && response.result) {
@@ -185,19 +196,21 @@ export function useConversation() {
         error: error instanceof Error ? error.message : 'Unknown error occurred'
       }))
     }
-  }, [state.conversationId, state.currentQuestions, state.productDescription])
+  }, [state.currentQuestions, state.productDescription]) // Removed conversationId - using ref instead
 
   /**
    * Skip remaining questions and get best guess
    */
   const skip = useCallback(async () => {
-    if (!state.conversationId) return
+    // Use ref for conversationId to prevent stale closure issues
+    const currentConversationId = conversationIdRef.current
+    if (!currentConversationId) return
 
     setState(prev => ({ ...prev, status: 'loading' }))
 
     try {
       const sessionId = getSessionId()
-      const response = await skipToClassification(state.conversationId, sessionId)
+      const response = await skipToClassification(currentConversationId, sessionId)
 
       if (!response.success) {
         setState(prev => ({
@@ -224,7 +237,7 @@ export function useConversation() {
         error: error instanceof Error ? error.message : 'Unknown error occurred'
       }))
     }
-  }, [state.conversationId])
+  }, []) // Removed conversationId - using ref instead
 
   /**
    * Reset and start over