fix: Handle repeat content across cards

debjitbis08 · debjitbis08 · commit 5d7ab9ba4e25 · 2026-04-03T17:11:23.000+05:30
diff --git a/apps/web/src/components/LatexText.tsx b/apps/web/src/components/LatexText.tsx
@@ -157,7 +157,9 @@ function renderMarkdown(html: string): string {
   html = html.replace(/\*\*([^*]+?)\*\*/g, "<strong>$1</strong>");
 
   // Italic: *text* → <em> (but not inside <strong> tags' asterisks)
-  html = html.replace(/(?<!\*)\*([^*]+?)\*(?!\*)/g, "<em>$1</em>");
+  // Require no space after opening * and no space before closing * (CommonMark flanking rules)
+  // so that bare multiplication like 1 * 2 * 3 is not treated as emphasis.
+  html = html.replace(/(?<!\*)\*(?! )([^*]+?)(?<! )\*(?!\*)/g, "<em>$1</em>");
 
   // Process block-level elements within paragraph breaks
   const blocks = html.split(/\n\n+/);
diff --git a/apps/web/src/lib/cards/generate.ts b/apps/web/src/lib/cards/generate.ts
@@ -2,7 +2,7 @@ import type { AIProvider, AIUsage, ImagePart } from '../ai/index.ts'
 import type { Document, Chunk, InsertCard } from '@scroll-reader/db'
 import type { CardType, CardStrategy, CardContent, DocumentType, ReadingGoal } from '@scroll-reader/shared-types'
 import { resolveCardStrategy } from '@scroll-reader/shared-types'
-import { buildSmartPrompt } from './prompts.ts'
+import { buildSmartPrompt, summarizeCard } from './prompts.ts'
 
 interface AICard {
   type: CardType
@@ -29,6 +29,7 @@ export async function generateCardsForChunk(
   provider: AIProvider,
   cardTypes?: CardType[],
   images?: { base64: string; mimeType: string; alt: string }[],
+  recentCardSummaries?: string[],
 ): Promise<CardGenResult> {
   const strategy: CardStrategy = cardTypes
     ? { cardTypes, chunkInterval: 1 }
@@ -43,7 +44,7 @@ export async function generateCardsForChunk(
     mimeType: img.mimeType,
   }))
 
-  const prompt = buildSmartPrompt(chunk, prevChunk, doc, strategy, imageAlts)
+  const prompt = buildSmartPrompt(chunk, prevChunk, doc, strategy, imageAlts, recentCardSummaries)
   const response = await provider.generate(prompt, imageParts)
   const aiCards = parseAIResponse(response.text, strategy)
 
diff --git a/apps/web/src/lib/cards/prompts.ts b/apps/web/src/lib/cards/prompts.ts
@@ -1,5 +1,33 @@
 import type { Document, Chunk } from "@scroll-reader/db";
-import type { CardType, CardStrategy } from "@scroll-reader/shared-types";
+import type { CardType, CardStrategy, CardContent } from "@scroll-reader/shared-types";
+
+/** Compact one-line summary of a card for dedup context (keeps prompt small). */
+export function summarizeCard(type: CardType, content: CardContent): string {
+  const c = content as Record<string, unknown>
+  switch (type) {
+    case 'discover':
+    case 'raw_commentary':
+    case 'connect':
+      return `${type}: ${truncate(String(c.title ?? c.body ?? ''), 80)}`
+    case 'flashcard':
+      return `flashcard: ${truncate(String(c.question ?? ''), 80)}`
+    case 'quiz':
+      return `quiz: ${truncate(String(c.question ?? ''), 80)}`
+    case 'glossary':
+      return `glossary: ${truncate(String(c.term ?? ''), 40)}`
+    case 'contrast':
+      return `contrast: ${truncate(String(c.itemA ?? ''), 30)} vs ${truncate(String(c.itemB ?? ''), 30)}`
+    case 'passage':
+      return `passage: ${truncate(String(c.excerpt ?? ''), 80)}`
+    default:
+      return `${type}: (card)`
+  }
+}
+
+function truncate(s: string, max: number): string {
+  const oneLine = s.replace(/\n/g, ' ').trim()
+  return oneLine.length <= max ? oneLine : oneLine.slice(0, max) + '…'
+}
 
 const CARD_TYPE_DESCRIPTIONS: Record<CardType, string> = {
   discover:
@@ -30,6 +58,7 @@ export function buildSmartPrompt(
   doc: Document,
   strategy?: CardStrategy | null,
   imageAlts?: string[],
+  recentCardSummaries?: string[],
 ): string {
   const docLabel = doc.title ?? "Untitled";
 
@@ -76,8 +105,10 @@ You are a reading companion AI. Analyze the ${isCodeChunk ? "code sample" : "pas
 
 SUGGESTED CARD TYPES (you may adjust based on the content):
 ${typeDescriptions}
-${codeInstructions}${imageBlock}
-INSTRUCTIONS:
+${codeInstructions}${imageBlock}${recentCardSummaries && recentCardSummaries.length > 0 ? `ALREADY GENERATED (do NOT repeat these topics — find a fresh angle or skip if the passage covers the same ground):
+${recentCardSummaries.map((s) => `  - ${s}`).join("\n")}
+
+` : ""}INSTRUCTIONS:
 1. First, understand what kind of content this is (prose, reference table, notation, formula, code sample, exercises/questions, table of contents, etc.).
 2. If the content TEACHES something (explains a concept, presents an argument, demonstrates a technique), generate a "discover" card first when it is in the suggested types — it is the primary card type. Then add other types only if the content warrants them.
 3. If the content is primarily exercises, homework questions, discussion prompts, review questions, or a table of contents/index — do NOT generate a "discover" card. These are questions, not teachings. Instead, generate flashcard or quiz cards that ANSWER the most important questions if possible, or return an empty array if the questions are too open-ended or require external work.
diff --git a/apps/web/src/lib/pipeline.ts b/apps/web/src/lib/pipeline.ts
@@ -13,7 +13,8 @@ import {
 import type { ImageElement, TocEntry, ExtractConfig, ChunkerConfig, AIUsage, PipelineChunk, TocSection } from '@scroll-reader/pipeline'
 import { createProvider } from './ai/index.ts'
 import { generateCardsForChunk } from './cards/generate.ts'
-import type { CardType, DocumentType, ReadingGoal, Tier } from '@scroll-reader/shared-types'
+import { summarizeCard } from './cards/prompts.ts'
+import type { CardType, CardContent, DocumentType, ReadingGoal, Tier } from '@scroll-reader/shared-types'
 import { TIER_LIMITS, resolveCardStrategy } from '@scroll-reader/shared-types'
 import { BATCH_SIZE, EXTRACTOR_BIN, CHUNKER_BIN, FIGURE_EXTRACT_BIN } from 'astro:env/server'
 import { downloadDocument, deleteDocument, uploadImage } from './storage.ts'
@@ -676,6 +677,9 @@ export async function processDocument(doc: Document, cardBudget: number): Promis
       }
     }
 
+    const recentCardSummaries: string[] = []
+    const MAX_RECENT_SUMMARIES = 15
+
     for (let i = 0; i < chunksNeedingCards.length && budgetLeft > 0; i += BATCH_SIZE) {
       const batch = chunksNeedingCards.slice(i, i + BATCH_SIZE)
 
@@ -724,7 +728,7 @@ export async function processDocument(doc: Document, cardBudget: number): Promis
         const chunkCardTypes = existingTypesForChunk
           ? (cardTypes as CardType[]).filter((t) => !existingTypesForChunk.has(t))
           : cardTypes
-        const { cards: newCards, usage: cardUsage } = await generateCardsForChunk(chunk, prevChunk, doc as Document, provider, chunkCardTypes, images)
+        const { cards: newCards, usage: cardUsage } = await generateCardsForChunk(chunk, prevChunk, doc as Document, provider, chunkCardTypes, images, recentCardSummaries.length > 0 ? recentCardSummaries : undefined)
         attempted.add(chunk.id)
 
         if (cardUsage) {
@@ -736,6 +740,14 @@ export async function processDocument(doc: Document, cardBudget: number): Promis
           cardsGenerated += newCards.length
           budgetLeft -= newCards.length
 
+          // Accumulate summaries for dedup context (rolling window)
+          for (const card of newCards) {
+            recentCardSummaries.push(summarizeCard(card.cardType as CardType, card.content as CardContent))
+          }
+          if (recentCardSummaries.length > MAX_RECENT_SUMMARIES) {
+            recentCardSummaries.splice(0, recentCardSummaries.length - MAX_RECENT_SUMMARIES)
+          }
+
           // Write back to catalog_cards cache for future users
           if (chunkToCatalogChunk) {
             const catalogChunkId = chunkToCatalogChunk.get(chunk.id)
diff --git a/apps/worker/src/cards/generate.ts b/apps/worker/src/cards/generate.ts
@@ -2,7 +2,7 @@ import type { AIProvider, AIUsage } from '../ai/index.ts'
 import type { Document, Chunk, InsertCard } from '@scroll-reader/db'
 import type { CardType, CardStrategy, CardContent, DocumentType, ReadingGoal } from '@scroll-reader/shared-types'
 import { resolveCardStrategy } from '@scroll-reader/shared-types'
-import { buildSmartPrompt } from './prompts.ts'
+import { buildSmartPrompt, summarizeCard } from './prompts.ts'
 
 interface AICard {
   type: CardType
@@ -28,6 +28,7 @@ export async function generateCardsForChunk(
   doc: Document,
   provider: AIProvider,
   cardTypes?: CardType[],
+  recentCardSummaries?: string[],
 ): Promise<CardGenResult> {
   const strategy: CardStrategy = cardTypes
     ? { cardTypes, chunkInterval: 1 }
@@ -36,7 +37,7 @@ export async function generateCardsForChunk(
         (doc.readingGoal ?? 'reflective') as ReadingGoal,
       )
 
-  const prompt = buildSmartPrompt(chunk, prevChunk, doc, strategy)
+  const prompt = buildSmartPrompt(chunk, prevChunk, doc, strategy, recentCardSummaries)
   const response = await provider.generate(prompt)
   const aiCards = parseAIResponse(response.text, strategy)
 
diff --git a/apps/worker/src/cards/prompts.ts b/apps/worker/src/cards/prompts.ts
@@ -1,5 +1,33 @@
 import type { Document, Chunk } from "@scroll-reader/db";
-import type { CardType, CardStrategy } from "@scroll-reader/shared-types";
+import type { CardType, CardStrategy, CardContent } from "@scroll-reader/shared-types";
+
+/** Compact one-line summary of a card for dedup context (keeps prompt small). */
+export function summarizeCard(type: CardType, content: CardContent): string {
+  const c = content as Record<string, unknown>
+  switch (type) {
+    case 'discover':
+    case 'raw_commentary':
+    case 'connect':
+      return `${type}: ${truncate(String(c.title ?? c.body ?? ''), 80)}`
+    case 'flashcard':
+      return `flashcard: ${truncate(String(c.question ?? ''), 80)}`
+    case 'quiz':
+      return `quiz: ${truncate(String(c.question ?? ''), 80)}`
+    case 'glossary':
+      return `glossary: ${truncate(String(c.term ?? ''), 40)}`
+    case 'contrast':
+      return `contrast: ${truncate(String(c.itemA ?? ''), 30)} vs ${truncate(String(c.itemB ?? ''), 30)}`
+    case 'passage':
+      return `passage: ${truncate(String(c.excerpt ?? ''), 80)}`
+    default:
+      return `${type}: (card)`
+  }
+}
+
+function truncate(s: string, max: number): string {
+  const oneLine = s.replace(/\n/g, ' ').trim()
+  return oneLine.length <= max ? oneLine : oneLine.slice(0, max) + '…'
+}
 
 const CARD_TYPE_DESCRIPTIONS: Record<CardType, string> = {
   discover:
@@ -29,6 +57,7 @@ export function buildSmartPrompt(
   prevChunk: Chunk | null,
   doc: Document,
   strategy?: CardStrategy | null,
+  recentCardSummaries?: string[],
 ): string {
   const docLabel = doc.title ?? "Untitled";
 
@@ -68,8 +97,10 @@ You are a reading companion AI. Analyze the ${isCodeChunk ? "code sample" : "pas
 
 SUGGESTED CARD TYPES (you may adjust based on the content):
 ${typeDescriptions}
-${codeInstructions}
-INSTRUCTIONS:
+${codeInstructions}${recentCardSummaries && recentCardSummaries.length > 0 ? `ALREADY GENERATED (do NOT repeat these topics — find a fresh angle or skip if the passage covers the same ground):
+${recentCardSummaries.map((s) => `  - ${s}`).join("\n")}
+
+` : ""}INSTRUCTIONS:
 1. First, understand what kind of content this is (prose, reference table, notation, formula, code sample, exercises/questions, table of contents, etc.).
 2. If the content TEACHES something (explains a concept, presents an argument, demonstrates a technique), generate a "discover" card first when it is in the suggested types — it is the primary card type. Then add other types only if the content warrants them.
 3. If the content is primarily exercises, homework questions, discussion prompts, review questions, or a table of contents/index — do NOT generate a "discover" card. These are questions, not teachings. Instead, generate flashcard or quiz cards that ANSWER the most important questions if possible, or return an empty array if the questions are too open-ended or require external work.
diff --git a/apps/worker/src/pipeline.ts b/apps/worker/src/pipeline.ts
@@ -5,12 +5,14 @@ import { db } from './db.ts'
 import { documents, chunks, cards, aiUsageLogs } from '@scroll-reader/db'
 import type { Document, Chunk } from '@scroll-reader/db'
 import type { AIProvider, AIUsage } from './ai/index.ts'
+import type { CardType, CardContent } from '@scroll-reader/shared-types'
 import {
   extractDocument, callSegmenter, callChunker, aiChunk,
   mergeConsecutiveCode, foldSmallCodeIntoText,
 } from '@scroll-reader/pipeline'
 import type { ExtractConfig, ChunkerConfig, PipelineChunk } from '@scroll-reader/pipeline'
 import { generateCardsForChunk } from './cards/generate.ts'
+import { summarizeCard } from './cards/prompts.ts'
 
 const HERE = dirname(fileURLToPath(import.meta.url))
 const extractConfig: ExtractConfig = {
@@ -224,18 +226,28 @@ export async function processDocument(filePath: string, userId: string): Promise
     // Generate cards for text and code chunks (skip image-only chunks)
     const cardChunkRows = insertedChunks.filter((c) => c.chunkType === 'text' || c.chunkType === 'code')
     let totalCards = 0
+    const recentCardSummaries: string[] = []
+    const MAX_RECENT_SUMMARIES = 15
 
     for (let i = 0; i < cardChunkRows.length; i++) {
       const chunk = cardChunkRows[i]
       // Fetch chunk N-1 regardless of type — image chunks provide alt text context
       const prevChunk: Chunk | null = insertedChunks[insertedChunks.indexOf(chunk) - 1] ?? null
 
-      const { cards: newCards, usage: cardUsage } = await generateCardsForChunk(chunk, prevChunk, doc as Document, provider)
+      const { cards: newCards, usage: cardUsage } = await generateCardsForChunk(chunk, prevChunk, doc as Document, provider, undefined, recentCardSummaries.length > 0 ? recentCardSummaries : undefined)
       if (cardUsage) {
         logUsage(userId, doc.id, 'card_generation', provider.name, provider.model, cardUsage, chunk.id)
       }
       await db.insert(cards).values(newCards)
       totalCards += newCards.length
+
+      // Accumulate summaries for dedup context (rolling window)
+      for (const card of newCards) {
+        recentCardSummaries.push(summarizeCard(card.cardType as CardType, card.content as CardContent))
+      }
+      if (recentCardSummaries.length > MAX_RECENT_SUMMARIES) {
+        recentCardSummaries.splice(0, recentCardSummaries.length - MAX_RECENT_SUMMARIES)
+      }
     }
 
     // --- Done ---
