Don't flag same-title books in different languages as duplicates

app/src/Book.jsx

@@ -251,10 +251,19 @@ function Clippings({ session, book, revision = 0 }) {
   )
 }
 
-const norm = (s) => (s ?? '').toLowerCase().replace(/[^a-z0-9]+/g, ' ').trim()
-// Same normalized title (and author when both have one): probably the same book synced twice.
+// Letters, marks and digits of every script: an ASCII-only pattern reduces Japanese titles to their
+// digits, and dropping marks merges Thai or Hindi words that differ only in a tone or vowel sign.
+const norm = (s) => (s ?? '').normalize('NFKC').toLowerCase().replace(/[^\p{L}\p{M}\p{N}]+/gu, ' ').trim()
+// Language only by its main subtag ("en" and "en-GB" match). Unknown on either side counts as a match.
+const sameLanguage = (a, b) => !a || !b || a.split('-')[0].toLowerCase() === b.split('-')[0].toLowerCase()
+// Same normalized title (and author and language when both have one): probably the same book synced
+// twice. Editions in two languages can share a title, e.g. "1984" in English and in German.
 export const looksLikeSame = (a, b) =>
-  a.document !== b.document && norm(a.title) && norm(a.title) === norm(b.title) && (!a.author || !b.author || norm(a.author) === norm(b.author))
+  a.document !== b.document &&
+  norm(a.title) &&
+  norm(a.title) === norm(b.title) &&
+  (!a.author || !b.author || norm(a.author) === norm(b.author)) &&
+  sameLanguage(a.language, b.language)
 
 const field =
   'h-11 w-full rounded-xl bg-surface px-3 text-base text-stone-900 ring-1 ring-stone-950/10 outline-none placeholder:text-stone-400 focus:ring-2 focus:ring-brand-500/60'

app/src/Library.test.jsx

@@ -46,3 +46,40 @@ describe('Library completion updates', () => {
     expect(queryByText('Continue reading')).toBeNull()
   })
 })
+
+describe('duplicate notice', () => {
+  const book = (document, title) => ({ document, title, author: 'Unknown Author', status: 'finished', percentage: 1 })
+
+  it('tells different Japanese titles apart', () => {
+    const books = [
+      book('grammar', '小学1年生 言葉と文法にぐーんと強くなる (くもんの国語集中学習)'),
+      book('kanji', '小学1年生 漢字にぐーんと強くなる (くもんの国語集中学習)'),
+    ]
+    const { container } = render(<Library session={session} books={books} />)
+    expect(container.textContent).not.toContain('appears more than once')
+  })
+
+  it('tells Thai titles apart that differ only in a tone mark', () => {
+    const books = [book('forest', 'ป่า'), book('aunt', 'ป้า')]
+    const { container } = render(<Library session={session} books={books} />)
+    expect(container.textContent).not.toContain('appears more than once')
+  })
+
+  it('tells the same title apart when the languages differ', () => {
+    const books = [{ ...book('en', '1984'), language: 'en' }, { ...book('de', '1984'), language: 'de' }]
+    const { container } = render(<Library session={session} books={books} />)
+    expect(container.textContent).not.toContain('appears more than once')
+  })
+
+  it('treats a regional variant as the same language', () => {
+    const books = [{ ...book('a', '1984'), language: 'en' }, { ...book('b', '1984'), language: 'en-GB' }]
+    const { container } = render(<Library session={session} books={books} />)
+    expect(container.textContent).toContain('appears more than once')
+  })
+
+  it('still finds a Japanese title synced twice, full-width forms included', () => {
+    const books = [book('a', '変な家２　～11の間取り図～'), book('b', '変な家2 ~11の間取り図~')]
+    const { container } = render(<Library session={session} books={books} />)
+    expect(container.textContent).toContain('appears more than once')
+  })
+})