feat: store a book's language

docs/API.md

@@ -94,6 +94,7 @@ Validates credentials. `200 {"authorized": "OK"}` or `401`.
     "filename": "Foundryside - Robert Jackson Bennett.epub",
     "title": "Foundryside",
     "authors": "Robert Jackson Bennett",
+    "language": "en",
     "bookfusion_id": "36835"
   }
   ```
@@ -102,6 +103,10 @@ Validates credentials. `200 {"authorized": "OK"}` or `401`.
   `(user, document)`; fields the client omits never overwrite previously stored values. Retrieve
   via `GET /api/v1/documents` or joined into `GET /api/v1/progress`.
 
+  **Language.** `language` is the EPUB's `dc:language` as a BCP 47 tag (`"ja"`, `"de-DE"`). It
+  only fills a blank, so a language set by hand through `PUT /api/v1/documents/{document}/info`
+  stays. A value that isn't a language tag is ignored; the sync itself still succeeds.
+
   **Service ids.** Any extra `<service>_id` field (from a CrossPoint plugin's book sidecar,
   e.g. `bookfusion_id`) pre-seeds an exact match for the connector of that name: progress is
   pushed straight to that record, skipping title/author search. Unknown-connector ids are
@@ -625,7 +630,9 @@ the server has `GOOGLE_BOOKS_API_KEY`, then Amazon's "Print length" via SearchAP
 
 Manual fixes when the lookup got it wrong: `{"cover_url": "https://...", "page_count": 433}` (either
 field optional). Manual values stick, since lookups only fill blanks; `null` clears a field so it's
-looked up again. Returns the stored `{document, cover_url, page_count}`.
+looked up again. `language` (a BCP 47 tag) can be set the same way and then stays over device
+syncs; `null` clears it so the next device sync fills it again. Returns the stored
+`{document, cover_url, page_count, title, author, language}`.
 
 #### GET /api/v1/documents/{document}/cover/candidates?q=
 

migrations/0034_document_language.sql

@@ -0,0 +1,2 @@
+-- A book's language (BCP 47 tag, e.g. "ja" or "de-DE"), from the reader's EPUB metadata or set by hand.
+ALTER TABLE documents ADD COLUMN language TEXT;

src/models/merge.ts

@@ -148,8 +148,8 @@ export function mergeDocuments(
 
     // documents: canonical metadata wins, alias fills the gaps.
     db.prepare(
-      `INSERT INTO documents (user_id, document, title, author, filename, filesize, status, status_at, pause_reason, start_date, finished_date, cover_url, page_count, updated_at)
-       SELECT user_id, ?, title, author, filename, filesize, status, status_at, pause_reason, start_date, finished_date, cover_url, page_count, ? FROM documents WHERE user_id = ? AND document = ?
+      `INSERT INTO documents (user_id, document, title, author, filename, filesize, status, status_at, pause_reason, start_date, finished_date, cover_url, page_count, language, updated_at)
+       SELECT user_id, ?, title, author, filename, filesize, status, status_at, pause_reason, start_date, finished_date, cover_url, page_count, language, ? FROM documents WHERE user_id = ? AND document = ?
        ON CONFLICT(user_id, document) DO UPDATE SET
          title = COALESCE(documents.title, excluded.title),
          author = COALESCE(documents.author, excluded.author),
@@ -162,6 +162,7 @@ export function mergeDocuments(
          finished_date = COALESCE(documents.finished_date, excluded.finished_date),
          cover_url = COALESCE(documents.cover_url, excluded.cover_url),
          page_count = COALESCE(documents.page_count, excluded.page_count),
+         language = COALESCE(documents.language, excluded.language),
          updated_at = excluded.updated_at`
     ).run(into, now, userId, from);
     db.prepare('DELETE FROM documents WHERE user_id = ? AND document = ?').run(userId, from);

src/routes/kosync.ts

@@ -43,6 +43,8 @@ export interface DocumentMetadata {
   filename: string | null;
   title: string | null;
   authors: string | null;
+  /** BCP 47 tag from the EPUB (`dc:language`), e.g. "ja"; null when absent or malformed. */
+  language: string | null;
   /**
    * Service book ids from the CrossPoint plugin sidecar ("<book>.meta.json"),
    * keyed by service name. A plugin that downloads a book records e.g.
@@ -55,7 +57,13 @@ export interface DocumentMetadata {
 
 // Sidecar convention: a flat `<service>_id` field names the connector
 // ("bookfusion_id" -> connector "bookfusion"). Reserved keys are not ids.
-const RESERVED_META_KEYS = new Set(['filename', 'title', 'authors', 'source']);
+const RESERVED_META_KEYS = new Set(['filename', 'title', 'authors', 'language', 'source']);
+
+/** A BCP 47 language tag ("en", "ja", "de-DE", "zh-Hant"), trimmed; null if it doesn't look like one. */
+export function languageTag(v: unknown): string | null {
+  const tag = typeof v === 'string' ? v.trim() : '';
+  return /^[a-z]{2,3}(-[a-z0-9]{2,8}){0,3}$/i.test(tag) ? tag : null;
+}
 
 function parseMetadata(raw: unknown): DocumentMetadata | null {
   if (typeof raw !== 'object' || raw === null) return null;
@@ -72,10 +80,11 @@ function parseMetadata(raw: unknown): DocumentMetadata | null {
     filename: str(o.filename),
     title: str(o.title),
     authors: str(o.authors),
+    language: languageTag(o.language),
     externalIds,
   };
   const hasId = Object.keys(externalIds).length > 0;
-  return meta.filename || meta.title || meta.authors || hasId ? meta : null;
+  return meta.filename || meta.title || meta.authors || meta.language || hasId ? meta : null;
 }
 
 /** Stores progress-PUT metadata without clobbering fields the client didn't send. */
@@ -87,14 +96,16 @@ export function upsertDocumentMetadata(
   updatedAt: number
 ): void {
   db.prepare(
-    `INSERT INTO documents (user_id, document, title, author, filename, updated_at)
-     VALUES (?, ?, ?, ?, ?, ?)
+    `INSERT INTO documents (user_id, document, title, author, filename, language, updated_at)
+     VALUES (?, ?, ?, ?, ?, ?, ?)
      ON CONFLICT(user_id, document) DO UPDATE SET
        title = CASE WHEN documents.meta_manual THEN documents.title ELSE COALESCE(excluded.title, documents.title) END,
        author = CASE WHEN documents.meta_manual THEN documents.author ELSE COALESCE(excluded.author, documents.author) END,
        filename = COALESCE(excluded.filename, documents.filename),
+       -- Only fills a blank: an EPUB's language doesn't change, and one set by hand stays.
+       language = COALESCE(documents.language, excluded.language),
        updated_at = excluded.updated_at`
-  ).run(userId, document, meta.title, meta.authors, meta.filename, updatedAt);
+  ).run(userId, document, meta.title, meta.authors, meta.filename, meta.language, updatedAt);
 }
 
 /**

src/routes/v1/documents.ts

@@ -1,7 +1,7 @@
 import { Hono } from 'hono';
 import { withTransaction, type DB } from '../../db/db.js';
 import { kosyncError, type AppEnv } from '../../auth/middleware.js';
-import { isValidDocument } from '../kosync.js';
+import { isValidDocument, languageTag } from '../kosync.js';
 import { nowSeconds } from '../../models/sync.js';
 import { mergeDocuments, resolveDocument, unmergeDocument } from '../../models/merge.js';
 import { coverCandidates, documentInfo } from '../../models/cover.js';
@@ -271,9 +271,10 @@ export function documentRoutes(db: DB, http?: HttpTransport): Hono<AppEnv> {
     return c.json({ items: await coverCandidates(http ?? fetchTransport, title, q ? '' : (meta?.author ?? '')) });
   });
 
-  // Manual cover / print page count / title and author when lookups got it wrong.
+  // Manual cover / print page count / title and author / language when lookups got it wrong.
   // Manual values stick (lookups only fill blanks, and device syncs don't replace an
-  // edited title); null clears cover or page count so it's looked up again.
+  // edited title or language); null clears cover or page count so it's looked up again,
+  // and language so the next device sync fills it.
   app.put('/documents/:document/info', async (c) => {
     const param = c.req.param('document');
     let body: unknown;
@@ -288,12 +289,13 @@ export function documentRoutes(db: DB, http?: HttpTransport): Hono<AppEnv> {
     const text = (v: unknown) => (typeof v === 'string' ? v.trim().slice(0, 500) : v);
     const title = text(o.title);
     const author = text(o.author);
+    const language = o.language === undefined || o.language === null ? o.language : languageTag(o.language);
     // A title is required once edited; an empty author means "no author".
     const titleOk = title === undefined || (typeof title === 'string' && title.length > 0);
     const authorOk = author === undefined || typeof author === 'string';
     const coverOk = cover === undefined || cover === null || (typeof cover === 'string' && /^https?:\/\/\S{1,2000}$/.test(cover));
     const pagesOk = pages === undefined || pages === null || (Number.isInteger(pages) && (pages as number) > 0 && (pages as number) <= 100000);
-    if (!isValidDocument(param) || !coverOk || !pagesOk || !titleOk || !authorOk) {
+    if (!isValidDocument(param) || !coverOk || !pagesOk || !titleOk || !authorOk || (language === null && o.language !== null)) {
       return kosyncError(c, 403, 2003, 'Invalid request');
     }
     const user = c.get('user');
@@ -306,6 +308,9 @@ export function documentRoutes(db: DB, http?: HttpTransport): Hono<AppEnv> {
     if (pages !== undefined) {
       db.prepare('UPDATE documents SET page_count = ?, cover_checked_at = CASE WHEN ? IS NULL THEN NULL ELSE cover_checked_at END WHERE user_id = ? AND document = ?').run(pages as number | null, pages as number | null, user.id, document);
     }
+    if (language !== undefined) {
+      db.prepare('UPDATE documents SET language = ? WHERE user_id = ? AND document = ?').run(language as string | null, user.id, document);
+    }
     if (title !== undefined || author !== undefined) {
       // New title/author: everything found with the old one is looked up again
       // (blank cover and page count, Hardcover details, automatic service matches).
@@ -317,17 +322,17 @@ export function documentRoutes(db: DB, http?: HttpTransport): Hono<AppEnv> {
       ).run((title as string | undefined) ?? null, author !== undefined ? 1 : 0, (author as string | undefined) ?? '', user.id, document);
       db.prepare("DELETE FROM connector_matches WHERE user_id = ? AND document = ? AND source IN ('auto', 'none')").run(user.id, document);
     }
-    const row = db.prepare('SELECT cover_url, page_count, title, author FROM documents WHERE user_id = ? AND document = ?').get(user.id, document) as {
-      cover_url: string | null; page_count: number | null; title: string | null; author: string | null;
+    const row = db.prepare('SELECT cover_url, page_count, title, author, language FROM documents WHERE user_id = ? AND document = ?').get(user.id, document) as {
+      cover_url: string | null; page_count: number | null; title: string | null; author: string | null; language: string | null;
     };
-    return c.json({ document, cover_url: row.cover_url, page_count: row.page_count, title: row.title, author: row.author });
+    return c.json({ document, cover_url: row.cover_url, page_count: row.page_count, title: row.title, author: row.author, language: row.language });
   });
 
   app.get('/documents', (c) => {
     const user = c.get('user');
     const rows = db
       .prepare(
-        'SELECT document, title, author, filename, filesize, status, status_at, pause_reason, cover_url, page_count, updated_at FROM documents WHERE user_id = ? ORDER BY updated_at DESC LIMIT 500'
+        'SELECT document, title, author, filename, filesize, language, status, status_at, pause_reason, cover_url, page_count, updated_at FROM documents WHERE user_id = ? ORDER BY updated_at DESC LIMIT 500'
       )
       .all(user.id);
     return c.json({ items: rows });

src/routes/v1/progress.ts

@@ -90,7 +90,7 @@ export function progressRoutes(db: DB, refreshProgress: ProgressRefresh = async
     const rows = db
       .prepare(
         `SELECT p.document, p.device_id, p.device, p.percentage, p.progress, p.position, p.updated_at,
-                d.title, d.author, d.filename, d.cover_url, d.page_count,
+                d.title, d.author, d.filename, d.language, d.cover_url, d.page_count,
                 d.hc_slug, d.moods, d.genres, d.content_warnings, d.rating, d.series, d.series_position, d.release_year,
                 COALESCE(d.status, CASE WHEN p.percentage >= 0.98 OR ${completedBookStatsSql('p.user_id', 'p.document')}
                   THEN 'finished' ELSE 'reading' END) AS status,
@@ -121,6 +121,7 @@ export function progressRoutes(db: DB, refreshProgress: ProgressRefresh = async
       title: string | null;
       author: string | null;
       filename: string | null;
+      language: string | null;
       cover_url: string | null;
       page_count: number | null;
       hc_slug: string | null;
@@ -158,6 +159,7 @@ export function progressRoutes(db: DB, refreshProgress: ProgressRefresh = async
           title: r.title,
           author: r.author,
           filename: r.filename,
+          language: r.language,
           status: r.status,
           // When a manual status was set (null when derived from progress).
           status_at: r.status_at ?? null,

test/document-language.test.ts

@@ -0,0 +1,51 @@
+import { describe, expect, it } from 'vitest';
+import { DOC, makeTestApp, registerUser } from './helpers.js';
+
+const OTHER = 'ffffffffffffffffffffffffffffffff';
+
+describe('book language', () => {
+  it('comes from device metadata, sticks once set by hand, and survives a merge', async () => {
+    const { app } = makeTestApp();
+    const { headers } = await registerUser(app);
+    const sync = (document: string, language?: string) =>
+      app.request('/syncs/progress', {
+        method: 'PUT',
+        headers,
+        body: JSON.stringify({ document, progress: '/body/DocFragment[2]', percentage: 0.2, device: 'X4', device_id: 'x4',
+          metadata: { title: '変な家２', authors: '雨穴', ...(language === undefined ? {} : { language }) } }),
+      });
+    const put = (body: unknown) => app.request(`/api/v1/documents/${DOC}/info`, { method: 'PUT', headers, body: JSON.stringify(body) });
+    const language = async (document = DOC) => {
+      const items = ((await (await app.request('/api/v1/progress', { headers })).json()) as { items: { document: string; language: string | null }[] }).items;
+      return items.find((i) => i.document === document)?.language;
+    };
+
+    // A malformed tag is dropped without failing the sync.
+    expect((await sync(DOC, 'Japanese!')).status).toBe(200);
+    expect(await language()).toBeNull();
+
+    await sync(DOC, ' ja ');
+    expect(await language()).toBe('ja');
+    const docs = (await (await app.request('/api/v1/documents', { headers })).json()) as { items: { language: string }[] };
+    expect(docs.items[0].language).toBe('ja');
+
+    // Set by hand, it stays over later device syncs.
+    expect((await put({ language: 'not a tag' })).status).toBe(403);
+    expect(await (await put({ language: 'ja-JP' })).json()).toMatchObject({ language: 'ja-JP' });
+    await sync(DOC, 'en');
+    expect(await language()).toBe('ja-JP');
+
+    // Cleared, the next device sync fills it again.
+    await put({ language: null });
+    await sync(DOC, 'ja');
+    expect(await language()).toBe('ja');
+
+    // A merge fills a blank language from the merged-away book.
+    await sync(OTHER);
+    await put({ language: null });
+    await app.request(`/api/v1/documents/${OTHER}/info`, { method: 'PUT', headers, body: JSON.stringify({ language: 'ja' }) });
+    const merge = await app.request('/api/v1/documents/merge', { method: 'POST', headers, body: JSON.stringify({ document: OTHER, into: DOC }) });
+    expect(merge.status).toBe(200);
+    expect(await language()).toBe('ja');
+  });
+});