From 8dad363dfd33843ca1c5999efe55f7ec29005f4f Mon Sep 17 00:00:00 2001 From: Eszter Date: Sun, 25 Oct 2026 10:23:18 +0100 Subject: [PATCH 1/5] feat: per-book reading days, upload and download Clients can add a daily list to each book in PUT /stats/books, stored per device like the global daily totals. GET /stats/books/{document} returns each device's days and their sum, and the new GET /stats/books lists every book's stats a page at a time with its metadata. Filtered overviews use the days, and imported BookOrbit time now counts there too. --- docs/API.md | 30 ++++++- migrations/0035_book_days.sql | 11 +++ src/models/document.ts | 17 ++-- src/routes/account.ts | 1 + src/routes/v1/stats.ts | 155 +++++++++++++++++++++++++++++++--- test/book-days.test.ts | 82 ++++++++++++++++++ 6 files changed, 271 insertions(+), 25 deletions(-) create mode 100644 migrations/0035_book_days.sql create mode 100644 test/book-days.test.ts diff --git a/docs/API.md b/docs/API.md index 8c293f5..d44f0fd 100644 --- a/docs/API.md +++ b/docs/API.md @@ -562,10 +562,15 @@ Batch (max 20) of per-book `stats_v5` snapshots for one device: "avg_fwd": 12, "pace_n": 250, "eta": 5400, "start_manual": false, "finish_manual": false, "start_date": 1751000000, "finished_date": 0, - "tod": [0, 3000, 4000, 1400], "dow": [0, 0, 1200, 0, 2000, 3000, 2200]} + "tod": [0, 3000, 4000, 1400], "dow": [0, 0, 1200, 0, 2000, 3000, 2200], + "daily": [{"date": "2026-10-23", "seconds": 900}]} ]} ``` +`daily` is optional: the book's seconds per local day, up to 20 per item, in the same shape as the +global upload's. A day sent again keeps the larger value, so a client can resend today as it grows. +The answer counts them as `accepted_daily`. + #### GET /api/v1/stats/summary Server-side aggregate across all of the user's devices: @@ -609,8 +614,9 @@ Filters narrow the digest to matching books: `language` (main subtag, so `en` al `genre` (one of the book's genres) and `author` (part of the name). All ignore case and can be combined. A filtered digest counts only per-book data: the books' sync and session days, their finished counts and pages, and their measured time (per-book stats and reading sessions). Device-wide totals with no book -attached are left out. Per-book stats have no days, so `today_seconds` and `week_seconds` count only -reading sessions there. +attached are left out. Reading days and time per day come from the books' uploaded days, reading +sessions and imported time; where a book's sessions and its devices' days measure the same day, the +larger one counts. `group_by=language` or `group_by=genre` adds `groups`, one digest per value, for the 20 values with the most reading days. A book counts in each of its genres. The top-level fields stay the account's (or the @@ -628,7 +634,23 @@ for clients that render per-device numbers like the P2P screen does. #### GET /api/v1/stats/books/{document} Per-device rows plus a `combined` object (scalars/buckets summed, `completed` OR-ed, `avg_fwd` -weighted by `pace_n`, `start_date` = earliest non-zero, `finished_date` = latest). +weighted by `pace_n`, `start_date` = earliest non-zero, `finished_date` = latest). Each row's +`stats.daily` holds that device's days for the book; `combined.daily` sums every device per date. A +device restoring its own history reads its own row, so another device's time is never folded in. + +#### GET /api/v1/stats/books?device_id=&limit=&cursor= + +Every book's stats, one item per device and book, ordered by document. Each item carries the book's +`title`, `author`, `filename` and `language`, so a client can match it to a file without knowing the +hash, plus `stats` with `daily` as above. `device_id` keeps one device's items. `limit` defaults to 50 +(at most 200); pass the answer's `next` as `cursor` for the following page, until it is `null`. + +```json +{"items": [{"device_id": "A1B2C3D4", "document": "a1b2...", "title": "Kokoro", "author": null, + "filename": "kokoro.epub", "language": "ja", "updated_at": 1761300000, + "stats": {"v": 5, "sessions": 9, "...": "...", "daily": [{"date": "2026-10-23", "seconds": 900}]}}], + "next": "a1b2...A1B2C3D4"} +``` Merged documents (`POST /api/v1/documents/merge`, `{"document", "into", "stats"?}`): stats rows stay under the hash each device uploaded them with and are combined on read. With `"stats": true` diff --git a/migrations/0035_book_days.sql b/migrations/0035_book_days.sql new file mode 100644 index 0000000..e9a839f --- /dev/null +++ b/migrations/0035_book_days.sql @@ -0,0 +1,11 @@ +-- Per-book local calendar-day counters: stats_device_day, one row set per book. +CREATE TABLE stats_device_book_day ( + user_id INTEGER NOT NULL REFERENCES users(id), + device_id TEXT NOT NULL, + document TEXT NOT NULL, + date TEXT NOT NULL, + seconds INTEGER NOT NULL CHECK(seconds >= 0 AND seconds <= 4294967295), + updated_at INTEGER NOT NULL, + PRIMARY KEY (user_id, device_id, document, date) +); +CREATE INDEX stats_device_book_day_document ON stats_device_book_day (user_id, document); diff --git a/src/models/document.ts b/src/models/document.ts index b38bc90..9e37a45 100644 --- a/src/models/document.ts +++ b/src/models/document.ts @@ -11,6 +11,7 @@ const DOCUMENT_TABLES = [ 'connector_queue', 'connector_matches', 'stats_device_book', + 'stats_device_book_day', 'clippings', 'bookmarks', 'progress_samples', @@ -50,13 +51,15 @@ export function deleteDocumentData(db: DB, userId: number, document: string): nu } // Stats stay under the hash a device uploaded them with, so merged-in // copies still hold their own rows. - const aliasStats = db - .prepare( - `DELETE FROM stats_device_book WHERE user_id = ? AND document IN - (SELECT alias FROM document_aliases WHERE user_id = ? AND document = ?)` - ) - .run(userId, userId, document); - rows += Number(aliasStats.changes); + for (const table of ['stats_device_book', 'stats_device_book_day']) { + const aliasStats = db + .prepare( + `DELETE FROM ${table} WHERE user_id = ? AND document IN + (SELECT alias FROM document_aliases WHERE user_id = ? AND document = ?)` + ) + .run(userId, userId, document); + rows += Number(aliasStats.changes); + } // Merge mappings in either direction die with the book too. const aliasResult = db .prepare('DELETE FROM document_aliases WHERE user_id = ? AND (alias = ? OR document = ?)') diff --git a/src/routes/account.ts b/src/routes/account.ts index 0fb83af..ab37882 100644 --- a/src/routes/account.ts +++ b/src/routes/account.ts @@ -23,6 +23,7 @@ export function deleteKosyncUserData(db: DB, userId: number, username: string, { 'connector_reveals', 'connector_accounts', 'stats_device_book', + 'stats_device_book_day', 'stats_device_global', 'stats_device_day', 'clippings', diff --git a/src/routes/v1/stats.ts b/src/routes/v1/stats.ts index 40be028..1277d2a 100644 --- a/src/routes/v1/stats.ts +++ b/src/routes/v1/stats.ts @@ -10,6 +10,7 @@ import { parseGlobalStats, parseDailyReading, type BookStatsSnapshot, + type DailyReading, type GlobalStatsSnapshot, } from '../../models/stats.js'; import { computeActivity, type DocInfo, type LogRow } from '../../models/activity.js'; @@ -22,6 +23,8 @@ import { parseReadingSession } from '../../models/reading-sessions.js'; import { importBookorbitStats } from '../../connectors/bookorbit-stats.js'; const MAX_BOOK_BATCH = 20; +const DEFAULT_BOOK_PAGE = 50; +const MAX_BOOK_PAGE = 200; const MAX_INFO_LOOKUPS = 25; const EPOCH_DAY = Date.UTC(2000, 0, 1) / 86400000; @@ -30,6 +33,11 @@ const MAX_OVERVIEW_DAYS = 3650; // Genre groups can be many; the overview returns the ones with the most reading days. const MAX_OVERVIEW_GROUPS = 20; +/** Days from a date -> seconds map, in date order. */ +function sortedDays(days: Map): DailyReading[] { + return [...days].sort(([a], [b]) => a.localeCompare(b)).map(([date, seconds]) => ({ date, seconds })); +} + interface BookInfo { language: string | null; genres: string[]; @@ -230,14 +238,16 @@ export function statsRoutes(db: DB, http?: HttpTransport): Hono { if (!deviceId || !Array.isArray(items) || items.length === 0 || items.length > MAX_BOOK_BATCH) { return kosyncError(c, 403, 2003, 'Invalid request'); } - const parsed: { document: string; snapshot: BookStatsSnapshot }[] = []; + const parsed: { document: string; snapshot: BookStatsSnapshot; daily: DailyReading[] }[] = []; for (const raw of items) { const item = raw as Record; const snapshot = parseBookStats(item); - if (!isValidDocument(item.document) || !snapshot) { + // Optional: the book's seconds per local day, like the global upload's `daily`. + const daily = parseDailyReading(item?.daily); + if (!isValidDocument(item.document) || !snapshot || !daily) { return kosyncError(c, 403, 2003, 'Invalid request'); } - parsed.push({ document: item.document, snapshot }); + parsed.push({ document: item.document, snapshot, daily }); } const user = c.get('user'); const now = nowSeconds(); @@ -248,12 +258,20 @@ export function statsRoutes(db: DB, http?: HttpTransport): Hono { payload = excluded.payload, updated_at = excluded.updated_at` ); + const upsertDay = db.prepare( + `INSERT INTO stats_device_book_day (user_id, device_id, document, date, seconds, updated_at) + VALUES (?, ?, ?, ?, ?, ?) + ON CONFLICT(user_id, device_id, document, date) DO UPDATE SET + seconds = MAX(stats_device_book_day.seconds, excluded.seconds), + updated_at = excluded.updated_at` + ); withTransaction(db, () => { for (const p of parsed) { upsert.run(user.id, deviceId, p.document, JSON.stringify(p.snapshot), now); + for (const d of p.daily) upsertDay.run(user.id, deviceId, p.document, d.date, d.seconds, now); } }); - return c.json({ until: now, accepted: parsed.length }); + return c.json({ until: now, accepted: parsed.length, accepted_daily: parsed.reduce((n, p) => n + p.daily.length, 0) }); }); app.get('/stats/global', (c) => { @@ -322,17 +340,72 @@ export function statsRoutes(db: DB, http?: HttpTransport): Hono { const snapshots = rows.map((r) => JSON.parse(r.payload) as BookStatsSnapshot); const imported = readBookorbitStats(db, user.id, documents); if (imported.book.sessions) snapshots.push(imported.book); + const days = bookDays(user.id, documents); + // Each device's days, then the sum of every device's (BookOrbit's included) per date. + const combinedDays = new Map(); + for (const list of [...days.values(), imported.daily]) { + for (const d of list) combinedDays.set(d.date, (combinedDays.get(d.date) ?? 0) + d.seconds); + } return c.json({ document, - combined: combineBookStats(snapshots), + combined: { ...combineBookStats(snapshots), daily: sortedDays(combinedDays) }, bookorbit: bookorbitExclusions(imported), devices: [...rows.map((r) => ({ device_id: r.device_id, document: r.document, updated_at: r.updated_at, - stats: JSON.parse(r.payload), + stats: { ...JSON.parse(r.payload), daily: days.get(`${r.device_id}\n${r.document}`) ?? [] }, })), ...(imported.book.sessions ? [{ device_id: 'connector:bookorbit', document, - updated_at: imported.updated_at, stats: imported.book }] : [])], + updated_at: imported.updated_at, stats: { ...imported.book, daily: imported.daily } }] : [])], + }); + }); + + // Every book's stats, a page at a time, for clients that restore or show them all: one item per + // device and book, with the book's metadata so a client can match it to a file without the hash. + // `device_id` keeps one device's; `cursor` is the previous page's `next`. + app.get('/stats/books', (c) => { + const user = c.get('user'); + const deviceId = c.req.query('device_id'); + const cursor = c.req.query('cursor') ?? ''; + const limitRaw = Number(c.req.query('limit') ?? DEFAULT_BOOK_PAGE); + const limit = Number.isInteger(limitRaw) && limitRaw >= 1 ? Math.min(limitRaw, MAX_BOOK_PAGE) : DEFAULT_BOOK_PAGE; + // The document is always 32 characters; the device id follows it. + const afterDocument = cursor.slice(0, 32); + const afterDevice = cursor.slice(32); + const rows = db + .prepare( + `SELECT s.device_id, s.document, s.payload, s.updated_at, d.title, d.author, d.filename, d.language + FROM stats_device_book s + LEFT JOIN documents d ON d.user_id = s.user_id AND d.document = s.document + WHERE s.user_id = ? AND (? IS NULL OR s.device_id = ?) + AND (s.document > ? OR (s.document = ? AND s.device_id > ?)) + ORDER BY s.document, s.device_id LIMIT ?` + ) + .all(user.id, deviceId ?? null, deviceId ?? null, afterDocument, afterDocument, afterDevice, limit + 1) as { + device_id: string; + document: string; + payload: string; + updated_at: number; + title: string | null; + author: string | null; + filename: string | null; + language: string | null; + }[]; + const page = rows.slice(0, limit); + const days = bookDays(user.id, [...new Set(page.map((r) => r.document))], deviceId); + const last = page[page.length - 1]; + return c.json({ + items: page.map((r) => ({ + device_id: r.device_id, + document: r.document, + title: r.title, + author: r.author, + filename: r.filename, + language: r.language, + updated_at: r.updated_at, + stats: { ...JSON.parse(r.payload), daily: days.get(`${r.device_id}\n${r.document}`) ?? [] }, + })), + next: rows.length > limit ? `${last.document}${last.device_id}` : null, }); }); @@ -449,6 +522,29 @@ export function statsRoutes(db: DB, http?: HttpTransport): Hono { return c.json(out); }); + /** Per-book days by "device_id\ndocument", in date order; only `deviceId`'s when given. */ + function bookDays(userId: number, documents: string[], deviceId?: string) { + const out = new Map(); + if (!documents.length) return out; + const rows = db + .prepare( + `SELECT device_id, document, date, seconds FROM stats_device_book_day + WHERE user_id = ? AND document IN (${documents.map(() => '?').join(',')}) AND (? IS NULL OR device_id = ?) + ORDER BY date` + ) + .all(userId, ...documents, deviceId ?? null, deviceId ?? null) as { + device_id: string; + document: string; + date: string; + seconds: number; + }[]; + for (const r of rows) { + const key = `${r.device_id}\n${r.document}`; + out.set(key, [...(out.get(key) ?? []), { date: r.date, seconds: Number(r.seconds) }]); + } + return out; + } + /** Language, genres and author per book, and each book's measured time per day (merged copies folded). */ function booksFor(userId: number) { // Merged copies: their time counts toward the book they were merged into (unless kept apart), @@ -479,7 +575,7 @@ export function statsRoutes(db: DB, http?: HttpTransport): Hono { author: r.author, }); } - // Totals from the per-book snapshots; time per day from reading sessions, on the reader's local day. + // Totals from the per-book snapshots, time per day from their uploaded days; reading sessions on top. const time = new Map(); const entry = (d: string) => { const key = canonical(d); @@ -501,26 +597,57 @@ export function statsRoutes(db: DB, http?: HttpTransport): Hono { t.seconds = combined.seconds; t.sessions = combined.sessions; } - const sessionTotals = new Map(); + const dayOf = (date: string) => Math.floor(Date.parse(`${date}T00:00:00Z`) / 86400000) - EPOCH_DAY; + // Per-book days devices uploaded with their snapshots. + for (const r of db.prepare('SELECT document, date, seconds FROM stats_device_book_day WHERE user_id = ?').all(userId) as { + document: string; + date: string; + seconds: number; + }[]) { + const t = entry(r.document); + const d = dayOf(r.date); + t.perDay.set(d, (t.perDay.get(d) ?? 0) + Number(r.seconds)); + } + // Time imported from BookOrbit is one more device's. One lookup per book, only for accounts that + // link it; a single grouped read would scale better if libraries get large. + const hasImports = db + .prepare("SELECT 1 FROM connector_matches WHERE user_id = ? AND connector_id = 'bookorbit' LIMIT 1") + .get(userId); + if (hasImports) { + for (const document of info.keys()) { + const imported = readBookorbitStats(db, userId, statsDocuments(db, userId, document)); + if (!imported.book.sessions) continue; + const t = entry(document); + t.seconds += imported.book.seconds; + t.sessions += imported.book.sessions; + for (const day of imported.daily) { + const d = dayOf(day.date); + t.perDay.set(d, (t.perDay.get(d) ?? 0) + day.seconds); + } + } + } + const sessionTotals = new Map }>(); for (const s of db.prepare('SELECT document, started_at, seconds, utc_offset_minutes FROM reading_sessions WHERE user_id = ?').all(userId) as { document: string; started_at: number; seconds: number; utc_offset_minutes: number; }[]) { - const t = entry(s.document); + const key = canonical(s.document); + const total = sessionTotals.get(key) ?? { seconds: 0, count: 0, perDay: new Map() }; const d = Math.floor((Number(s.started_at) + Number(s.utc_offset_minutes) * 60) / 86400) - EPOCH_DAY; - t.perDay.set(d, (t.perDay.get(d) ?? 0) + Number(s.seconds)); - const total = sessionTotals.get(canonical(s.document)) ?? { seconds: 0, count: 0 }; + total.perDay.set(d, (total.perDay.get(d) ?? 0) + Number(s.seconds)); total.seconds += Number(s.seconds); total.count += 1; - sessionTotals.set(canonical(s.document), total); + sessionTotals.set(key, total); } - // A book's sessions and its snapshot measure the same reading: take whichever saw more. + // A book's sessions and its snapshots measure the same reading: take whichever saw more, in + // total and per day. for (const [document, total] of sessionTotals) { const t = entry(document); t.seconds = Math.max(t.seconds, total.seconds); t.sessions = Math.max(t.sessions, total.count); + for (const [d, s] of total.perDay) t.perDay.set(d, Math.max(t.perDay.get(d) ?? 0, s)); } return { info, time }; } diff --git a/test/book-days.test.ts b/test/book-days.test.ts new file mode 100644 index 0000000..ad90444 --- /dev/null +++ b/test/book-days.test.ts @@ -0,0 +1,82 @@ +import { afterEach, describe, expect, it, vi } from 'vitest'; +import { deleteDocumentData } from '../src/models/document.js'; +import { makeTestApp, registerUser } from './helpers.js'; + +const A = 'a'.repeat(32); +const B = 'b'.repeat(32); + +const book = (document: string, over: Record = {}) => ({ + document, v: 5, sessions: 2, seconds: 1200, pages: 0, completed: false, avg_fwd: 0, pace_n: 0, eta: 0, + start_manual: false, finish_manual: false, start_date: 0, finished_date: 0, tod: [0, 0, 0, 0], + dow: [0, 0, 0, 0, 0, 0, 0], ...over, +}); + +afterEach(() => vi.useRealTimers()); + +describe('per-book days', () => { + it('are uploaded with the snapshot, kept at the larger value and returned per device and combined', async () => { + const { app } = makeTestApp(); + const { headers } = await registerUser(app); + const put = (deviceId: string, items: unknown[]) => + app.request('/api/v1/stats/books', { method: 'PUT', headers, body: JSON.stringify({ device_id: deviceId, items }) }); + + const res = await put('x4', [book(A, { daily: [{ date: '2026-10-20', seconds: 600 }, { date: '2026-10-21', seconds: 600 }] })]); + expect(await res.json()).toMatchObject({ accepted: 1, accepted_daily: 2 }); + await put('x4', [book(A, { daily: [{ date: '2026-10-21', seconds: 300 }] })]); // a smaller resend changes nothing + await put('kobo', [book(A, { daily: [{ date: '2026-10-21', seconds: 120 }] })]); + await put('kobo', [book(B)]); // no days: still accepted + + const one = await (await app.request(`/api/v1/stats/books/${A}`, { headers })).json(); + const x4 = one.devices.find((d: { device_id: string }) => d.device_id === 'x4'); + expect(x4.stats.daily).toEqual([{ date: '2026-10-20', seconds: 600 }, { date: '2026-10-21', seconds: 600 }]); + expect(one.combined.daily).toEqual([{ date: '2026-10-20', seconds: 600 }, { date: '2026-10-21', seconds: 720 }]); + + const bad = await put('x4', [book(A, { daily: [{ date: '2026-02-30', seconds: 60 }] })]); + expect(bad.status).toBe(403); + }); + + it('list every book a page at a time, with metadata, optionally for one device', async () => { + const { app, db } = makeTestApp(); + const { username, headers } = await registerUser(app); + const { id } = db.prepare('SELECT id FROM users WHERE username = ?').get(username) as { id: number }; + db.prepare('INSERT INTO documents (user_id, document, title, filename, language, updated_at) VALUES (?, ?, ?, ?, ?, 0)') + .run(id, A, 'Kokoro', 'kokoro.epub', 'ja'); + const put = (deviceId: string, items: unknown[]) => + app.request('/api/v1/stats/books', { method: 'PUT', headers, body: JSON.stringify({ device_id: deviceId, items }) }); + await put('x4', [book(A, { daily: [{ date: '2026-10-20', seconds: 600 }] }), book(B)]); + await put('kobo', [book(A)]); + + const first = await (await app.request('/api/v1/stats/books?limit=2', { headers })).json(); + expect(first.items.map((i: { device_id: string; document: string }) => [i.document, i.device_id])).toEqual([ + [A, 'kobo'], + [A, 'x4'], + ]); + expect(first.items[1]).toMatchObject({ title: 'Kokoro', filename: 'kokoro.epub', language: 'ja' }); + expect(first.items[1].stats.daily).toEqual([{ date: '2026-10-20', seconds: 600 }]); + expect(first.items[0].stats.daily).toEqual([]); + const second = await (await app.request(`/api/v1/stats/books?limit=2&cursor=${first.next}`, { headers })).json(); + expect(second.items.map((i: { document: string }) => i.document)).toEqual([B]); + expect(second.next).toBeNull(); + + const own = await (await app.request('/api/v1/stats/books?device_id=x4', { headers })).json(); + expect(own.items.map((i: { device_id: string }) => i.device_id)).toEqual(['x4', 'x4']); + }); + + it('drive reading days and today in a filtered overview, and go with the book', async () => { + vi.useFakeTimers({ toFake: ['Date'] }); + vi.setSystemTime(new Date('2026-10-24T18:00:00Z')); + const { app, db } = makeTestApp(); + const { username, headers } = await registerUser(app); + const { id } = db.prepare('SELECT id FROM users WHERE username = ?').get(username) as { id: number }; + db.prepare('INSERT INTO documents (user_id, document, language, updated_at) VALUES (?, ?, ?, 0)').run(id, A, 'ja'); + const daily = [{ date: '2026-10-23', seconds: 900 }, { date: '2026-10-24', seconds: 300 }]; + await app.request('/api/v1/stats/books', { + method: 'PUT', headers, body: JSON.stringify({ device_id: 'x4', items: [book(A, { daily })] }), + }); + const o = await (await app.request('/api/v1/stats/overview?days=10&language=ja', { headers })).json(); + expect(o).toMatchObject({ reading_days: 2, current_streak: 2, seconds: 1200, today_seconds: 300, week_seconds: 1200 }); + + deleteDocumentData(db, id, A); + expect(db.prepare('SELECT COUNT(*) AS n FROM stats_device_book_day').get()).toEqual({ n: 0 }); + }); +}); From ed56506d94de7bc79a2e1bdcc8878b0aece0cd89 Mon Sep 17 00:00:00 2001 From: Eszter Date: Sun, 25 Oct 2026 10:23:18 +0100 Subject: [PATCH 2/5] perf: read imported time only for books matched to BookOrbit --- src/routes/v1/stats.ts | 32 ++++++++++++++++---------------- test/book-days.test.ts | 20 ++++++++++++++++++++ 2 files changed, 36 insertions(+), 16 deletions(-) diff --git a/src/routes/v1/stats.ts b/src/routes/v1/stats.ts index 1277d2a..73ea1d1 100644 --- a/src/routes/v1/stats.ts +++ b/src/routes/v1/stats.ts @@ -608,22 +608,22 @@ export function statsRoutes(db: DB, http?: HttpTransport): Hono { const d = dayOf(r.date); t.perDay.set(d, (t.perDay.get(d) ?? 0) + Number(r.seconds)); } - // Time imported from BookOrbit is one more device's. One lookup per book, only for accounts that - // link it; a single grouped read would scale better if libraries get large. - const hasImports = db - .prepare("SELECT 1 FROM connector_matches WHERE user_id = ? AND connector_id = 'bookorbit' LIMIT 1") - .get(userId); - if (hasImports) { - for (const document of info.keys()) { - const imported = readBookorbitStats(db, userId, statsDocuments(db, userId, document)); - if (!imported.book.sessions) continue; - const t = entry(document); - t.seconds += imported.book.seconds; - t.sessions += imported.book.sessions; - for (const day of imported.daily) { - const d = dayOf(day.date); - t.perDay.set(d, (t.perDay.get(d) ?? 0) + day.seconds); - } + // Time imported from BookOrbit is one more device's. Only books matched to a BookOrbit book have + // any, so those are the only ones read. + const matched = new Set( + (db.prepare("SELECT DISTINCT document FROM connector_matches WHERE user_id = ? AND connector_id = 'bookorbit'") + .all(userId) as { document: string }[]).map((r) => canonical(r.document)) + ); + for (const document of matched) { + if (!info.has(document)) continue; + const imported = readBookorbitStats(db, userId, statsDocuments(db, userId, document)); + if (!imported.book.sessions) continue; + const t = entry(document); + t.seconds += imported.book.seconds; + t.sessions += imported.book.sessions; + for (const day of imported.daily) { + const d = dayOf(day.date); + t.perDay.set(d, (t.perDay.get(d) ?? 0) + day.seconds); } } const sessionTotals = new Map }>(); diff --git a/test/book-days.test.ts b/test/book-days.test.ts index ad90444..c9005e1 100644 --- a/test/book-days.test.ts +++ b/test/book-days.test.ts @@ -79,4 +79,24 @@ describe('per-book days', () => { deleteDocumentData(db, id, A); expect(db.prepare('SELECT COUNT(*) AS n FROM stats_device_book_day').get()).toEqual({ n: 0 }); }); + + it('count imported BookOrbit time in a filtered overview, only for the matched book', async () => { + vi.useFakeTimers({ toFake: ['Date'] }); + vi.setSystemTime(new Date('2026-10-24T18:00:00Z')); + const { app, db } = makeTestApp(); + const { username, headers } = await registerUser(app); + const { id } = db.prepare('SELECT id FROM users WHERE username = ?').get(username) as { id: number }; + const doc = db.prepare('INSERT INTO documents (user_id, document, language, updated_at) VALUES (?, ?, ?, 0)'); + doc.run(id, A, 'ja'); + doc.run(id, B, 'ja'); + db.prepare("INSERT INTO connector_accounts (user_id, connector_id, cred_enc, created_at, updated_at) VALUES (?, 'bookorbit', '', 0, 0)").run(id); + db.prepare("INSERT INTO connector_matches (user_id, connector_id, document, external_id, source, updated_at) VALUES (?, 'bookorbit', ?, 'orbit-1', 'manual', 0)").run(id, A); + db.prepare("INSERT INTO bookorbit_stats_sync (user_id, external_id, active_generation, timezone) VALUES (?, 'orbit-1', 1, 'UTC')").run(id); + const start = Date.parse('2026-10-24T08:00:00Z') / 1000; + db.prepare('INSERT INTO bookorbit_stats_sessions (user_id, external_id, generation, session_id, started_at, ended_at, seconds) VALUES (?, ?, 1, 1, ?, ?, 600)') + .run(id, 'orbit-1', start, start + 600); + + const o = await (await app.request('/api/v1/stats/overview?days=10&group_by=language', { headers })).json(); + expect(o.groups.ja).toMatchObject({ reading_days: 1, seconds: 600, sessions: 1, today_seconds: 600 }); + }); }); From 514901dfe98fd35a1e7316ad957b5e7edfd218c0 Mon Sep 17 00:00:00 2001 From: Eszter Date: Sun, 25 Oct 2026 10:23:18 +0100 Subject: [PATCH 3/5] fix: self-review of per-book days An empty device_id no longer filters for a device called "". The overview sums devices' days in SQL, one date helper serves both places, and the docs say the book list shows merged copies under their own hash. --- docs/API.md | 3 ++- src/routes/v1/stats.ts | 28 +++++++++++++++++----------- test/book-days.test.ts | 2 ++ 3 files changed, 21 insertions(+), 12 deletions(-) diff --git a/docs/API.md b/docs/API.md index d44f0fd..faf1631 100644 --- a/docs/API.md +++ b/docs/API.md @@ -642,7 +642,8 @@ device restoring its own history reads its own row, so another device's time is Every book's stats, one item per device and book, ordered by document. Each item carries the book's `title`, `author`, `filename` and `language`, so a client can match it to a file without knowing the -hash, plus `stats` with `daily` as above. `device_id` keeps one device's items. `limit` defaults to 50 +hash, plus `stats` with `daily` as above. Items are stored rows: a merged copy is listed under its own hash, where +`GET /stats/books/{document}` folds it into its book. `device_id` keeps one device's items. `limit` defaults to 50 (at most 200); pass the answer's `next` as `cursor` for the following page, until it is `null`. ```json diff --git a/src/routes/v1/stats.ts b/src/routes/v1/stats.ts index 73ea1d1..7907b29 100644 --- a/src/routes/v1/stats.ts +++ b/src/routes/v1/stats.ts @@ -33,6 +33,11 @@ const MAX_OVERVIEW_DAYS = 3650; // Genre groups can be many; the overview returns the ones with the most reading days. const MAX_OVERVIEW_GROUPS = 20; +/** A YYYY-MM-DD date as days since 2000-01-01. */ +function dayNumber(date: string): number { + return Math.floor(Date.parse(`${date}T00:00:00Z`) / 86400000) - EPOCH_DAY; +} + /** Days from a date -> seconds map, in date order. */ function sortedDays(days: Map): DailyReading[] { return [...days].sort(([a], [b]) => a.localeCompare(b)).map(([date, seconds]) => ({ date, seconds })); @@ -365,7 +370,8 @@ export function statsRoutes(db: DB, http?: HttpTransport): Hono { // `device_id` keeps one device's; `cursor` is the previous page's `next`. app.get('/stats/books', (c) => { const user = c.get('user'); - const deviceId = c.req.query('device_id'); + // An empty device_id means no filter, not a device called "". + const deviceId = c.req.query('device_id') || undefined; const cursor = c.req.query('cursor') ?? ''; const limitRaw = Number(c.req.query('limit') ?? DEFAULT_BOOK_PAGE); const limit = Number.isInteger(limitRaw) && limitRaw >= 1 ? Math.min(limitRaw, MAX_BOOK_PAGE) : DEFAULT_BOOK_PAGE; @@ -430,7 +436,6 @@ export function statsRoutes(db: DB, http?: HttpTransport): Hono { } const activity = await activityFor(user.id, tz); const day = (t: number) => Math.floor(t / 86400000) - EPOCH_DAY; // days since 2000-01-01 - const dayOf = (name: string) => day(Date.parse(`${name}T00:00:00Z`)); const today = day(Date.now() - tz * 60000); const year = new Date(Date.now() - tz * 60000).getUTCFullYear(); @@ -469,13 +474,13 @@ export function statsRoutes(db: DB, http?: HttpTransport): Hono { const account = () => { const { summary, daily } = summaryFor(user.id); const read = new Set(); - for (const d of activity.reading_days) read.add(dayOf(d)); - for (const [d, seconds] of daily) if (seconds > 0) read.add(dayOf(d)); + for (const d of activity.reading_days) read.add(dayNumber(d)); + for (const [d, seconds] of daily) if (seconds > 0) read.add(dayNumber(d)); if (summary.anchor_day > 0 && summary.history_b64) { const bits = Buffer.from(summary.history_b64, 'base64'); for (let n = 0; n < bits.length * 8; n++) if (bits[n >> 3] & (1 << (n & 7))) read.add(summary.anchor_day - n); } - const perDay = new Map([...daily].map(([d, seconds]) => [dayOf(d), seconds] as [number, number])); + const perDay = new Map([...daily].map(([d, seconds]) => [dayNumber(d), seconds] as [number, number])); return digest(read, activity.books, activity.pages_total, { seconds: summary.seconds, sessions: summary.sessions, perDay }); }; if (!filter.active && !groupBy) return c.json(account()); @@ -484,7 +489,7 @@ export function statsRoutes(db: DB, http?: HttpTransport): Hono { const books = booksFor(user.id); const forBooks = (keep: (d: string) => boolean) => { const read = new Set(); - for (const d of activity.days) if (d.books.some((b) => keep(b.document))) read.add(dayOf(d.day)); + for (const d of activity.days) if (d.books.some((b) => keep(b.document))) read.add(dayNumber(d.day)); const perDay = new Map(); let seconds = 0; let sessions = 0; @@ -597,15 +602,16 @@ export function statsRoutes(db: DB, http?: HttpTransport): Hono { t.seconds = combined.seconds; t.sessions = combined.sessions; } - const dayOf = (date: string) => Math.floor(Date.parse(`${date}T00:00:00Z`) / 86400000) - EPOCH_DAY; - // Per-book days devices uploaded with their snapshots. - for (const r of db.prepare('SELECT document, date, seconds FROM stats_device_book_day WHERE user_id = ?').all(userId) as { + // Per-book days devices uploaded with their snapshots, every device's summed. + for (const r of db + .prepare('SELECT document, date, SUM(seconds) AS seconds FROM stats_device_book_day WHERE user_id = ? GROUP BY document, date') + .all(userId) as { document: string; date: string; seconds: number; }[]) { const t = entry(r.document); - const d = dayOf(r.date); + const d = dayNumber(r.date); t.perDay.set(d, (t.perDay.get(d) ?? 0) + Number(r.seconds)); } // Time imported from BookOrbit is one more device's. Only books matched to a BookOrbit book have @@ -622,7 +628,7 @@ export function statsRoutes(db: DB, http?: HttpTransport): Hono { t.seconds += imported.book.seconds; t.sessions += imported.book.sessions; for (const day of imported.daily) { - const d = dayOf(day.date); + const d = dayNumber(day.date); t.perDay.set(d, (t.perDay.get(d) ?? 0) + day.seconds); } } diff --git a/test/book-days.test.ts b/test/book-days.test.ts index c9005e1..9a6081a 100644 --- a/test/book-days.test.ts +++ b/test/book-days.test.ts @@ -60,6 +60,8 @@ describe('per-book days', () => { const own = await (await app.request('/api/v1/stats/books?device_id=x4', { headers })).json(); expect(own.items.map((i: { device_id: string }) => i.device_id)).toEqual(['x4', 'x4']); + const all = await (await app.request('/api/v1/stats/books?device_id=', { headers })).json(); + expect(all.items).toHaveLength(3); }); it('drive reading days and today in a filtered overview, and go with the book', async () => { From 44a703bcfde64f287ee2fb396b5563d66c7fe845 Mon Sep 17 00:00:00 2001 From: Eszter Date: Sun, 25 Oct 2026 10:23:18 +0100 Subject: [PATCH 4/5] feat: since filter on the book list --- docs/API.md | 6 ++++-- src/routes/v1/stats.ts | 12 +++++++++--- test/book-days.test.ts | 7 +++++++ 3 files changed, 20 insertions(+), 5 deletions(-) diff --git a/docs/API.md b/docs/API.md index faf1631..50d91d6 100644 --- a/docs/API.md +++ b/docs/API.md @@ -638,19 +638,21 @@ weighted by `pace_n`, `start_date` = earliest non-zero, `finished_date` = latest `stats.daily` holds that device's days for the book; `combined.daily` sums every device per date. A device restoring its own history reads its own row, so another device's time is never folded in. -#### GET /api/v1/stats/books?device_id=&limit=&cursor= +#### GET /api/v1/stats/books?device_id=&limit=&cursor=&since= Every book's stats, one item per device and book, ordered by document. Each item carries the book's `title`, `author`, `filename` and `language`, so a client can match it to a file without knowing the hash, plus `stats` with `daily` as above. Items are stored rows: a merged copy is listed under its own hash, where `GET /stats/books/{document}` folds it into its book. `device_id` keeps one device's items. `limit` defaults to 50 (at most 200); pass the answer's `next` as `cursor` for the following page, until it is `null`. +`since` keeps the books uploaded at or after that time: pass the `until` of an earlier answer (its first page's) to +fetch only what changed since then. ```json {"items": [{"device_id": "A1B2C3D4", "document": "a1b2...", "title": "Kokoro", "author": null, "filename": "kokoro.epub", "language": "ja", "updated_at": 1761300000, "stats": {"v": 5, "sessions": 9, "...": "...", "daily": [{"date": "2026-10-23", "seconds": 900}]}}], - "next": "a1b2...A1B2C3D4"} + "next": "a1b2...A1B2C3D4", "until": 1761300100} ``` Merged documents (`POST /api/v1/documents/merge`, `{"document", "into", "stats"?}`): stats rows diff --git a/src/routes/v1/stats.ts b/src/routes/v1/stats.ts index 7907b29..2635675 100644 --- a/src/routes/v1/stats.ts +++ b/src/routes/v1/stats.ts @@ -367,11 +367,15 @@ export function statsRoutes(db: DB, http?: HttpTransport): Hono { // Every book's stats, a page at a time, for clients that restore or show them all: one item per // device and book, with the book's metadata so a client can match it to a file without the hash. - // `device_id` keeps one device's; `cursor` is the previous page's `next`. + // `device_id` keeps one device's; `cursor` is the previous page's `next`; `since` (a previous answer's + // `until`) keeps the books uploaded since then. app.get('/stats/books', (c) => { const user = c.get('user'); // An empty device_id means no filter, not a device called "". const deviceId = c.req.query('device_id') || undefined; + const sinceRaw = Number(c.req.query('since') ?? 0); + const since = Number.isSafeInteger(sinceRaw) && sinceRaw > 0 ? sinceRaw : 0; + const now = nowSeconds(); const cursor = c.req.query('cursor') ?? ''; const limitRaw = Number(c.req.query('limit') ?? DEFAULT_BOOK_PAGE); const limit = Number.isInteger(limitRaw) && limitRaw >= 1 ? Math.min(limitRaw, MAX_BOOK_PAGE) : DEFAULT_BOOK_PAGE; @@ -383,11 +387,11 @@ export function statsRoutes(db: DB, http?: HttpTransport): Hono { `SELECT s.device_id, s.document, s.payload, s.updated_at, d.title, d.author, d.filename, d.language FROM stats_device_book s LEFT JOIN documents d ON d.user_id = s.user_id AND d.document = s.document - WHERE s.user_id = ? AND (? IS NULL OR s.device_id = ?) + WHERE s.user_id = ? AND (? IS NULL OR s.device_id = ?) AND s.updated_at >= ? AND (s.document > ? OR (s.document = ? AND s.device_id > ?)) ORDER BY s.document, s.device_id LIMIT ?` ) - .all(user.id, deviceId ?? null, deviceId ?? null, afterDocument, afterDocument, afterDevice, limit + 1) as { + .all(user.id, deviceId ?? null, deviceId ?? null, since, afterDocument, afterDocument, afterDevice, limit + 1) as { device_id: string; document: string; payload: string; @@ -412,6 +416,8 @@ export function statsRoutes(db: DB, http?: HttpTransport): Hono { stats: { ...JSON.parse(r.payload), daily: days.get(`${r.device_id}\n${r.document}`) ?? [] }, })), next: rows.length > limit ? `${last.document}${last.device_id}` : null, + // Pass as `since` next time. Inclusive, so a book uploaded in this same second comes again. + until: now, }); }); diff --git a/test/book-days.test.ts b/test/book-days.test.ts index 9a6081a..67e9cd9 100644 --- a/test/book-days.test.ts +++ b/test/book-days.test.ts @@ -62,6 +62,13 @@ describe('per-book days', () => { expect(own.items.map((i: { device_id: string }) => i.device_id)).toEqual(['x4', 'x4']); const all = await (await app.request('/api/v1/stats/books?device_id=', { headers })).json(); expect(all.items).toHaveLength(3); + + // Only what was uploaded since an earlier answer's `until`. + const until = all.until as number; + db.prepare('UPDATE stats_device_book SET updated_at = ? WHERE device_id = ?').run(until - 10, 'kobo'); + db.prepare('UPDATE stats_device_book SET updated_at = ? WHERE device_id = ?').run(until, 'x4'); + const changed = await (await app.request(`/api/v1/stats/books?since=${until}`, { headers })).json(); + expect(changed.items.map((i: { device_id: string }) => i.device_id)).toEqual(['x4', 'x4']); }); it('drive reading days and today in a filtered overview, and go with the book', async () => { From b1f96e4286a72fb8bc04301e688d8686670205d0 Mon Sep 17 00:00:00 2001 From: Eszter Date: Sun, 25 Oct 2026 10:23:18 +0100 Subject: [PATCH 5/5] fix: cursor for any document length, zero-second days The book list cursor splits the document from the device id at ':' instead of assuming a 32-character document, and the docs say to treat it as opaque and URL-encode it. Per-book days with no seconds no longer count as reading days in filtered overviews. --- docs/API.md | 5 +++-- src/routes/v1/stats.ts | 11 ++++++----- test/book-days.test.ts | 31 +++++++++++++++++++++++++++++++ 3 files changed, 40 insertions(+), 7 deletions(-) diff --git a/docs/API.md b/docs/API.md index 50d91d6..c3d9de2 100644 --- a/docs/API.md +++ b/docs/API.md @@ -644,7 +644,8 @@ Every book's stats, one item per device and book, ordered by document. Each item `title`, `author`, `filename` and `language`, so a client can match it to a file without knowing the hash, plus `stats` with `daily` as above. Items are stored rows: a merged copy is listed under its own hash, where `GET /stats/books/{document}` folds it into its book. `device_id` keeps one device's items. `limit` defaults to 50 -(at most 200); pass the answer's `next` as `cursor` for the following page, until it is `null`. +(at most 200); pass the answer's `next`, URL-encoded, as `cursor` for the following page, until it is `null`. +Treat it as opaque: its format may change. `since` keeps the books uploaded at or after that time: pass the `until` of an earlier answer (its first page's) to fetch only what changed since then. @@ -652,7 +653,7 @@ fetch only what changed since then. {"items": [{"device_id": "A1B2C3D4", "document": "a1b2...", "title": "Kokoro", "author": null, "filename": "kokoro.epub", "language": "ja", "updated_at": 1761300000, "stats": {"v": 5, "sessions": 9, "...": "...", "daily": [{"date": "2026-10-23", "seconds": 900}]}}], - "next": "a1b2...A1B2C3D4", "until": 1761300100} + "next": "a1b2...:A1B2C3D4", "until": 1761300100} ``` Merged documents (`POST /api/v1/documents/merge`, `{"document", "into", "stats"?}`): stats rows diff --git a/src/routes/v1/stats.ts b/src/routes/v1/stats.ts index 2635675..5b56362 100644 --- a/src/routes/v1/stats.ts +++ b/src/routes/v1/stats.ts @@ -379,9 +379,10 @@ export function statsRoutes(db: DB, http?: HttpTransport): Hono { const cursor = c.req.query('cursor') ?? ''; const limitRaw = Number(c.req.query('limit') ?? DEFAULT_BOOK_PAGE); const limit = Number.isInteger(limitRaw) && limitRaw >= 1 ? Math.min(limitRaw, MAX_BOOK_PAGE) : DEFAULT_BOOK_PAGE; - // The document is always 32 characters; the device id follows it. - const afterDocument = cursor.slice(0, 32); - const afterDevice = cursor.slice(32); + // Documents never contain ':', so the first one splits the document from the device id. + const sep = cursor.indexOf(':'); + const afterDocument = sep < 0 ? '' : cursor.slice(0, sep); + const afterDevice = sep < 0 ? '' : cursor.slice(sep + 1); const rows = db .prepare( `SELECT s.device_id, s.document, s.payload, s.updated_at, d.title, d.author, d.filename, d.language @@ -415,7 +416,7 @@ export function statsRoutes(db: DB, http?: HttpTransport): Hono { updated_at: r.updated_at, stats: { ...JSON.parse(r.payload), daily: days.get(`${r.device_id}\n${r.document}`) ?? [] }, })), - next: rows.length > limit ? `${last.document}${last.device_id}` : null, + next: rows.length > limit ? `${last.document}:${last.device_id}` : null, // Pass as `since` next time. Inclusive, so a book uploaded in this same second comes again. until: now, }); @@ -610,7 +611,7 @@ export function statsRoutes(db: DB, http?: HttpTransport): Hono { } // Per-book days devices uploaded with their snapshots, every device's summed. for (const r of db - .prepare('SELECT document, date, SUM(seconds) AS seconds FROM stats_device_book_day WHERE user_id = ? GROUP BY document, date') + .prepare('SELECT document, date, SUM(seconds) AS seconds FROM stats_device_book_day WHERE user_id = ? GROUP BY document, date HAVING SUM(seconds) > 0') .all(userId) as { document: string; date: string; diff --git a/test/book-days.test.ts b/test/book-days.test.ts index 67e9cd9..e22b1bb 100644 --- a/test/book-days.test.ts +++ b/test/book-days.test.ts @@ -108,4 +108,35 @@ describe('per-book days', () => { const o = await (await app.request('/api/v1/stats/overview?days=10&group_by=language', { headers })).json(); expect(o.groups.ja).toMatchObject({ reading_days: 1, seconds: 600, sessions: 1, today_seconds: 600 }); }); + + it('page through documents of any length, and leave zero-second days out of reading days', async () => { + vi.useFakeTimers({ toFake: ['Date'] }); + vi.setSystemTime(new Date('2026-10-24T18:00:00Z')); + const { app, db } = makeTestApp(); + const { username, headers } = await registerUser(app); + const { id } = db.prepare('SELECT id FROM users WHERE username = ?').get(username) as { id: number }; + const docs = ['a', 'b'.repeat(64), 'c'.repeat(32), 'd.e-f_1']; + await app.request('/api/v1/stats/books', { + method: 'PUT', headers, body: JSON.stringify({ device_id: 'dev:1&x', items: docs.map((d) => book(d)) }), + }); + const seen: string[] = []; + let cursor: string | null = ''; + for (let i = 0; cursor !== null && i < 10; i++) { + const page = await ( + await app.request(`/api/v1/stats/books?limit=1${cursor ? `&cursor=${encodeURIComponent(cursor)}` : ''}`, { headers }) + ).json(); + seen.push(...page.items.map((it: { document: string }) => it.document)); + cursor = page.next; + } + expect(seen).toEqual([...docs].sort()); + + db.prepare('INSERT INTO documents (user_id, document, language, updated_at) VALUES (?, ?, ?, 0)').run(id, A, 'ja'); + await app.request('/api/v1/stats/books', { + method: 'PUT', headers, + body: JSON.stringify({ device_id: 'x4', items: [book(A, { daily: [{ date: '2026-10-24', seconds: 0 }] })] }), + }); + const o = await (await app.request('/api/v1/stats/overview?days=10&language=ja', { headers })).json(); + expect(o.reading_days).toBe(0); + }); }); +