From 22e300c6a9a50ed6d71e0126ccde6b22b106493e Mon Sep 17 00:00:00 2001 From: Eszter Date: Sat, 24 Oct 2026 19:37:57 +0200 Subject: [PATCH] Add GET /api/v1/stats/overview: the Stats page's totals in a few hundred bytes Counts and streaks over all history and every source; the reading-day bitmap is windowed by ?days= (default 365, max 3650) so readers can fetch it cheaply. --- docs/API.md | 19 ++++ src/routes/v1/stats.ts | 181 ++++++++++++++++++++++++------------ test/stats-overview.test.ts | 50 ++++++++++ 3 files changed, 193 insertions(+), 57 deletions(-) create mode 100644 test/stats-overview.test.ts diff --git a/docs/API.md b/docs/API.md index b768f6f..547119e 100644 --- a/docs/API.md +++ b/docs/API.md @@ -575,6 +575,25 @@ Server-side aggregate across all of the user's devices: } ``` +#### GET /api/v1/stats/overview?tz=&days= + +A small digest for readers that want to show the account's totals on the device. It has the same numbers +the Stats page shows, from every source: syncs, chosen start and finish dates, and device stats. + +```json +{"anchor_day": 9793, "days": 365, "history_b64": "...", "reading_days": 53, + "current_streak": 2, "longest_streak": 6, "books_finished": 496, "finished_this_year": 31, + "pages_read": 142177, "seconds": 2880, "sessions": 0, "today_seconds": 420, "week_seconds": 2880} +``` + +- `tz` is the caller's UTC offset in minutes as JS `getTimezoneOffset()` gives it. It decides what "today" and "this year" are. +- `anchor_day` is today, as days since 2000-01-01. In `history_b64`, bit N (LSB-first per byte) is set when day + `anchor_day - N` was a reading day. The bitmap covers the last `days` days (default 365, at most 3650), so + the response stays a few hundred bytes. +- All the counts cover the whole history, whatever `days` is: `reading_days`, the streaks, the finished counts and the pages. + The time fields come from device stats: `seconds`, `sessions`, `today_seconds`, and `week_seconds` for the + last 7 days including today. + #### GET /api/v1/stats/global Raw per-device snapshots (`{"devices": [{"device_id", "device", "updated_at", "stats": {...}}]}`) diff --git a/src/routes/v1/stats.ts b/src/routes/v1/stats.ts index ddce99d..2cd3ce4 100644 --- a/src/routes/v1/stats.ts +++ b/src/routes/v1/stats.ts @@ -24,6 +24,16 @@ import { importBookorbitStats } from '../../connectors/bookorbit-stats.js'; const MAX_BOOK_BATCH = 20; const MAX_INFO_LOOKUPS = 25; +const EPOCH_DAY = Date.UTC(2000, 0, 1) / 86400000; +const DEFAULT_OVERVIEW_DAYS = 365; +const MAX_OVERVIEW_DAYS = 3650; + +/** The caller's UTC offset in minutes as JS getTimezoneOffset() gives it; 0 when absent or implausible. */ +function tzFrom(raw: string | undefined): number { + const tz = Number(raw ?? 0); + return Number.isFinite(tz) && Math.abs(tz) <= 14 * 60 ? tz : 0; +} + function deviceIdFrom(o: Record): string | null { return typeof o.device_id === 'string' && o.device_id.length > 0 && o.device_id.length <= 128 ? o.device_id @@ -33,6 +43,71 @@ function deviceIdFrom(o: Record): string | null { export function statsRoutes(db: DB, http?: HttpTransport): Hono { const app = new Hono(); + // Device snapshots combined across devices, plus seconds per day. + function summaryFor(userId: number) { + const rows = db + .prepare( + 'SELECT device_id, device, payload, updated_at FROM stats_device_global WHERE user_id = ?' + ) + .all(userId) as { device_id: string; device: string; payload: string; updated_at: number }[]; + const snapshots = rows.map((r) => JSON.parse(r.payload) as GlobalStatsSnapshot); + const imported = readBookorbitStats(db, userId); + if (imported.global.sessions) snapshots.push(imported.global); + const summary = combineGlobalStats(snapshots); + const daily = new Map(); + for (const d of [...db.prepare( + 'SELECT date, SUM(seconds) AS seconds FROM stats_device_day WHERE user_id = ? GROUP BY date ORDER BY date' + ).all(userId) as { date: string; seconds: number }[], ...imported.daily]) { + daily.set(d.date, (daily.get(d.date) ?? 0) + d.seconds); + } + return { rows, imported, summary, daily }; + } + + async function activityFor(userId: number, tz: number) { + // Books never looked up yet get their page count now (bounded per request). + const pending = db + .prepare( + `SELECT document FROM documents WHERE user_id = ? AND cover_checked_at IS NULL + AND (title IS NOT NULL OR filename IS NOT NULL) LIMIT ?` + ) + .all(userId, MAX_INFO_LOOKUPS) as { document: string }[]; + await Promise.all(pending.map((d) => documentInfo(db, userId, d.document, http).catch(() => null))); + + const rows = db + .prepare('SELECT document, percentage, at FROM progress_log WHERE user_id = ?') + .all(userId) as unknown as LogRow[]; + const docs = new Map( + ( + db + .prepare('SELECT document, page_count, status, status_at, start_date, finished_date FROM documents WHERE user_id = ?') + .all(userId) as unknown as (DocInfo & { document: string })[] + ).map((d) => [d.document, d]) + ); + // CrossInk's own finish dates, combined across devices like /stats/books/:document. + const merged = statsAliases(db, userId); + const snapshotsByDoc = new Map(); + for (const r of db + .prepare('SELECT document, payload FROM stats_device_book WHERE user_id = ?') + .all(userId) as { document: string; payload: string }[]) { + const document = merged.get(r.document) ?? r.document; + const list = snapshotsByDoc.get(document) ?? []; + list.push(JSON.parse(r.payload) as BookStatsSnapshot); + snapshotsByDoc.set(document, list); + } + const finishedDates = new Map(); + const startDates = new Map(); + for (const [document, snapshots] of snapshotsByDoc) { + const combined = combineBookStats(snapshots); + if (combined.completed && combined.finished_date > 0) finishedDates.set(document, combined.finished_date); + if (combined.start_date > 0) startDates.set(document, combined.start_date); + } + const activity = computeActivity(rows, docs, tz, finishedDates, startDates); + const imported = readBookorbitStats(db, userId); + activity.reading_days = [...new Set([...activity.reading_days, ...imported.daily.map((d) => d.date)])].sort(); + return activity; + } + + // Cumulative session checkpoints retain one identity, including stale retries. app.put('/stats/sessions', async (c) => { let body: any; @@ -177,21 +252,7 @@ export function statsRoutes(db: DB, http?: HttpTransport): Hono { app.get('/stats/summary', (c) => { const user = c.get('user'); - const rows = db - .prepare( - 'SELECT device_id, device, payload, updated_at FROM stats_device_global WHERE user_id = ?' - ) - .all(user.id) as { device_id: string; device: string; payload: string; updated_at: number }[]; - const snapshots = rows.map((r) => JSON.parse(r.payload) as GlobalStatsSnapshot); - const imported = readBookorbitStats(db, user.id); - if (imported.global.sessions) snapshots.push(imported.global); - const summary = combineGlobalStats(snapshots); - const daily = new Map(); - for (const d of [...db.prepare( - 'SELECT date, SUM(seconds) AS seconds FROM stats_device_day WHERE user_id = ? GROUP BY date ORDER BY date' - ).all(user.id) as { date: string; seconds: number }[], ...imported.daily]) { - daily.set(d.date, (daily.get(d.date) ?? 0) + d.seconds); - } + const { rows, imported, summary, daily } = summaryFor(user.id); return c.json({ ...summary, daily: [...daily].sort(([a], [b]) => a.localeCompare(b)).map(([date, seconds]) => ({ date, seconds, minutes: seconds / 60 })), @@ -243,51 +304,57 @@ export function statsRoutes(db: DB, http?: HttpTransport): Hono { // Pages/books activity from the progress history (print pages), for readers // that don't send stats. ?tz= is the client's Date#getTimezoneOffset(). - app.get('/stats/activity', async (c) => { - const user = c.get('user'); - const tzRaw = Number(c.req.query('tz') ?? 0); - const tz = Number.isFinite(tzRaw) && Math.abs(tzRaw) <= 14 * 60 ? tzRaw : 0; - // Books never looked up yet get their page count now (bounded per request). - const pending = db - .prepare( - `SELECT document FROM documents WHERE user_id = ? AND cover_checked_at IS NULL - AND (title IS NOT NULL OR filename IS NOT NULL) LIMIT ?` - ) - .all(user.id, MAX_INFO_LOOKUPS) as { document: string }[]; - await Promise.all(pending.map((d) => documentInfo(db, user.id, d.document, http).catch(() => null))); + app.get('/stats/activity', async (c) => c.json(await activityFor(c.get('user').id, tzFrom(c.req.query('tz'))))); - const rows = db - .prepare('SELECT document, percentage, at FROM progress_log WHERE user_id = ?') - .all(user.id) as unknown as LogRow[]; - const docs = new Map( - ( - db - .prepare('SELECT document, page_count, status, status_at, start_date, finished_date FROM documents WHERE user_id = ?') - .all(user.id) as unknown as (DocInfo & { document: string })[] - ).map((d) => [d.document, d]) - ); - // CrossInk's own finish dates, combined across devices like /stats/books/:document. - const merged = statsAliases(db, user.id); - const snapshotsByDoc = new Map(); - for (const r of db - .prepare('SELECT document, payload FROM stats_device_book WHERE user_id = ?') - .all(user.id) as { document: string; payload: string }[]) { - const document = merged.get(r.document) ?? r.document; - const list = snapshotsByDoc.get(document) ?? []; - list.push(JSON.parse(r.payload) as BookStatsSnapshot); - snapshotsByDoc.set(document, list); + // A device-sized digest of what the Stats page shows: the same finished counts, pages and streaks, + // and reading days from every source. Counts cover all history; only the reading-day bitmap is + // windowed (`days`, default 365, at most 3650) so the response stays a few hundred bytes. + app.get('/stats/overview', async (c) => { + const user = c.get('user'); + const tz = tzFrom(c.req.query('tz')); + const daysRaw = Number(c.req.query('days') ?? DEFAULT_OVERVIEW_DAYS); + const days = Number.isInteger(daysRaw) && daysRaw >= 1 ? Math.min(daysRaw, MAX_OVERVIEW_DAYS) : DEFAULT_OVERVIEW_DAYS; + const activity = await activityFor(user.id, tz); + const { summary, daily } = summaryFor(user.id); + const day = (t: number) => Math.floor(t / 86400000) - EPOCH_DAY; // days since 2000-01-01 + const dayName = (d: number) => new Date((d + EPOCH_DAY) * 86400000).toISOString().slice(0, 10); + const today = day(Date.now() - tz * 60000); + const read = new Set(); + for (const d of activity.reading_days) read.add(day(Date.parse(`${d}T00:00:00Z`))); + for (const [d, seconds] of daily) if (seconds > 0) read.add(day(Date.parse(`${d}T00:00:00Z`))); + if (summary.anchor_day > 0 && summary.history_b64) { + const bits = Buffer.from(summary.history_b64, 'base64'); + for (let n = 0; n < bits.length * 8; n++) if (bits[n >> 3] & (1 << (n & 7))) read.add(summary.anchor_day - n); } - const finishedDates = new Map(); - const startDates = new Map(); - for (const [document, snapshots] of snapshotsByDoc) { - const combined = combineBookStats(snapshots); - if (combined.completed && combined.finished_date > 0) finishedDates.set(document, combined.finished_date); - if (combined.start_date > 0) startDates.set(document, combined.start_date); + const past = [...read].filter((d) => d <= today).sort((a, b) => a - b); + let longest = 0; + for (let i = 0, run = 0; i < past.length; i++) { + run = i > 0 && past[i] === past[i - 1] + 1 ? run + 1 : 1; + longest = Math.max(longest, run); } - const activity = computeActivity(rows, docs, tz, finishedDates, startDates); - const imported = readBookorbitStats(db, user.id); - activity.reading_days = [...new Set([...activity.reading_days, ...imported.daily.map((d) => d.date)])].sort(); - return c.json(activity); + // Today counts once read; until then the streak runs to yesterday. + let current = 0; + for (let d = read.has(today) ? today : today - 1; read.has(d); d--) current++; + const history = Buffer.alloc(Math.ceil(days / 8)); + for (const d of past) if (today - d < days) history[(today - d) >> 3] |= 1 << ((today - d) & 7); + const year = new Date(Date.now() - tz * 60000).getUTCFullYear(); + const finished = activity.books.filter((b) => b.finished_at); + return c.json({ + anchor_day: today, + days, + history_b64: history.toString('base64'), + reading_days: past.length, + current_streak: current, + longest_streak: longest, + books_finished: finished.length, + finished_this_year: finished.filter((b) => new Date((b.finished_at! - tz * 60) * 1000).getUTCFullYear() === year).length, + pages_read: activity.pages_total, + // Reading time as the reader measured it (devices that upload stats); none for synced-only books. + seconds: summary.seconds, + sessions: summary.sessions, + today_seconds: daily.get(dayName(today)) ?? 0, + week_seconds: [0, 1, 2, 3, 4, 5, 6].reduce((sum, n) => sum + (daily.get(dayName(today - n)) ?? 0), 0), + }); }); return app; diff --git a/test/stats-overview.test.ts b/test/stats-overview.test.ts new file mode 100644 index 0000000..97b922a --- /dev/null +++ b/test/stats-overview.test.ts @@ -0,0 +1,50 @@ +import { afterEach, describe, expect, it, vi } from 'vitest'; +import { makeTestApp, registerUser } from './helpers.js'; + +const at = (day: string) => Date.parse(`${day}T12:00:00Z`) / 1000; + +afterEach(() => vi.useRealTimers()); + +describe('GET /api/v1/stats/overview', () => { + it('sums every source into counts, streaks and a windowed reading-day bitmap', async () => { + vi.useFakeTimers({ toFake: ['Date'] }); + vi.setSystemTime(new Date('2026-10-24T18:00:00Z')); + const { app, db } = makeTestApp(); + const { username, headers } = await registerUser(app); + const { id } = db.prepare('SELECT id FROM users WHERE username = ?').get(username) as { id: number }; + const log = db.prepare('INSERT INTO progress_log (user_id, document, device_id, percentage, at) VALUES (?, ?, ?, ?, ?)'); + log.run(id, 'a'.repeat(32), 'libby', 0.2, at('2026-10-20')); // synced from elsewhere + log.run(id, 'a'.repeat(32), 'libby', 0.4, at('2026-10-21')); + log.run(id, 'b'.repeat(32), 'goodreads', 1, at('2025-12-01')); // finished last year + log.run(id, 'c'.repeat(32), 'goodreads', 1, at('2026-03-01')); // finished this year + // The reader's own minutes: yesterday and today. + const global = { + device_id: 'x4', device: 'CrossPoint', v: 5, sessions: 0, seconds: 3000, pages: 0, completed: 0, + tod: [0, 0, 0, 0], dow: [0, 0, 0, 0, 0, 0, 0], anchor_day: 0, history_b64: '', streak: 0, + daily: [{ date: '2026-10-23', seconds: 2460 }, { date: '2026-10-24', seconds: 540 }], + }; + expect((await app.request('/api/v1/stats/global', { method: 'PUT', headers, body: JSON.stringify(global) })).status).toBe(200); + + const res = await app.request('/api/v1/stats/overview?days=10', { headers }); + expect(res.status).toBe(200); + const o = await res.json(); + expect(o).toMatchObject({ + anchor_day: 9793, // 2026-10-24 + days: 10, + reading_days: 6, + current_streak: 2, // 10-23 and 10-24; 10-22 was a gap + longest_streak: 2, + books_finished: 2, + finished_this_year: 1, + seconds: 3000, + today_seconds: 540, + week_seconds: 3000, + }); + // Bit N = today - N within the 10-day window: today, yesterday, 3 and 4 days ago. + expect(Buffer.from(o.history_b64, 'base64')).toEqual(Buffer.from([0b00011011, 0])); + + // The default window is a year; the cap keeps a careless request small. + expect(Buffer.from((await (await app.request('/api/v1/stats/overview', { headers })).json()).history_b64, 'base64').length).toBe(46); + expect((await (await app.request('/api/v1/stats/overview?days=99999', { headers })).json()).days).toBe(3650); + }); +});