From 8a0fc72412295a92850b2c2fa2fa8cd8145eb5f6 Mon Sep 17 00:00:00 2001 From: Yusuke Hirao Date: Thu, 16 Jul 2026 15:04:27 +0900 Subject: [PATCH] feat(beholder)!: enrich PageData with mainContents metrics and scrollHeight Collect main-content quantitative data during scrape via page.evaluate, and record desktop/mobile scrollHeight from the image scan path or a light measure path when captureImages is false. Non-HTML responses skip both. BREAKING CHANGE: PageData now requires mainContents and scrollHeight fields. Co-authored-by: Cursor --- packages/@d-zero/beholder/README.md | 41 ++- .../beholder/src/get-main-contents.spec.ts | 282 ++++++++++++++++ .../@d-zero/beholder/src/get-main-contents.ts | 306 ++++++++++++++++++ packages/@d-zero/beholder/src/index.ts | 11 + .../beholder/src/is-html-content-type.spec.ts | 17 + .../beholder/src/is-html-content-type.ts | 20 ++ .../src/measure-scroll-height.spec.ts | 47 +++ .../beholder/src/measure-scroll-height.ts | 52 +++ packages/@d-zero/beholder/src/scraper.ts | 98 ++++-- packages/@d-zero/beholder/src/types.ts | 196 +++++++++++ 10 files changed, 1036 insertions(+), 34 deletions(-) create mode 100644 packages/@d-zero/beholder/src/get-main-contents.spec.ts create mode 100644 packages/@d-zero/beholder/src/get-main-contents.ts create mode 100644 packages/@d-zero/beholder/src/is-html-content-type.spec.ts create mode 100644 packages/@d-zero/beholder/src/is-html-content-type.ts create mode 100644 packages/@d-zero/beholder/src/measure-scroll-height.spec.ts create mode 100644 packages/@d-zero/beholder/src/measure-scroll-height.ts diff --git a/packages/@d-zero/beholder/README.md b/packages/@d-zero/beholder/README.md index 290e7fd2..c1cf77e1 100644 --- a/packages/@d-zero/beholder/README.md +++ b/packages/@d-zero/beholder/README.md @@ -1,6 +1,6 @@ # `@d-zero/beholder` -Puppeteer の `Page` を受け取り、単一ページのメタデータ・リンク・画像・ネットワークリソースを収集するインプロセス型スクレイパー。結果は `ScrapeResult` として戻り値で返却される（イベント経由ではない）。ブラウザ管理は呼び出し側の責任。 +Puppeteer の `Page` を受け取り、単一ページのメタデータ・リンク・画像・ネットワークリソースに加え、メインコンテンツの定量データと desktop/mobile の `scrollHeight` を収集するインプロセス型スクレイパー。結果は `ScrapeResult` として戻り値で返却される（イベント経由ではない）。ブラウザ管理は呼び出し側の責任。 ## Installation @@ -26,13 +26,48 @@ const result = await scraper.scrapeStart(page, parseUrl('https://example.com'), isExternal: false, }); -if (result.type === 'success') { - console.log(result.pageData?.meta.title); +if (result.type === 'success' && result.pageData) { + console.log(result.pageData.meta.title); + console.log(result.pageData.mainContents?.wordCount); + console.log(result.pageData.scrollHeight); } ``` 設計判断（イベントではなく戻り値で返す理由、`page` のライフサイクル責務、リトライ機構など）は `src/scraper.ts` の JSDoc を参照。 +## 取得フロー + +```mermaid +flowchart TD + goto[page.goto] + ct{content-type is text/html?} + early[both null] + html[get HTML] + ext{isExternal?} + extOut[title-only; both null] + idle[networkidle] + mc["page.evaluate: mainContents"] + anchors[getAnchorList / getMeta] + imgs{captureImages?} + fetchImg["#fetchImages + collect scrollHeight"] + light["measureScrollHeight light path"] + done[return PageData] + + goto --> ct + ct -->|no| early + ct -->|yes| html --> ext + ext -->|yes| extOut + ext -->|no| idle --> mc --> anchors --> imgs + imgs -->|yes| fetchImg --> done + imgs -->|no| light --> done +``` + +## 返却データ（`PageData` の追加分） + +- `mainContents`: メイン領域の定量（`wordCount` / `bodyWordCount`、見出し・画像・表・ボタン・iframe / video / audio / canvas の配列、検出した `main` の識別情報）。メイン未検出時もオブジェクトは返り、配列は空・`wordCount` は 0（非 HTML 等で取得しなかったときだけ `null`） +- `scrollHeight`: `{ desktop, mobile }`（各 `number | null`）。未計測時はフィールド全体が `null` +- 配列要素のフィールド詳細は型（`MainContents*` / `ScrollHeightData`）を参照 + ## DOM 文字列からメタ抽出（Puppeteer なし） HTML 文字列を jsdom などでパースしてから `Meta` を取り出したい場合、`extractMetaFromDocument` を使う。`Scraper` が内部で呼ぶ `collectHead → detectTags → classify` パイプラインと同じ実装を再利用するため、戻り値の `Meta` 形状は `scrapeStart` と同一。DOM ライブラリ（jsdom 等）はユーザランドの責務。 diff --git a/packages/@d-zero/beholder/src/get-main-contents.spec.ts b/packages/@d-zero/beholder/src/get-main-contents.spec.ts new file mode 100644 index 00000000..a3bcb3bc --- /dev/null +++ b/packages/@d-zero/beholder/src/get-main-contents.spec.ts @@ -0,0 +1,282 @@ +import type { Page } from 'puppeteer'; + +import { JSDOM } from 'jsdom'; +import { describe, expect, it, vi } from 'vitest'; + +import { extractMainContentsFromDocument, getMainContents } from './get-main-contents.js'; + +/** + * @param html + */ +function createDocument(html: string): Document { + return new JSDOM(html, { url: 'https://example.com/' }).window.document; +} + +/** + * @param html + * @param mainContentSelector + */ +function extract(html: string, mainContentSelector: string | null = null) { + return extractMainContentsFromDocument(mainContentSelector, createDocument(html)); +} + +describe('extractMainContentsFromDocument', () => { + it('returns empty main metrics when no main content element is found, but counts body text', () => { + const result = extract( + 'T
ABC
No main
', + ); + + expect(result).toEqual({ + title: 'T', + main: null, + wordCount: 0, + bodyWordCount: 9, + headings: [], + images: [], + tables: [], + buttons: [], + iframes: [], + videos: [], + audios: [], + canvases: [], + }); + }); + + it('detects
and counts word length', () => { + const result = extract('
こんにちは世界
'); + + expect(result.wordCount).toBe(7); + expect(result.bodyWordCount).toBeGreaterThanOrEqual(7); + expect(result.main?.nodeName).toBe('MAIN'); + }); + + it('detects [role="main"]', () => { + const result = extract('
Content
'); + + expect(result.wordCount).toBe(7); + expect(result.main?.role).toBe('main'); + }); + + it('uses custom mainContentSelector when provided', () => { + const result = extract( + '
Custom main
', + '#page-body', + ); + + expect(result.wordCount).toBe(10); + expect(result.main?.id).toBe('page-body'); + }); + + it('falls back to built-in selectors when custom selector is invalid', () => { + const result = extract( + '
Still works
', + '[[[invalid', + ); + + expect(result.main?.nodeName).toBe('MAIN'); + expect(result.wordCount).toBe(10); + }); + + it('extracts headings from main content', () => { + const result = extract(` +
+

Title

+

Subtitle

+

Section

+
+ `); + + expect(result.headings).toEqual([ + { text: 'Title', level: 1 }, + { text: 'Subtitle', level: 2 }, + { text: 'Section', level: 3 }, + ]); + }); + + it('extracts images from main content', () => { + const result = extract(` +
+ Image A + Image B + +
+ `); + + expect(result.images).toHaveLength(3); + expect(result.images[0]?.alt).toBe('Image A'); + expect(result.images[0]?.src).toContain('a.png'); + }); + + it('extracts table metadata from main content', () => { + const result = extract(` +
+ + + + + + + +
AB
Footer
Merged
12
+
+ `); + + expect(result.tables).toEqual([ + { + rows: 4, + cols: 2, + hasHeader: true, + hasFooter: true, + hasMergedCell: true, + }, + ]); + }); + + it('extracts buttons from main content', () => { + const result = extract(` +
+ + Go + Click + +
+ `); + + expect(result.buttons).toHaveLength(4); + expect(result.buttons[0]).toMatchObject({ + nodeName: 'BUTTON', + text: 'Save', + disabled: false, + }); + expect(result.buttons[1]?.role).toBe('button'); + }); + + it('extracts iframe, video, audio, and canvas from main content', () => { + const result = extract(` +
+ + + + +
+ `); + + expect(result.iframes).toHaveLength(1); + expect(result.iframes[0]).toMatchObject({ + title: 'Embed', + width: '640', + height: '360', + }); + expect(result.iframes[0]?.src).toContain('/embed'); + + expect(result.videos).toHaveLength(1); + expect(result.videos[0]?.src).toContain('/movie.mp4'); + expect(result.videos[0]?.poster).toContain('/poster.jpg'); + expect(result.videos[0]?.width).toBe(320); + expect(result.videos[0]?.height).toBe(180); + + expect(result.audios).toHaveLength(1); + expect(result.audios[0]?.src).toContain('/sound.mp3'); + + expect(result.canvases).toEqual([{ width: 100, height: 50 }]); + }); + + it('treats whitespace-only main text as zero word count', () => { + const result = extract('
\n\t
'); + + expect(result.wordCount).toBe(0); + }); + + it('detects #content fallback selector', () => { + const result = extract( + '
Fallback content
', + ); + + expect(result.wordCount).toBe(15); + }); + + it('detects .main fallback selector', () => { + const result = extract( + '
Class-based main
', + ); + + expect(result.wordCount).toBe(15); + }); + + it('returns first matching element in DOM order when multiple Phase-1 selectors match', () => { + const result = extract(` + +
Semantic main
+
ID content
+ + `); + + expect(result.main?.nodeName).toBe('MAIN'); + expect(result.wordCount).toBe(12); + }); + + it('uses Phase-2 class*=main when Phase-1 finds nothing', () => { + const result = extract( + '
Phase two
', + ); + + expect(result.main?.classList).toContain('page-main-area'); + // "Phase two" → "Phasetwo" (8) after whitespace removal + expect(result.wordCount).toBe(8); + }); + + it('uses Phase-2 id*=content case-insensitively when Phase-1 finds nothing', () => { + const result = extract( + '
Primary
', + ); + + expect(result.main?.id).toBe('PrimaryContent'); + expect(result.wordCount).toBe(7); + }); + + it('prefers Phase-1
over an earlier header-content that would match Phase-2', () => { + const result = extract(` + +
Chrome
+
Real main
+ + `); + + expect(result.main?.nodeName).toBe('MAIN'); + expect(result.wordCount).toBe(8); + }); + + it('may match class*=main on maintenance via Phase-2 (accepted false positive)', () => { + const result = extract( + '
Down
', + ); + + expect(result.main?.classList).toContain('maintenance'); + expect(result.wordCount).toBe(4); + }); + + it('builds a simple diagnostic selector from tag, id, and classes', () => { + const result = extract( + '
X
', + ); + + expect(result.main?.selector).toBe('div#page-body.foo.bar'); + }); +}); + +describe('getMainContents', () => { + it('passes only the selector into page.evaluate so Document defaults in the page realm', async () => { + const evaluate = vi.fn( + (fn: typeof extractMainContentsFromDocument, sel: string | null) => + Promise.resolve( + fn(sel, createDocument('
Hi
')), + ), + ); + const page = { evaluate } as unknown as Page; + + const result = await getMainContents(page, { mainContentSelector: '#x' }); + + expect(evaluate).toHaveBeenCalledWith(extractMainContentsFromDocument, '#x'); + expect(result.wordCount).toBe(2); + }); +}); diff --git a/packages/@d-zero/beholder/src/get-main-contents.ts b/packages/@d-zero/beholder/src/get-main-contents.ts new file mode 100644 index 00000000..780f6ac9 --- /dev/null +++ b/packages/@d-zero/beholder/src/get-main-contents.ts @@ -0,0 +1,306 @@ +/** + * Main-content quantitative extraction for scraped HTML pages. + * + * Runs entirely against a `Document` (Puppeteer page realm or jsdom) so the + * same logic is unit-tested without a browser. Puppeteer callers use + * {@link getMainContents}, which passes this function into `page.evaluate`. + * + * WHY no `@medv/finder`: selector strings are diagnostic only; a tag+id+class // cspell:disable-line + * path is enough and avoids a Node-only dependency inside the page realm. + * @module + */ + +import type { MainContentsData } from './types.js'; +import type { Page } from 'puppeteer'; + +/** + * Extract main-content metrics from a `Document`. + * + * This function must remain free of closure over imports so Puppeteer can + * serialize it into `page.evaluate`. + * + * Argument order matches `page.evaluate(fn, mainContentSelector)`: the selector + * is the first argument; `doc` defaults to the page-realm global `document` + * when omitted (jsdom tests pass an explicit `Document` as the second argument). + * @param mainContentSelector - Optional selector prepended to the default list. + * @param doc - The document to inspect (defaults to the global `document`). + * @returns Quantitative main-content data (never `null`; empty when no main region). + * @example + * ```ts + * // In page.evaluate / browser: + * const data = extractMainContentsFromDocument('#page-body'); + * // In jsdom tests: + * const data = extractMainContentsFromDocument('#page-body', document); + * ``` + */ +export function extractMainContentsFromDocument( + mainContentSelector: string | null = null, + doc: Document = document, +): MainContentsData { + /** + * @param text + */ + function removeSpaces(text: string | null): string | null { + return text?.trim().replaceAll(/\s+/g, '') || null; + } + + /** + * @param value + */ + function cssEscape(value: string): string { + if (typeof CSS !== 'undefined' && typeof CSS.escape === 'function') { + return CSS.escape(value); + } + return value.replaceAll(/([ !"#$%&'()*+,./:;<=>?@[\\\]^`{|}~])/g, '\\$1'); + } + + /** + * @param el + */ + function buildSelector(el: Element): string { + const tag = el.nodeName.toLowerCase(); + const id = el.id ? `#${cssEscape(el.id)}` : ''; + const classes = [...el.classList].map((c) => `.${cssEscape(c)}`).join(''); + return `${tag}${id}${classes}`; + } + + /** + * @param media + */ + function resolveMediaSrc(media: HTMLMediaElement): string { + if (media.currentSrc) { + return media.currentSrc; + } + const attrSrc = media.getAttribute('src'); + if (attrSrc) { + try { + return new URL(attrSrc, doc.baseURI).href; + } catch { + return attrSrc; + } + } + const source = media.querySelector('source[src]'); + const sourceSrc = source?.getAttribute('src'); + if (sourceSrc) { + try { + return new URL(sourceSrc, doc.baseURI).href; + } catch { + return sourceSrc; + } + } + return ''; + } + + const title = doc.title?.trim() ?? ''; + const bodyWordCount = removeSpaces(doc.body?.textContent ?? null)?.length ?? 0; + + const selectors = [ + 'main', + '[role="main"]', + '#main', + '.main', + '#content', + '.content', + '#contents', + '.contents', + '#main-content', + '.main-content', + '#main_content', + '.main_content', + '#mainContent', + '.mainContent', + ]; + if (mainContentSelector) { + selectors.unshift(mainContentSelector); + } + + let $main: Element | null = null; + try { + $main = doc.querySelector(selectors.join(',')); + } catch { + // Invalid custom selector: retry without it so built-in selectors still run. + if (mainContentSelector) { + selectors.shift(); + $main = doc.querySelector(selectors.join(',')); + } + } + + if (!$main) { + const fallbackSelectors = [ + '[id*="main" i]', + '[class*="main" i]', + '[id*="content" i]', + '[class*="content" i]', + ]; + for (const sel of fallbackSelectors) { + const candidate = doc.querySelector(sel); + if (candidate && candidate !== doc.body && candidate !== doc.documentElement) { + $main = candidate; + break; + } + } + } + + if (!$main) { + return { + title, + main: null, + wordCount: 0, + bodyWordCount, + headings: [], + images: [], + tables: [], + buttons: [], + iframes: [], + videos: [], + audios: [], + canvases: [], + }; + } + + const headings: MainContentsData['headings'] = []; + for (const $heading of $main.querySelectorAll( + 'h1, h2, h3, h4, h5, h6', + )) { + headings.push({ + text: removeSpaces($heading.textContent), + level: Number.parseInt($heading.nodeName.replace(/h/i, ''), 10) as + 1 | 2 | 3 | 4 | 5 | 6, + }); + } + + const images: MainContentsData['images'] = []; + for (const $img of $main.querySelectorAll( + 'img, input[type="image"]', + )) { + images.push({ + src: $img.src, + alt: $img.alt, + }); + } + + const tables: MainContentsData['tables'] = []; + for (const $table of $main.querySelectorAll('table')) { + tables.push({ + rows: $table.querySelectorAll('tr').length, + cols: $table.querySelector('tr')?.querySelectorAll('th, td').length || 0, + hasHeader: !!$table.querySelector('thead'), + hasFooter: !!$table.querySelector('tfoot'), + hasMergedCell: !!$table.querySelector('[colspan], [rowspan]'), + }); + } + + const win = doc.defaultView; + const HTMLButton = win?.HTMLButtonElement; + const HTMLInput = win?.HTMLInputElement; + + const buttons: MainContentsData['buttons'] = []; + for (const $el of $main.querySelectorAll( + 'button, [role="button"], [class*="button"], [class*="btn"]', + )) { + const isButton = HTMLButton ? $el instanceof HTMLButton : $el.nodeName === 'BUTTON'; + const isInput = HTMLInput ? $el instanceof HTMLInput : $el.nodeName === 'INPUT'; + const isFormControl = isButton || isInput; + let text = removeSpaces($el.textContent); + if (!text && isInput) { + text = removeSpaces($el.getAttribute('value')); + } + const disabledProp = + isFormControl && 'disabled' in $el + ? Boolean(($el as HTMLButtonElement).disabled) + : false; + buttons.push({ + nodeName: $el.nodeName, + role: $el.getAttribute('role'), + type: isFormControl ? ($el as HTMLButtonElement | HTMLInputElement).type : null, + text, + disabled: disabledProp || $el.getAttribute('aria-disabled') === 'true', + }); + } + + const iframes: MainContentsData['iframes'] = []; + for (const $el of $main.querySelectorAll('iframe')) { + iframes.push({ + src: $el.src, + title: $el.hasAttribute('title') ? $el.getAttribute('title') : null, + width: $el.getAttribute('width'), + height: $el.getAttribute('height'), + }); + } + + const videos: MainContentsData['videos'] = []; + for (const $el of $main.querySelectorAll('video')) { + const posterAttr = $el.getAttribute('poster'); + let poster: string | null = null; + if (posterAttr) { + try { + poster = new URL(posterAttr, doc.baseURI).href; + } catch { + poster = posterAttr; + } + } + videos.push({ + src: resolveMediaSrc($el), + poster, + width: $el.width, + height: $el.height, + }); + } + + const audios: MainContentsData['audios'] = []; + for (const $el of $main.querySelectorAll('audio')) { + audios.push({ + src: resolveMediaSrc($el), + }); + } + + const canvases: MainContentsData['canvases'] = []; + for (const $el of $main.querySelectorAll('canvas')) { + canvases.push({ + width: $el.width, + height: $el.height, + }); + } + + return { + title, + main: { + nodeName: $main.nodeName, + id: $main.id || null, + classList: [...$main.classList], + role: $main.getAttribute('role'), + selector: buildSelector($main), + }, + wordCount: removeSpaces($main.textContent)?.length ?? 0, + bodyWordCount, + headings, + images, + tables, + buttons, + iframes, + videos, + audios, + canvases, + }; +} + +/** + * Extract main-content metrics from a Puppeteer page via a single `page.evaluate`. + * @param page - Puppeteer page whose DOM has finished loading. + * @param options - Optional main-content selector override. + * @param options.mainContentSelector + * @returns Quantitative main-content data. + * @example + * ```ts + * const mainContents = await getMainContents(page, { mainContentSelector: '#page-body' }); + * ``` + */ +export async function getMainContents( + page: Page, + options?: { mainContentSelector?: string | null }, +): Promise { + return page.evaluate( + extractMainContentsFromDocument, + options?.mainContentSelector ?? null, + ); +} diff --git a/packages/@d-zero/beholder/src/index.ts b/packages/@d-zero/beholder/src/index.ts index fa55b14e..58394443 100644 --- a/packages/@d-zero/beholder/src/index.ts +++ b/packages/@d-zero/beholder/src/index.ts @@ -27,6 +27,17 @@ export type { ImageElement, SkippedPageData, NetworkLog, + ScrollHeightData, + MainContentsData, + MainContentsMainTag, + MainContentsHeading, + MainContentsImage, + MainContentsTable, + MainContentsButton, + MainContentsIframe, + MainContentsVideo, + MainContentsAudio, + MainContentsCanvas, OpenGraphMeta, OgArticleMeta, OgBookMeta, diff --git a/packages/@d-zero/beholder/src/is-html-content-type.spec.ts b/packages/@d-zero/beholder/src/is-html-content-type.spec.ts new file mode 100644 index 00000000..a418061a --- /dev/null +++ b/packages/@d-zero/beholder/src/is-html-content-type.spec.ts @@ -0,0 +1,17 @@ +import { describe, expect, it } from 'vitest'; + +import { isHtmlContentType } from './is-html-content-type.js'; + +describe('isHtmlContentType', () => { + it('accepts text/html in any letter case and with surrounding whitespace', () => { + expect(isHtmlContentType('text/html')).toBe(true); + expect(isHtmlContentType('Text/HTML')).toBe(true); + expect(isHtmlContentType(' text/html ')).toBe(true); + }); + + it('rejects non-HTML and null', () => { + expect(isHtmlContentType('application/pdf')).toBe(false); + expect(isHtmlContentType('text/plain')).toBe(false); + expect(isHtmlContentType(null)).toBe(false); + }); +}); diff --git a/packages/@d-zero/beholder/src/is-html-content-type.ts b/packages/@d-zero/beholder/src/is-html-content-type.ts new file mode 100644 index 00000000..1df0074f --- /dev/null +++ b/packages/@d-zero/beholder/src/is-html-content-type.ts @@ -0,0 +1,20 @@ +/** + * Determine whether a Content-Type media type is HTML. + * + * MIME types are case-insensitive (RFC 2045). Surrounding whitespace after + * parameter stripping is also tolerated so `text/html `, `Text/HTML`, and + * `text/html; charset=utf-8` (after `;` split) all classify as HTML. + * @param contentType - The media type portion of a Content-Type header + * (parameters already stripped), or `null` when unknown. + * @returns `true` when the media type is `text/html` in any letter case. + * @example + * ```ts + * isHtmlContentType('text/html'); // true + * isHtmlContentType('Text/HTML'); // true + * isHtmlContentType('application/pdf'); // false + * // Callers should strip parameters first (e.g. split on ';') before calling. + * ``` + */ +export function isHtmlContentType(contentType: string | null): boolean { + return contentType !== null && contentType.trim().toLowerCase() === 'text/html'; +} diff --git a/packages/@d-zero/beholder/src/measure-scroll-height.spec.ts b/packages/@d-zero/beholder/src/measure-scroll-height.spec.ts new file mode 100644 index 00000000..14f45f07 --- /dev/null +++ b/packages/@d-zero/beholder/src/measure-scroll-height.spec.ts @@ -0,0 +1,47 @@ +import type { Page } from 'puppeteer'; + +import { beforeEach, describe, expect, it, vi } from 'vitest'; + +import { measureScrollHeight } from './measure-scroll-height.js'; + +describe('measureScrollHeight', () => { + let page: Page; + let scrollHeights: number[]; + + beforeEach(() => { + scrollHeights = [2400, 5200]; + let call = 0; + page = { + setViewport: vi.fn(() => Promise.resolve()), + evaluate: vi.fn(() => Promise.resolve(scrollHeights[call++] ?? 0)), + } as unknown as Page; + }); + + it('returns desktop and mobile heights without scrolling the page', async () => { + const result = await measureScrollHeight(page); + + expect(result).toEqual({ desktop: 2400, mobile: 5200 }); + expect(page.setViewport).toHaveBeenCalledTimes(2); + expect(page.setViewport).toHaveBeenNthCalledWith(1, { + width: 1280, + height: 800, + deviceScaleFactor: 1, + }); + expect(page.setViewport).toHaveBeenNthCalledWith(2, { + width: 320, + height: 800, + deviceScaleFactor: 2, + }); + }); + + it('returns null for a preset that throws', async () => { + page.setViewport = vi + .fn() + .mockResolvedValueOnce() + .mockRejectedValueOnce(new Error('viewport failed')); + + const result = await measureScrollHeight(page); + + expect(result).toEqual({ desktop: 2400, mobile: null }); + }); +}); diff --git a/packages/@d-zero/beholder/src/measure-scroll-height.ts b/packages/@d-zero/beholder/src/measure-scroll-height.ts new file mode 100644 index 00000000..1b823b58 --- /dev/null +++ b/packages/@d-zero/beholder/src/measure-scroll-height.ts @@ -0,0 +1,52 @@ +/** + * Lightweight scrollHeight measurement without `scrollAllOver`. + * + * Used when `captureImages` is false so callers still receive desktop/mobile + * heights without the cost of full-page scrolling for image extraction. + * @module + */ + +import type { ScrollHeightData } from './types.js'; +import type { Page } from 'puppeteer'; + +import { devicePresets } from '@d-zero/puppeteer-page-scan'; + +/** + * Measure `document.body.scrollHeight` at desktop-compact and mobile-small viewports. + * + * Sets each viewport, reads height once, and does **not** call `scrollAllOver`. + * A failed preset yields `null` for that side only. + * @param page - Puppeteer page whose DOM has finished loading. + * @returns Heights for desktop and mobile presets. + * @example + * ```ts + * const scrollHeight = await measureScrollHeight(page); + * // { desktop: 2400, mobile: 5200 } + * ``` + */ +export async function measureScrollHeight(page: Page): Promise { + const desktop = await measureAtPreset(page, 'desktop-compact'); + const mobile = await measureAtPreset(page, 'mobile-small'); + return { desktop, mobile }; +} + +/** + * @param page + * @param key + */ +async function measureAtPreset( + page: Page, + key: 'desktop-compact' | 'mobile-small', +): Promise { + const preset: { width: number; resolution?: number } = devicePresets[key]; + try { + await page.setViewport({ + width: preset.width, + height: 800, + deviceScaleFactor: preset.resolution ?? 1, + }); + return await page.evaluate(() => document.body.scrollHeight); + } catch { + return null; + } +} diff --git a/packages/@d-zero/beholder/src/scraper.ts b/packages/@d-zero/beholder/src/scraper.ts index d371e379..e04fb658 100644 --- a/packages/@d-zero/beholder/src/scraper.ts +++ b/packages/@d-zero/beholder/src/scraper.ts @@ -10,6 +10,7 @@ import type { PageData, ParseURLOptions, Resource, + ScrollHeightData, SkippedPageData, } from './types.js'; import type { PageScanPhase } from '@d-zero/puppeteer-page-scan'; @@ -28,8 +29,11 @@ import { getImageList, getMeta, } from './dom-evaluation.js'; +import { getMainContents } from './get-main-contents.js'; import { isError } from './is-error.js'; +import { isHtmlContentType } from './is-html-content-type.js'; import { keywordCheck } from './keyword-check.js'; +import { measureScrollHeight } from './measure-scroll-height.js'; import { emptyMeta } from './meta/classify.js'; import { findDisconnectionFailures } from './network-disconnection.js'; import { parseUrl } from './parse-url.js'; @@ -148,6 +152,8 @@ export default class Scraper extends EventEmitter { imageList: [], anchorList: [], html: '', + mainContents: null, + scrollHeight: null, isSkipped: false, }; @@ -176,12 +182,14 @@ export default class Scraper extends EventEmitter { pageData: { ...headResult, isTarget: false, + mainContents: headResult.mainContents ?? null, + scrollHeight: headResult.scrollHeight ?? null, }, resources, }; } - if (headResult === null || headResult.contentType === 'text/html') { + if (headResult === null || isHtmlContentType(headResult.contentType)) { const fetchResult = await this.#fetchData( page, url, @@ -555,7 +563,7 @@ export default class Scraper extends EventEmitter { const _contentLength = Number.parseInt(responseHeaders['content-length'] ?? ''); const contentLength = Number.isFinite(_contentLength) ? _contentLength : null; - if (contentType !== 'text/html') { + if (!isHtmlContentType(contentType)) { return { url, isTarget: false, @@ -570,6 +578,8 @@ export default class Scraper extends EventEmitter { imageList: [], anchorList: [], html: '', + mainContents: null, + scrollHeight: null, isSkipped: false, }; } @@ -619,6 +629,8 @@ export default class Scraper extends EventEmitter { imageList: [], anchorList: [], html, + mainContents: null, + scrollHeight: null, isSkipped: false, }; } @@ -644,6 +656,10 @@ export default class Scraper extends EventEmitter { throw new Error(`Network disconnection detected during page load: ${errorSummary}`); } + const mainContents = await getMainContents(page, { + mainContentSelector: options?.mainContentSelector, + }); + void this.emit('changePhase', { pid: process.pid, name: 'getAnchors', @@ -671,24 +687,29 @@ export default class Scraper extends EventEmitter { domEvaluationTimeout, ); - const imageList = captureImages - ? await (async () => { - void this.emit('changePhase', { - pid: process.pid, - name: 'extractImages', - url, - isExternal, - message: `%countdown(${domEvaluationTimeout},extractImages_${url.withoutHash},s)%s`, - }); - return this.#fetchImages( - page, - url.withoutHashAndAuth, - isExternal, - imageLoadTimeout, - domEvaluationTimeout, - ); - })() - : []; + let imageList: ImageElement[] = []; + let scrollHeight: ScrollHeightData | null = null; + + if (captureImages) { + void this.emit('changePhase', { + pid: process.pid, + name: 'extractImages', + url, + isExternal, + message: `%countdown(${domEvaluationTimeout},extractImages_${url.withoutHash},s)%s`, + }); + const fetched = await this.#fetchImages( + page, + url.withoutHashAndAuth, + isExternal, + imageLoadTimeout, + domEvaluationTimeout, + ); + imageList = fetched.imageList; + scrollHeight = fetched.scrollHeight; + } else { + scrollHeight = await measureScrollHeight(page); + } return { url, @@ -704,6 +725,8 @@ export default class Scraper extends EventEmitter { anchorList, imageList, html, + mainContents, + scrollHeight, isSkipped: false, }; } @@ -719,12 +742,12 @@ export default class Scraper extends EventEmitter { * changes and triggers a reload. Isolating each device preset allows partial * results — if one viewport fails, the other can still succeed. * - * WHY retryable with 20-min timeout and `fallback: []`: Image extraction is - * best-effort. If all retries fail, an empty array is returned rather than - * failing the entire page scrape. The 20-min wall clock accommodates pages - * whose mobile-small `scrollHeight` reaches ~300k px (observed on - * responsive data tables, which take ~5 min to scroll). A shorter timeout - * causes a second retry to start while the previous attempt's + * WHY retryable with 20-min timeout and empty fallback: Image extraction is + * best-effort. If all retries fail, empty images and null scroll heights are + * returned rather than failing the entire page scrape. The 20-min wall clock + * accommodates pages whose mobile-small `scrollHeight` reaches ~300k px + * (observed on responsive data tables, which take ~5 min to scroll). A shorter + * timeout causes a second retry to start while the previous attempt's * `scrollAllOver` is still running its `page.evaluate` calls in the * background — `Promise.race` in `retry.ts` does not cancel `fn()`. The * collision then surfaces as "Attempted to use detached Frame" or @@ -736,16 +759,22 @@ export default class Scraper extends EventEmitter { * preset entirely keeps the timeout-vs-background-evaluate collision from * ever being triggered, at the cost of losing that viewport's image data * for those pages. See {@link MAX_SCROLL_HEIGHT} for the chosen threshold. + * + * Scroll heights are still recorded when scrolling is skipped for the + * height limit — the measurement from `beforePageScan` is kept. * @param page - Puppeteer page instance * @param url - The page URL string (without hash and auth) * @param isExternal - Whether the page is external * @param imageLoadTimeout - Timeout (ms) for waiting images to complete loading * @param domEvaluationTimeout - Timeout (ms) for the in-page image extraction `page.evaluate` - * @returns Array of image elements from all device presets (may be partial if some viewports failed) + * @returns Image elements plus desktop/mobile scroll heights from the scan path */ @retryable({ timeout: 20 * 60 * 1000, - fallback: [], + fallback: { + imageList: [], + scrollHeight: { desktop: null, mobile: null }, + }, onWait(this: Scraper, determinedInterval, retryCount, methodName, error) { void this.emit('changePhase', { pid: process.pid, @@ -771,15 +800,20 @@ export default class Scraper extends EventEmitter { isExternal: boolean, imageLoadTimeout: number, domEvaluationTimeout: number, - ): Promise { + ): Promise<{ imageList: ImageElement[]; scrollHeight: ScrollHeightData }> { const listener = this.#createPageScanListener(isExternal); - const devices: { key: string; preset: { width: number; resolution?: number } }[] = [ + const devices: { + key: 'desktop-compact' | 'mobile-small'; + preset: { width: number; resolution?: number }; + }[] = [ { key: 'desktop-compact', preset: devicePresets['desktop-compact'] }, { key: 'mobile-small', preset: devicePresets['mobile-small'] }, ]; const imageList: ImageElement[] = []; + const scrollHeight: ScrollHeightData = { desktop: null, mobile: null }; for (const { key, preset } of devices) { + const scrollKey = key === 'desktop-compact' ? 'desktop' : 'mobile'; try { void this.emit('changePhase', { pid: process.pid, @@ -798,6 +832,8 @@ export default class Scraper extends EventEmitter { maxScrollHeight: MAX_SCROLL_HEIGHT, }); + scrollHeight[scrollKey] = scanResult.scrollHeight; + if (!scanResult.scrolled) { void this.emit('changePhase', { pid: process.pid, @@ -845,6 +881,6 @@ export default class Scraper extends EventEmitter { } } - return imageList; + return { imageList, scrollHeight }; } } diff --git a/packages/@d-zero/beholder/src/types.ts b/packages/@d-zero/beholder/src/types.ts index 2345775b..93bd7872 100644 --- a/packages/@d-zero/beholder/src/types.ts +++ b/packages/@d-zero/beholder/src/types.ts @@ -103,10 +103,201 @@ export type PageData = { /** HTML snapshot of the rendered DOM. */ html: string; + /** + * Quantitative main-content metrics extracted in the browser after load. + * `null` when the page is non-HTML, external, non-HTTP, or otherwise not fully scraped. + * When HTML is scraped but no main region is found, this is still an object with empty arrays and `wordCount: 0`. + */ + mainContents: MainContentsData | null; + + /** + * `document.body.scrollHeight` at desktop-compact and mobile-small viewports. + * `null` when not measured (non-HTML, external, non-HTTP, or skipped). + */ + scrollHeight: ScrollHeightData | null; + /** Always `false` for successfully scraped pages. See {@link SkippedPageData} for skipped pages. */ isSkipped: false; }; +/** + * `document.body.scrollHeight` measured at the scraper's desktop and mobile device presets. + */ +export type ScrollHeightData = { + /** Height at `desktop-compact` (width 1280), or `null` if that preset failed. */ + desktop: number | null; + /** Height at `mobile-small` (width 320 @ 2x), or `null` if that preset failed. */ + mobile: number | null; +}; + +/** + * Quantitative metrics for the detected main content region of a page. + * @example + * ```ts + * const { mainContents } = pageData; + * if (mainContents) { + * console.log(mainContents.wordCount, mainContents.headings.length); + * } + * ``` + */ +export type MainContentsData = { + /** `document.title` trimmed. Empty string when the title is empty. */ + title: string; + /** + * Identifying info for the detected main region element. + * `null` when no selector matched (arrays are empty and `wordCount` is `0`). + */ + main: MainContentsMainTag | null; + /** + * Character count of the main region's `textContent` after whitespace removal. + * `0` when no main region was found or the text is whitespace-only. + */ + wordCount: number; + /** + * Character count of `document.body` `textContent` after whitespace removal. + * Measured even when no main region is found (`0` if `body` is missing). + */ + bodyWordCount: number; + /** Headings (`h1`–`h6`) inside the main region, in DOM order. */ + headings: MainContentsHeading[]; + /** Images (`img`, `input[type=image]`) inside the main region, in DOM order. */ + images: MainContentsImage[]; + /** Tables inside the main region, in DOM order. */ + tables: MainContentsTable[]; + /** Button-like elements inside the main region, in DOM order. */ + buttons: MainContentsButton[]; + /** Iframes inside the main region, in DOM order. */ + iframes: MainContentsIframe[]; + /** Videos inside the main region, in DOM order. */ + videos: MainContentsVideo[]; + /** Audios inside the main region, in DOM order. */ + audios: MainContentsAudio[]; + /** Canvases inside the main region, in DOM order. */ + canvases: MainContentsCanvas[]; +}; + +/** + * Identifying information about the DOM element used as the main content area. + */ +export type MainContentsMainTag = { + /** Element tag name (e.g. `"MAIN"`, `"DIV"`), as reported by `nodeName`. */ + nodeName: string; + /** The element's `id`, or `null` when empty/absent. */ + id: string | null; + /** CSS class names on the element. */ + classList: string[]; + /** The WAI-ARIA `role` attribute value, or `null` when absent. */ + role: string | null; + /** + * A simple diagnostic CSS selector built from tag + id + classes + * (not `@medv/finder`). // cspell:disable-line + */ + selector: string; +}; + +/** + * A heading element (`h1`–`h6`) found within the main content area. + */ +export type MainContentsHeading = { + /** Heading text after whitespace removal, or `null` when empty. */ + text: string | null; + /** Heading level from the tag name. */ + level: 1 | 2 | 3 | 4 | 5 | 6; +}; + +/** + * An `` or `` element found in the main content. + */ +export type MainContentsImage = { + /** Resolved absolute `src` URL. */ + src: string; + /** `alt` attribute value (may be an empty string). */ + alt: string; +}; + +/** + * Structural summary of a `` found within the main content. + */ +export type MainContentsTable = { + /** Number of `` elements. */ + rows: number; + /** Number of `th`/`td` cells in the first row, or `0` when there is no row. */ + cols: number; + /** Whether the table contains a ``. */ + hasHeader: boolean; + /** Whether the table contains a ``. */ + hasFooter: boolean; + /** Whether any cell uses `colspan` or `rowspan`. */ + hasMergedCell: boolean; +}; + +/** + * A button-like element found in the main content + * (`button`, `[role=button]`, `[class*=button]`, `[class*=btn]`). + */ +export type MainContentsButton = { + /** Element tag name (e.g. `"BUTTON"`, `"A"`, `"DIV"`). */ + nodeName: string; + /** `role` attribute, or `null` when absent. */ + role: string | null; + /** `type` for `