',
+ );
+
+ expect(result.wordCount).toBe(15);
+ });
+
+ it('returns first matching element in DOM order when multiple Phase-1 selectors match', () => {
+ const result = extract(`
+
+ Semantic main
+
',
+ );
+
+ expect(result.main?.id).toBe('PrimaryContent');
+ expect(result.wordCount).toBe(7);
+ });
+
+ it('prefers Phase-1 over an earlier header-content that would match Phase-2', () => {
+ const result = extract(`
+
+
Chrome
+ Real main
+
+ `);
+
+ expect(result.main?.nodeName).toBe('MAIN');
+ expect(result.wordCount).toBe(8);
+ });
+
+ it('may match class*=main on maintenance via Phase-2 (accepted false positive)', () => {
+ const result = extract(
+ '
Down
',
+ );
+
+ expect(result.main?.classList).toContain('maintenance');
+ expect(result.wordCount).toBe(4);
+ });
+
+ it('builds a simple diagnostic selector from tag, id, and classes', () => {
+ const result = extract(
+ '
X
',
+ );
+
+ expect(result.main?.selector).toBe('div#page-body.foo.bar');
+ });
+});
+
+describe('getMainContents', () => {
+ it('passes only the selector into page.evaluate so Document defaults in the page realm', async () => {
+ const evaluate = vi.fn(
+ (fn: typeof extractMainContentsFromDocument, sel: string | null) =>
+ Promise.resolve(
+ fn(sel, createDocument('Hi')),
+ ),
+ );
+ const page = { evaluate } as unknown as Page;
+
+ const result = await getMainContents(page, { mainContentSelector: '#x' });
+
+ expect(evaluate).toHaveBeenCalledWith(extractMainContentsFromDocument, '#x');
+ expect(result.wordCount).toBe(2);
+ });
+});
diff --git a/packages/@d-zero/beholder/src/get-main-contents.ts b/packages/@d-zero/beholder/src/get-main-contents.ts
new file mode 100644
index 00000000..780f6ac9
--- /dev/null
+++ b/packages/@d-zero/beholder/src/get-main-contents.ts
@@ -0,0 +1,306 @@
+/**
+ * Main-content quantitative extraction for scraped HTML pages.
+ *
+ * Runs entirely against a `Document` (Puppeteer page realm or jsdom) so the
+ * same logic is unit-tested without a browser. Puppeteer callers use
+ * {@link getMainContents}, which passes this function into `page.evaluate`.
+ *
+ * WHY no `@medv/finder`: selector strings are diagnostic only; a tag+id+class // cspell:disable-line
+ * path is enough and avoids a Node-only dependency inside the page realm.
+ * @module
+ */
+
+import type { MainContentsData } from './types.js';
+import type { Page } from 'puppeteer';
+
+/**
+ * Extract main-content metrics from a `Document`.
+ *
+ * This function must remain free of closure over imports so Puppeteer can
+ * serialize it into `page.evaluate`.
+ *
+ * Argument order matches `page.evaluate(fn, mainContentSelector)`: the selector
+ * is the first argument; `doc` defaults to the page-realm global `document`
+ * when omitted (jsdom tests pass an explicit `Document` as the second argument).
+ * @param mainContentSelector - Optional selector prepended to the default list.
+ * @param doc - The document to inspect (defaults to the global `document`).
+ * @returns Quantitative main-content data (never `null`; empty when no main region).
+ * @example
+ * ```ts
+ * // In page.evaluate / browser:
+ * const data = extractMainContentsFromDocument('#page-body');
+ * // In jsdom tests:
+ * const data = extractMainContentsFromDocument('#page-body', document);
+ * ```
+ */
+export function extractMainContentsFromDocument(
+ mainContentSelector: string | null = null,
+ doc: Document = document,
+): MainContentsData {
+ /**
+ * @param text
+ */
+ function removeSpaces(text: string | null): string | null {
+ return text?.trim().replaceAll(/\s+/g, '') || null;
+ }
+
+ /**
+ * @param value
+ */
+ function cssEscape(value: string): string {
+ if (typeof CSS !== 'undefined' && typeof CSS.escape === 'function') {
+ return CSS.escape(value);
+ }
+ return value.replaceAll(/([ !"#$%&'()*+,./:;<=>?@[\\\]^`{|}~])/g, '\\$1');
+ }
+
+ /**
+ * @param el
+ */
+ function buildSelector(el: Element): string {
+ const tag = el.nodeName.toLowerCase();
+ const id = el.id ? `#${cssEscape(el.id)}` : '';
+ const classes = [...el.classList].map((c) => `.${cssEscape(c)}`).join('');
+ return `${tag}${id}${classes}`;
+ }
+
+ /**
+ * @param media
+ */
+ function resolveMediaSrc(media: HTMLMediaElement): string {
+ if (media.currentSrc) {
+ return media.currentSrc;
+ }
+ const attrSrc = media.getAttribute('src');
+ if (attrSrc) {
+ try {
+ return new URL(attrSrc, doc.baseURI).href;
+ } catch {
+ return attrSrc;
+ }
+ }
+ const source = media.querySelector('source[src]');
+ const sourceSrc = source?.getAttribute('src');
+ if (sourceSrc) {
+ try {
+ return new URL(sourceSrc, doc.baseURI).href;
+ } catch {
+ return sourceSrc;
+ }
+ }
+ return '';
+ }
+
+ const title = doc.title?.trim() ?? '';
+ const bodyWordCount = removeSpaces(doc.body?.textContent ?? null)?.length ?? 0;
+
+ const selectors = [
+ 'main',
+ '[role="main"]',
+ '#main',
+ '.main',
+ '#content',
+ '.content',
+ '#contents',
+ '.contents',
+ '#main-content',
+ '.main-content',
+ '#main_content',
+ '.main_content',
+ '#mainContent',
+ '.mainContent',
+ ];
+ if (mainContentSelector) {
+ selectors.unshift(mainContentSelector);
+ }
+
+ let $main: Element | null = null;
+ try {
+ $main = doc.querySelector(selectors.join(','));
+ } catch {
+ // Invalid custom selector: retry without it so built-in selectors still run.
+ if (mainContentSelector) {
+ selectors.shift();
+ $main = doc.querySelector(selectors.join(','));
+ }
+ }
+
+ if (!$main) {
+ const fallbackSelectors = [
+ '[id*="main" i]',
+ '[class*="main" i]',
+ '[id*="content" i]',
+ '[class*="content" i]',
+ ];
+ for (const sel of fallbackSelectors) {
+ const candidate = doc.querySelector(sel);
+ if (candidate && candidate !== doc.body && candidate !== doc.documentElement) {
+ $main = candidate;
+ break;
+ }
+ }
+ }
+
+ if (!$main) {
+ return {
+ title,
+ main: null,
+ wordCount: 0,
+ bodyWordCount,
+ headings: [],
+ images: [],
+ tables: [],
+ buttons: [],
+ iframes: [],
+ videos: [],
+ audios: [],
+ canvases: [],
+ };
+ }
+
+ const headings: MainContentsData['headings'] = [];
+ for (const $heading of $main.querySelectorAll(
+ 'h1, h2, h3, h4, h5, h6',
+ )) {
+ headings.push({
+ text: removeSpaces($heading.textContent),
+ level: Number.parseInt($heading.nodeName.replace(/h/i, ''), 10) as
+ 1 | 2 | 3 | 4 | 5 | 6,
+ });
+ }
+
+ const images: MainContentsData['images'] = [];
+ for (const $img of $main.querySelectorAll(
+ 'img, input[type="image"]',
+ )) {
+ images.push({
+ src: $img.src,
+ alt: $img.alt,
+ });
+ }
+
+ const tables: MainContentsData['tables'] = [];
+ for (const $table of $main.querySelectorAll('table')) {
+ tables.push({
+ rows: $table.querySelectorAll('tr').length,
+ cols: $table.querySelector('tr')?.querySelectorAll('th, td').length || 0,
+ hasHeader: !!$table.querySelector('thead'),
+ hasFooter: !!$table.querySelector('tfoot'),
+ hasMergedCell: !!$table.querySelector('[colspan], [rowspan]'),
+ });
+ }
+
+ const win = doc.defaultView;
+ const HTMLButton = win?.HTMLButtonElement;
+ const HTMLInput = win?.HTMLInputElement;
+
+ const buttons: MainContentsData['buttons'] = [];
+ for (const $el of $main.querySelectorAll(
+ 'button, [role="button"], [class*="button"], [class*="btn"]',
+ )) {
+ const isButton = HTMLButton ? $el instanceof HTMLButton : $el.nodeName === 'BUTTON';
+ const isInput = HTMLInput ? $el instanceof HTMLInput : $el.nodeName === 'INPUT';
+ const isFormControl = isButton || isInput;
+ let text = removeSpaces($el.textContent);
+ if (!text && isInput) {
+ text = removeSpaces($el.getAttribute('value'));
+ }
+ const disabledProp =
+ isFormControl && 'disabled' in $el
+ ? Boolean(($el as HTMLButtonElement).disabled)
+ : false;
+ buttons.push({
+ nodeName: $el.nodeName,
+ role: $el.getAttribute('role'),
+ type: isFormControl ? ($el as HTMLButtonElement | HTMLInputElement).type : null,
+ text,
+ disabled: disabledProp || $el.getAttribute('aria-disabled') === 'true',
+ });
+ }
+
+ const iframes: MainContentsData['iframes'] = [];
+ for (const $el of $main.querySelectorAll('iframe')) {
+ iframes.push({
+ src: $el.src,
+ title: $el.hasAttribute('title') ? $el.getAttribute('title') : null,
+ width: $el.getAttribute('width'),
+ height: $el.getAttribute('height'),
+ });
+ }
+
+ const videos: MainContentsData['videos'] = [];
+ for (const $el of $main.querySelectorAll('video')) {
+ const posterAttr = $el.getAttribute('poster');
+ let poster: string | null = null;
+ if (posterAttr) {
+ try {
+ poster = new URL(posterAttr, doc.baseURI).href;
+ } catch {
+ poster = posterAttr;
+ }
+ }
+ videos.push({
+ src: resolveMediaSrc($el),
+ poster,
+ width: $el.width,
+ height: $el.height,
+ });
+ }
+
+ const audios: MainContentsData['audios'] = [];
+ for (const $el of $main.querySelectorAll('audio')) {
+ audios.push({
+ src: resolveMediaSrc($el),
+ });
+ }
+
+ const canvases: MainContentsData['canvases'] = [];
+ for (const $el of $main.querySelectorAll('canvas')) {
+ canvases.push({
+ width: $el.width,
+ height: $el.height,
+ });
+ }
+
+ return {
+ title,
+ main: {
+ nodeName: $main.nodeName,
+ id: $main.id || null,
+ classList: [...$main.classList],
+ role: $main.getAttribute('role'),
+ selector: buildSelector($main),
+ },
+ wordCount: removeSpaces($main.textContent)?.length ?? 0,
+ bodyWordCount,
+ headings,
+ images,
+ tables,
+ buttons,
+ iframes,
+ videos,
+ audios,
+ canvases,
+ };
+}
+
+/**
+ * Extract main-content metrics from a Puppeteer page via a single `page.evaluate`.
+ * @param page - Puppeteer page whose DOM has finished loading.
+ * @param options - Optional main-content selector override.
+ * @param options.mainContentSelector
+ * @returns Quantitative main-content data.
+ * @example
+ * ```ts
+ * const mainContents = await getMainContents(page, { mainContentSelector: '#page-body' });
+ * ```
+ */
+export async function getMainContents(
+ page: Page,
+ options?: { mainContentSelector?: string | null },
+): Promise {
+ return page.evaluate(
+ extractMainContentsFromDocument,
+ options?.mainContentSelector ?? null,
+ );
+}
diff --git a/packages/@d-zero/beholder/src/index.ts b/packages/@d-zero/beholder/src/index.ts
index fa55b14e..58394443 100644
--- a/packages/@d-zero/beholder/src/index.ts
+++ b/packages/@d-zero/beholder/src/index.ts
@@ -27,6 +27,17 @@ export type {
ImageElement,
SkippedPageData,
NetworkLog,
+ ScrollHeightData,
+ MainContentsData,
+ MainContentsMainTag,
+ MainContentsHeading,
+ MainContentsImage,
+ MainContentsTable,
+ MainContentsButton,
+ MainContentsIframe,
+ MainContentsVideo,
+ MainContentsAudio,
+ MainContentsCanvas,
OpenGraphMeta,
OgArticleMeta,
OgBookMeta,
diff --git a/packages/@d-zero/beholder/src/is-html-content-type.spec.ts b/packages/@d-zero/beholder/src/is-html-content-type.spec.ts
new file mode 100644
index 00000000..a418061a
--- /dev/null
+++ b/packages/@d-zero/beholder/src/is-html-content-type.spec.ts
@@ -0,0 +1,17 @@
+import { describe, expect, it } from 'vitest';
+
+import { isHtmlContentType } from './is-html-content-type.js';
+
+describe('isHtmlContentType', () => {
+ it('accepts text/html in any letter case and with surrounding whitespace', () => {
+ expect(isHtmlContentType('text/html')).toBe(true);
+ expect(isHtmlContentType('Text/HTML')).toBe(true);
+ expect(isHtmlContentType(' text/html ')).toBe(true);
+ });
+
+ it('rejects non-HTML and null', () => {
+ expect(isHtmlContentType('application/pdf')).toBe(false);
+ expect(isHtmlContentType('text/plain')).toBe(false);
+ expect(isHtmlContentType(null)).toBe(false);
+ });
+});
diff --git a/packages/@d-zero/beholder/src/is-html-content-type.ts b/packages/@d-zero/beholder/src/is-html-content-type.ts
new file mode 100644
index 00000000..1df0074f
--- /dev/null
+++ b/packages/@d-zero/beholder/src/is-html-content-type.ts
@@ -0,0 +1,20 @@
+/**
+ * Determine whether a Content-Type media type is HTML.
+ *
+ * MIME types are case-insensitive (RFC 2045). Surrounding whitespace after
+ * parameter stripping is also tolerated so `text/html `, `Text/HTML`, and
+ * `text/html; charset=utf-8` (after `;` split) all classify as HTML.
+ * @param contentType - The media type portion of a Content-Type header
+ * (parameters already stripped), or `null` when unknown.
+ * @returns `true` when the media type is `text/html` in any letter case.
+ * @example
+ * ```ts
+ * isHtmlContentType('text/html'); // true
+ * isHtmlContentType('Text/HTML'); // true
+ * isHtmlContentType('application/pdf'); // false
+ * // Callers should strip parameters first (e.g. split on ';') before calling.
+ * ```
+ */
+export function isHtmlContentType(contentType: string | null): boolean {
+ return contentType !== null && contentType.trim().toLowerCase() === 'text/html';
+}
diff --git a/packages/@d-zero/beholder/src/measure-scroll-height.spec.ts b/packages/@d-zero/beholder/src/measure-scroll-height.spec.ts
new file mode 100644
index 00000000..14f45f07
--- /dev/null
+++ b/packages/@d-zero/beholder/src/measure-scroll-height.spec.ts
@@ -0,0 +1,47 @@
+import type { Page } from 'puppeteer';
+
+import { beforeEach, describe, expect, it, vi } from 'vitest';
+
+import { measureScrollHeight } from './measure-scroll-height.js';
+
+describe('measureScrollHeight', () => {
+ let page: Page;
+ let scrollHeights: number[];
+
+ beforeEach(() => {
+ scrollHeights = [2400, 5200];
+ let call = 0;
+ page = {
+ setViewport: vi.fn(() => Promise.resolve()),
+ evaluate: vi.fn(() => Promise.resolve(scrollHeights[call++] ?? 0)),
+ } as unknown as Page;
+ });
+
+ it('returns desktop and mobile heights without scrolling the page', async () => {
+ const result = await measureScrollHeight(page);
+
+ expect(result).toEqual({ desktop: 2400, mobile: 5200 });
+ expect(page.setViewport).toHaveBeenCalledTimes(2);
+ expect(page.setViewport).toHaveBeenNthCalledWith(1, {
+ width: 1280,
+ height: 800,
+ deviceScaleFactor: 1,
+ });
+ expect(page.setViewport).toHaveBeenNthCalledWith(2, {
+ width: 320,
+ height: 800,
+ deviceScaleFactor: 2,
+ });
+ });
+
+ it('returns null for a preset that throws', async () => {
+ page.setViewport = vi
+ .fn()
+ .mockResolvedValueOnce()
+ .mockRejectedValueOnce(new Error('viewport failed'));
+
+ const result = await measureScrollHeight(page);
+
+ expect(result).toEqual({ desktop: 2400, mobile: null });
+ });
+});
diff --git a/packages/@d-zero/beholder/src/measure-scroll-height.ts b/packages/@d-zero/beholder/src/measure-scroll-height.ts
new file mode 100644
index 00000000..1b823b58
--- /dev/null
+++ b/packages/@d-zero/beholder/src/measure-scroll-height.ts
@@ -0,0 +1,52 @@
+/**
+ * Lightweight scrollHeight measurement without `scrollAllOver`.
+ *
+ * Used when `captureImages` is false so callers still receive desktop/mobile
+ * heights without the cost of full-page scrolling for image extraction.
+ * @module
+ */
+
+import type { ScrollHeightData } from './types.js';
+import type { Page } from 'puppeteer';
+
+import { devicePresets } from '@d-zero/puppeteer-page-scan';
+
+/**
+ * Measure `document.body.scrollHeight` at desktop-compact and mobile-small viewports.
+ *
+ * Sets each viewport, reads height once, and does **not** call `scrollAllOver`.
+ * A failed preset yields `null` for that side only.
+ * @param page - Puppeteer page whose DOM has finished loading.
+ * @returns Heights for desktop and mobile presets.
+ * @example
+ * ```ts
+ * const scrollHeight = await measureScrollHeight(page);
+ * // { desktop: 2400, mobile: 5200 }
+ * ```
+ */
+export async function measureScrollHeight(page: Page): Promise {
+ const desktop = await measureAtPreset(page, 'desktop-compact');
+ const mobile = await measureAtPreset(page, 'mobile-small');
+ return { desktop, mobile };
+}
+
+/**
+ * @param page
+ * @param key
+ */
+async function measureAtPreset(
+ page: Page,
+ key: 'desktop-compact' | 'mobile-small',
+): Promise {
+ const preset: { width: number; resolution?: number } = devicePresets[key];
+ try {
+ await page.setViewport({
+ width: preset.width,
+ height: 800,
+ deviceScaleFactor: preset.resolution ?? 1,
+ });
+ return await page.evaluate(() => document.body.scrollHeight);
+ } catch {
+ return null;
+ }
+}
diff --git a/packages/@d-zero/beholder/src/scraper.ts b/packages/@d-zero/beholder/src/scraper.ts
index d371e379..e04fb658 100644
--- a/packages/@d-zero/beholder/src/scraper.ts
+++ b/packages/@d-zero/beholder/src/scraper.ts
@@ -10,6 +10,7 @@ import type {
PageData,
ParseURLOptions,
Resource,
+ ScrollHeightData,
SkippedPageData,
} from './types.js';
import type { PageScanPhase } from '@d-zero/puppeteer-page-scan';
@@ -28,8 +29,11 @@ import {
getImageList,
getMeta,
} from './dom-evaluation.js';
+import { getMainContents } from './get-main-contents.js';
import { isError } from './is-error.js';
+import { isHtmlContentType } from './is-html-content-type.js';
import { keywordCheck } from './keyword-check.js';
+import { measureScrollHeight } from './measure-scroll-height.js';
import { emptyMeta } from './meta/classify.js';
import { findDisconnectionFailures } from './network-disconnection.js';
import { parseUrl } from './parse-url.js';
@@ -148,6 +152,8 @@ export default class Scraper extends EventEmitter {
imageList: [],
anchorList: [],
html: '',
+ mainContents: null,
+ scrollHeight: null,
isSkipped: false,
};
@@ -176,12 +182,14 @@ export default class Scraper extends EventEmitter {
pageData: {
...headResult,
isTarget: false,
+ mainContents: headResult.mainContents ?? null,
+ scrollHeight: headResult.scrollHeight ?? null,
},
resources,
};
}
- if (headResult === null || headResult.contentType === 'text/html') {
+ if (headResult === null || isHtmlContentType(headResult.contentType)) {
const fetchResult = await this.#fetchData(
page,
url,
@@ -555,7 +563,7 @@ export default class Scraper extends EventEmitter {
const _contentLength = Number.parseInt(responseHeaders['content-length'] ?? '');
const contentLength = Number.isFinite(_contentLength) ? _contentLength : null;
- if (contentType !== 'text/html') {
+ if (!isHtmlContentType(contentType)) {
return {
url,
isTarget: false,
@@ -570,6 +578,8 @@ export default class Scraper extends EventEmitter {
imageList: [],
anchorList: [],
html: '',
+ mainContents: null,
+ scrollHeight: null,
isSkipped: false,
};
}
@@ -619,6 +629,8 @@ export default class Scraper extends EventEmitter {
imageList: [],
anchorList: [],
html,
+ mainContents: null,
+ scrollHeight: null,
isSkipped: false,
};
}
@@ -644,6 +656,10 @@ export default class Scraper extends EventEmitter {
throw new Error(`Network disconnection detected during page load: ${errorSummary}`);
}
+ const mainContents = await getMainContents(page, {
+ mainContentSelector: options?.mainContentSelector,
+ });
+
void this.emit('changePhase', {
pid: process.pid,
name: 'getAnchors',
@@ -671,24 +687,29 @@ export default class Scraper extends EventEmitter {
domEvaluationTimeout,
);
- const imageList = captureImages
- ? await (async () => {
- void this.emit('changePhase', {
- pid: process.pid,
- name: 'extractImages',
- url,
- isExternal,
- message: `%countdown(${domEvaluationTimeout},extractImages_${url.withoutHash},s)%s`,
- });
- return this.#fetchImages(
- page,
- url.withoutHashAndAuth,
- isExternal,
- imageLoadTimeout,
- domEvaluationTimeout,
- );
- })()
- : [];
+ let imageList: ImageElement[] = [];
+ let scrollHeight: ScrollHeightData | null = null;
+
+ if (captureImages) {
+ void this.emit('changePhase', {
+ pid: process.pid,
+ name: 'extractImages',
+ url,
+ isExternal,
+ message: `%countdown(${domEvaluationTimeout},extractImages_${url.withoutHash},s)%s`,
+ });
+ const fetched = await this.#fetchImages(
+ page,
+ url.withoutHashAndAuth,
+ isExternal,
+ imageLoadTimeout,
+ domEvaluationTimeout,
+ );
+ imageList = fetched.imageList;
+ scrollHeight = fetched.scrollHeight;
+ } else {
+ scrollHeight = await measureScrollHeight(page);
+ }
return {
url,
@@ -704,6 +725,8 @@ export default class Scraper extends EventEmitter {
anchorList,
imageList,
html,
+ mainContents,
+ scrollHeight,
isSkipped: false,
};
}
@@ -719,12 +742,12 @@ export default class Scraper extends EventEmitter {
* changes and triggers a reload. Isolating each device preset allows partial
* results — if one viewport fails, the other can still succeed.
*
- * WHY retryable with 20-min timeout and `fallback: []`: Image extraction is
- * best-effort. If all retries fail, an empty array is returned rather than
- * failing the entire page scrape. The 20-min wall clock accommodates pages
- * whose mobile-small `scrollHeight` reaches ~300k px (observed on
- * responsive data tables, which take ~5 min to scroll). A shorter timeout
- * causes a second retry to start while the previous attempt's
+ * WHY retryable with 20-min timeout and empty fallback: Image extraction is
+ * best-effort. If all retries fail, empty images and null scroll heights are
+ * returned rather than failing the entire page scrape. The 20-min wall clock
+ * accommodates pages whose mobile-small `scrollHeight` reaches ~300k px
+ * (observed on responsive data tables, which take ~5 min to scroll). A shorter
+ * timeout causes a second retry to start while the previous attempt's
* `scrollAllOver` is still running its `page.evaluate` calls in the
* background — `Promise.race` in `retry.ts` does not cancel `fn()`. The
* collision then surfaces as "Attempted to use detached Frame" or
@@ -736,16 +759,22 @@ export default class Scraper extends EventEmitter {
* preset entirely keeps the timeout-vs-background-evaluate collision from
* ever being triggered, at the cost of losing that viewport's image data
* for those pages. See {@link MAX_SCROLL_HEIGHT} for the chosen threshold.
+ *
+ * Scroll heights are still recorded when scrolling is skipped for the
+ * height limit — the measurement from `beforePageScan` is kept.
* @param page - Puppeteer page instance
* @param url - The page URL string (without hash and auth)
* @param isExternal - Whether the page is external
* @param imageLoadTimeout - Timeout (ms) for waiting images to complete loading
* @param domEvaluationTimeout - Timeout (ms) for the in-page image extraction `page.evaluate`
- * @returns Array of image elements from all device presets (may be partial if some viewports failed)
+ * @returns Image elements plus desktop/mobile scroll heights from the scan path
*/
@retryable({
timeout: 20 * 60 * 1000,
- fallback: [],
+ fallback: {
+ imageList: [],
+ scrollHeight: { desktop: null, mobile: null },
+ },
onWait(this: Scraper, determinedInterval, retryCount, methodName, error) {
void this.emit('changePhase', {
pid: process.pid,
@@ -771,15 +800,20 @@ export default class Scraper extends EventEmitter {
isExternal: boolean,
imageLoadTimeout: number,
domEvaluationTimeout: number,
- ): Promise {
+ ): Promise<{ imageList: ImageElement[]; scrollHeight: ScrollHeightData }> {
const listener = this.#createPageScanListener(isExternal);
- const devices: { key: string; preset: { width: number; resolution?: number } }[] = [
+ const devices: {
+ key: 'desktop-compact' | 'mobile-small';
+ preset: { width: number; resolution?: number };
+ }[] = [
{ key: 'desktop-compact', preset: devicePresets['desktop-compact'] },
{ key: 'mobile-small', preset: devicePresets['mobile-small'] },
];
const imageList: ImageElement[] = [];
+ const scrollHeight: ScrollHeightData = { desktop: null, mobile: null };
for (const { key, preset } of devices) {
+ const scrollKey = key === 'desktop-compact' ? 'desktop' : 'mobile';
try {
void this.emit('changePhase', {
pid: process.pid,
@@ -798,6 +832,8 @@ export default class Scraper extends EventEmitter {
maxScrollHeight: MAX_SCROLL_HEIGHT,
});
+ scrollHeight[scrollKey] = scanResult.scrollHeight;
+
if (!scanResult.scrolled) {
void this.emit('changePhase', {
pid: process.pid,
@@ -845,6 +881,6 @@ export default class Scraper extends EventEmitter {
}
}
- return imageList;
+ return { imageList, scrollHeight };
}
}
diff --git a/packages/@d-zero/beholder/src/types.ts b/packages/@d-zero/beholder/src/types.ts
index 2345775b..93bd7872 100644
--- a/packages/@d-zero/beholder/src/types.ts
+++ b/packages/@d-zero/beholder/src/types.ts
@@ -103,10 +103,201 @@ export type PageData = {
/** HTML snapshot of the rendered DOM. */
html: string;
+ /**
+ * Quantitative main-content metrics extracted in the browser after load.
+ * `null` when the page is non-HTML, external, non-HTTP, or otherwise not fully scraped.
+ * When HTML is scraped but no main region is found, this is still an object with empty arrays and `wordCount: 0`.
+ */
+ mainContents: MainContentsData | null;
+
+ /**
+ * `document.body.scrollHeight` at desktop-compact and mobile-small viewports.
+ * `null` when not measured (non-HTML, external, non-HTTP, or skipped).
+ */
+ scrollHeight: ScrollHeightData | null;
+
/** Always `false` for successfully scraped pages. See {@link SkippedPageData} for skipped pages. */
isSkipped: false;
};
+/**
+ * `document.body.scrollHeight` measured at the scraper's desktop and mobile device presets.
+ */
+export type ScrollHeightData = {
+ /** Height at `desktop-compact` (width 1280), or `null` if that preset failed. */
+ desktop: number | null;
+ /** Height at `mobile-small` (width 320 @ 2x), or `null` if that preset failed. */
+ mobile: number | null;
+};
+
+/**
+ * Quantitative metrics for the detected main content region of a page.
+ * @example
+ * ```ts
+ * const { mainContents } = pageData;
+ * if (mainContents) {
+ * console.log(mainContents.wordCount, mainContents.headings.length);
+ * }
+ * ```
+ */
+export type MainContentsData = {
+ /** `document.title` trimmed. Empty string when the title is empty. */
+ title: string;
+ /**
+ * Identifying info for the detected main region element.
+ * `null` when no selector matched (arrays are empty and `wordCount` is `0`).
+ */
+ main: MainContentsMainTag | null;
+ /**
+ * Character count of the main region's `textContent` after whitespace removal.
+ * `0` when no main region was found or the text is whitespace-only.
+ */
+ wordCount: number;
+ /**
+ * Character count of `document.body` `textContent` after whitespace removal.
+ * Measured even when no main region is found (`0` if `body` is missing).
+ */
+ bodyWordCount: number;
+ /** Headings (`h1`–`h6`) inside the main region, in DOM order. */
+ headings: MainContentsHeading[];
+ /** Images (`img`, `input[type=image]`) inside the main region, in DOM order. */
+ images: MainContentsImage[];
+ /** Tables inside the main region, in DOM order. */
+ tables: MainContentsTable[];
+ /** Button-like elements inside the main region, in DOM order. */
+ buttons: MainContentsButton[];
+ /** Iframes inside the main region, in DOM order. */
+ iframes: MainContentsIframe[];
+ /** Videos inside the main region, in DOM order. */
+ videos: MainContentsVideo[];
+ /** Audios inside the main region, in DOM order. */
+ audios: MainContentsAudio[];
+ /** Canvases inside the main region, in DOM order. */
+ canvases: MainContentsCanvas[];
+};
+
+/**
+ * Identifying information about the DOM element used as the main content area.
+ */
+export type MainContentsMainTag = {
+ /** Element tag name (e.g. `"MAIN"`, `"DIV"`), as reported by `nodeName`. */
+ nodeName: string;
+ /** The element's `id`, or `null` when empty/absent. */
+ id: string | null;
+ /** CSS class names on the element. */
+ classList: string[];
+ /** The WAI-ARIA `role` attribute value, or `null` when absent. */
+ role: string | null;
+ /**
+ * A simple diagnostic CSS selector built from tag + id + classes
+ * (not `@medv/finder`). // cspell:disable-line
+ */
+ selector: string;
+};
+
+/**
+ * A heading element (`h1`–`h6`) found within the main content area.
+ */
+export type MainContentsHeading = {
+ /** Heading text after whitespace removal, or `null` when empty. */
+ text: string | null;
+ /** Heading level from the tag name. */
+ level: 1 | 2 | 3 | 4 | 5 | 6;
+};
+
+/**
+ * An `` or `` element found in the main content.
+ */
+export type MainContentsImage = {
+ /** Resolved absolute `src` URL. */
+ src: string;
+ /** `alt` attribute value (may be an empty string). */
+ alt: string;
+};
+
+/**
+ * Structural summary of a `
` found within the main content.
+ */
+export type MainContentsTable = {
+ /** Number of `
` elements. */
+ rows: number;
+ /** Number of `th`/`td` cells in the first row, or `0` when there is no row. */
+ cols: number;
+ /** Whether the table contains a ``. */
+ hasHeader: boolean;
+ /** Whether the table contains a `
`. */
+ hasFooter: boolean;
+ /** Whether any cell uses `colspan` or `rowspan`. */
+ hasMergedCell: boolean;
+};
+
+/**
+ * A button-like element found in the main content
+ * (`button`, `[role=button]`, `[class*=button]`, `[class*=btn]`).
+ */
+export type MainContentsButton = {
+ /** Element tag name (e.g. `"BUTTON"`, `"A"`, `"DIV"`). */
+ nodeName: string;
+ /** `role` attribute, or `null` when absent. */
+ role: string | null;
+ /** `type` for `
+ 
+
+