From d09f8d21b910ac9079553829344620ad7e9ad149 Mon Sep 17 00:00:00 2001 From: Yusuke Hirao Date: Thu, 23 Jul 2026 16:02:40 +0900 Subject: [PATCH] feat(page-cluster)!: report landmark positions and main/chrome classification Add line/column position reporting for header/footer/nav/aside/form/ search/main landmark instances, with chrome-vs-content classification for the six excisable types against each final cluster's shell tokens. - extractLandmarks now also extracts
/role="main" - resolvePageClusterKeys* accept includeLandmarkPositions to return PageClusterKeyResult[] (clusterKey + PageLandmarkReport) instead of string[]; not supported on the streaming path (over 20,000 pages) - CLI gains --include-landmark-positions BREAKING CHANGE: ExtractLandmarksResult's six landmark fields change from string[] (raw HTML) to LandmarkInstance[] (html + position). A new `main` field is added. Callers reading .header/.footer/etc. as strings must switch to .html on each instance. --- packages/@d-zero/page-cluster/README.md | 54 +++- .../src/build-page-landmark-report.spec.ts | 64 +++++ .../src/build-page-landmark-report.ts | 107 ++++++++ packages/@d-zero/page-cluster/src/cli.spec.ts | 75 ++++++ packages/@d-zero/page-cluster/src/cli.ts | 66 ++++- .../src/extract-landmarks.spec.ts | 243 +++++++++++++++--- .../page-cluster/src/extract-landmarks.ts | 155 +++++++++-- .../src/is-chrome-landmark-instance.spec.ts | 43 ++++ .../src/is-chrome-landmark-instance.ts | 53 ++++ .../src/merge-cross-block-clusters.spec.ts | 38 ++- .../src/merge-cross-block-clusters.ts | 106 +------- .../src/offset-to-line-column.spec.ts | 60 +++++ .../page-cluster/src/offset-to-line-column.ts | 60 +++++ .../src/per-page-landmark-signatures.spec.ts | 52 +++- .../src/per-page-landmark-signatures.ts | 35 ++- ...esolve-page-cluster-keys-streaming.spec.ts | 41 +++ .../src/resolve-page-cluster-keys.spec.ts | 100 ++++++- .../src/resolve-page-cluster-keys.ts | 178 ++++++++++++- .../page-cluster/src/shell-quorum.spec.ts | 81 ++++++ .../@d-zero/page-cluster/src/shell-quorum.ts | 117 +++++++++ 20 files changed, 1535 insertions(+), 193 deletions(-) create mode 100644 packages/@d-zero/page-cluster/src/build-page-landmark-report.spec.ts create mode 100644 packages/@d-zero/page-cluster/src/build-page-landmark-report.ts create mode 100644 packages/@d-zero/page-cluster/src/is-chrome-landmark-instance.spec.ts create mode 100644 packages/@d-zero/page-cluster/src/is-chrome-landmark-instance.ts create mode 100644 packages/@d-zero/page-cluster/src/offset-to-line-column.spec.ts create mode 100644 packages/@d-zero/page-cluster/src/offset-to-line-column.ts create mode 100644 packages/@d-zero/page-cluster/src/shell-quorum.spec.ts create mode 100644 packages/@d-zero/page-cluster/src/shell-quorum.ts diff --git a/packages/@d-zero/page-cluster/README.md b/packages/@d-zero/page-cluster/README.md index 7d4a2685..9f2d1198 100644 --- a/packages/@d-zero/page-cluster/README.md +++ b/packages/@d-zero/page-cluster/README.md @@ -15,7 +15,7 @@ yarn add @d-zero/page-cluster ### CLI ```sh -page-cluster [--content-block-attribute ] < pages.jsonl > clusters.jsonl +page-cluster [--content-block-attribute ] [--include-landmark-positions] < pages.jsonl > clusters.jsonl ``` **入力**: JSONL 1 行 1 ページ。フィールドは以下。`html` 以外はすべて任意（`paths` / `stylesheetHrefs` がないと粗い分類になる）。 @@ -36,6 +36,45 @@ page-cluster [--content-block-attribute ] < pages.jsonl > clusters.jsonl { "id": "任意の識別子", "clusterKey": "..." } ``` +`--include-landmark-positions` を指定すると、各行に `landmarks` フィールドが追加される。header / footer / nav / aside / form / search / main のインスタンスごとに、HTML 内の位置（1-based の line/column と文字列オフセットの両方）を返す。header 〜 search の 6 種は追加で、そのページが属する最終クラスタ内での頻度分析（`shellQuorum`）に基づく `isChrome`（サイト/セクション共通の chrome か、ページ固有のコンテンツか）を持つ。`main` は常にコンテンツなので `isChrome` を持たない。 + +```json +{ + "id": "任意の識別子", + "clusterKey": "...", + "landmarks": { + "header": [ + { + "startLine": 1, + "startColumn": 7, + "endLine": 1, + "endColumn": 30, + "startOffset": 6, + "endOffset": 29, + "isChrome": true + } + ], + "footer": [], + "nav": [], + "aside": [], + "form": [], + "search": [], + "main": [ + { + "startLine": 2, + "startColumn": 1, + "endLine": 10, + "endColumn": 8, + "startOffset": 40, + "endOffset": 120 + } + ] + } +} +``` + +20,000 ページを超えるコーパス（ストリーミング経路）では `--include-landmark-positions` は使えない（エラーで終了する）。ストリーミング経路はリザーバサンプリングと近似割当を使うため、ページ単位の chrome 判定に必要な「そのページが属する最終クラスタの shell トークン」という概念を持たないため。 + クローラ出力が JSON 配列の場合は `jq` で line-delimited に変換して食わせる: ```sh @@ -45,6 +84,7 @@ jq -c '.[]' crawl-output.json | page-cluster > clusters.jsonl #### オプション - `--content-block-attribute ` — CMS が自由編集コンテンツブロックに付与している属性名（例: `data-bgb`）が分かっている場合に指定する。指定すると比較前にその属性を持つ要素配下を無視するので、同じテンプレートで本文構成だけ違うページを混同しなくなる。唯一の site-specific なオプションで、未指定でも `
` / `role="main"` を起点にした自動深さキャップが常時働く（詳細は `resolve-page-cluster-keys.ts` の JSDoc を参照） +- `--include-landmark-positions` — 出力の各行に上記の `landmarks` フィールドを追加する。20,000 ページ超のコーパスでは使えない。指定すると進捗表示（後述）は出なくなる（進捗を出さない非ストリーミング経路に常に振り分けられるため） - `--help` / `-h` — ヘルプを表示する - `--version` / `-v` — バージョンを表示する @@ -76,12 +116,12 @@ silence したい場合は `2>/dev/null`。ログに残したい場合は `2> pr サブパスエクスポート構成。import パスと提供関数の対応は以下。 -| import パス | 提供関数 | -| ---------------------------------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| `@d-zero/page-cluster` | `tokenize` — `` 配下を構造トークン列に変換する低レベルプリミティブ | -| `@d-zero/page-cluster/resolve-page-cluster-keys` | `resolvePageClusterKeys`（非同期・ファクトリ入力・メモリ有界のメインエントリー）、`resolvePageClusterKeysFromArray`（array 入力ラッパー）、`resolvePageClusterKeysInMemory`（同期・array 入力） | -| `@d-zero/page-cluster/extract-landmarks` | `extractLandmarks` — 6 種の HTML5 ランドマーク（header / footer / nav / aside / form / search）を抽出 | -| `@d-zero/page-cluster/resolve-landmark-variant-keys` | `resolveLandmarkVariantKeys` — 特定ランドマークのデザインバリアントでページを分類 | +| import パス | 提供関数 | +| ---------------------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `@d-zero/page-cluster` | `tokenize` — `` 配下を構造トークン列に変換する低レベルプリミティブ | +| `@d-zero/page-cluster/resolve-page-cluster-keys` | `resolvePageClusterKeys`（非同期・ファクトリ入力・メモリ有界のメインエントリー）、`resolvePageClusterKeysFromArray`（array 入力ラッパー）、`resolvePageClusterKeysInMemory`（同期・array 入力）。いずれも `includeLandmarkPositions: true` を渡すと `clusterKey` に加えて位置情報つきの `landmarks`（`PageLandmarkReport`）を返す | +| `@d-zero/page-cluster/extract-landmarks` | `extractLandmarks` — header / footer / nav / aside / form / search / main の 7 種を抽出し、インスタンスごとの生 HTML と HTML 内の位置（line/column・文字列オフセット）を返す | +| `@d-zero/page-cluster/resolve-landmark-variant-keys` | `resolveLandmarkVariantKeys` — 特定ランドマークのデザインバリアントでページを分類 | ```ts import { resolvePageClusterKeysFromArray } from '@d-zero/page-cluster/resolve-page-cluster-keys'; diff --git a/packages/@d-zero/page-cluster/src/build-page-landmark-report.spec.ts b/packages/@d-zero/page-cluster/src/build-page-landmark-report.spec.ts new file mode 100644 index 00000000..33c1659a --- /dev/null +++ b/packages/@d-zero/page-cluster/src/build-page-landmark-report.spec.ts @@ -0,0 +1,64 @@ +import { describe, expect, test } from 'vitest'; + +import { buildPageLandmarkReport } from './build-page-landmark-report.js'; +import { extractLandmarks } from './extract-landmarks.js'; +import { tokenize } from './tokenize.js'; + +describe('buildPageLandmarkReport', () => { + test('a header instance whose tokens are all shell tokens is reported as chrome', () => { + const landmarks = extractLandmarks( + '
content
', + ); + const shellTokens = new Set( + tokenize(`${landmarks.header[0]!.html}`).tokens, + ); + const report = buildPageLandmarkReport(landmarks, shellTokens); + expect(report.header).toHaveLength(1); + expect(report.header[0]!.isChrome).toBe(true); + }); + + test('an instance with no overlap with shell tokens is reported as content', () => { + const landmarks = extractLandmarks( + '

x

content
', + ); + const shellTokens = new Set(['body>totally>unrelated']); + const report = buildPageLandmarkReport(landmarks, shellTokens); + expect(report.header[0]!.isChrome).toBe(false); + }); + + test('main instances carry no isChrome field', () => { + const landmarks = extractLandmarks('
content
'); + const report = buildPageLandmarkReport(landmarks, new Set()); + expect(report.main).toHaveLength(1); + expect(Object.keys(report.main[0]!)).not.toContain('isChrome'); + }); + + test('a page with no landmarks yields all-empty arrays', () => { + const landmarks = extractLandmarks('content only'); + const report = buildPageLandmarkReport(landmarks, new Set()); + expect(report).toStrictEqual({ + header: [], + footer: [], + nav: [], + aside: [], + form: [], + search: [], + main: [], + }); + }); + + test('each reported instance carries the instance position, not its html', () => { + const landmarks = extractLandmarks('
H
M
'); + const report = buildPageLandmarkReport(landmarks, new Set()); + const source = landmarks.header[0]!; + expect(report.header[0]).toStrictEqual({ + startOffset: source.startOffset, + endOffset: source.endOffset, + startLine: source.startLine, + startColumn: source.startColumn, + endLine: source.endLine, + endColumn: source.endColumn, + isChrome: false, + }); + }); +}); diff --git a/packages/@d-zero/page-cluster/src/build-page-landmark-report.ts b/packages/@d-zero/page-cluster/src/build-page-landmark-report.ts new file mode 100644 index 00000000..6f401947 --- /dev/null +++ b/packages/@d-zero/page-cluster/src/build-page-landmark-report.ts @@ -0,0 +1,107 @@ +import type { + ExtractLandmarksResult, + LandmarkInstance, + LandmarkPosition, +} from './extract-landmarks.js'; +import type { TokenizeOptions } from './types.js'; + +import { isChromeLandmarkInstance } from './is-chrome-landmark-instance.js'; +import { ALL_LANDMARK_TYPES } from './per-page-landmark-signatures.js'; +import { tokenize } from './tokenize.js'; + +/** + * A landmark instance's position plus whether + * {@link ./is-chrome-landmark-instance.js | isChromeLandmarkInstance} judged + * it shared site/section chrome (`true`) or page-specific content (`false`), + * against the unit's {@link ./shell-quorum.js | shellQuorum} shell tokens. + */ +export type ReportedLandmarkInstance = LandmarkPosition & { readonly isChrome: boolean }; + +/** + * Per-page landmark position report built by + * {@link ./build-page-landmark-report.js | buildPageLandmarkReport}. `main` + * carries no `isChrome` verdict — it never participates in chrome/shell + * discovery (see `extractLandmarks`'s "main handling" note) and is always + * content. + */ +export type PageLandmarkReport = { + header: ReportedLandmarkInstance[]; + footer: ReportedLandmarkInstance[]; + nav: ReportedLandmarkInstance[]; + aside: ReportedLandmarkInstance[]; + form: ReportedLandmarkInstance[]; + search: ReportedLandmarkInstance[]; + main: LandmarkPosition[]; +}; + +/** + * Strips `html` off a {@link LandmarkInstance}, keeping only its position. + * `buildPageLandmarkReport`'s output is meant to be serialized per page + * across a whole corpus (the CLI's JSONL output), so the report + * deliberately excludes each instance's raw HTML to keep that payload from + * scaling with markup size — callers who also need the HTML already have + * `ExtractLandmarksResult` in hand. + * @param instance + */ +function toPosition(instance: LandmarkInstance): LandmarkPosition { + return { + startOffset: instance.startOffset, + endOffset: instance.endOffset, + startLine: instance.startLine, + startColumn: instance.startColumn, + endLine: instance.endLine, + endColumn: instance.endColumn, + }; +} + +/** + * Builds a page's landmark position report: every landmark instance's + * location, with `header`/`footer`/`nav`/`aside`/`form`/`search` instances + * additionally classified as chrome or content against `shellTokens`. + * + * Reads `landmarks` directly — the full, non-deduplicated instance list + * `extractLandmarks` produced — rather than going through + * {@link ./per-page-landmark-signatures.js | computePerPageLandmarkInstances}'s + * per-page-deduplicated `PerPageLandmarkInstance[]`: that dedupe collapses + * same-signature instances to one entry, which would silently drop the + * position of every duplicate instance a position report needs to include. + * @param landmarks + * @param shellTokens The unit-level shell token set from + * {@link ./shell-quorum.js | shellQuorum}, computed once per final cluster + * and shared across every member page's report. + * @param tokenizeOptions + * @example + * ```ts + * const shellTokens = shellQuorum(clusterPerPageInstances); + * const report = buildPageLandmarkReport(extractLandmarks(page.html), shellTokens); + * ``` + */ +export function buildPageLandmarkReport( + landmarks: ExtractLandmarksResult, + shellTokens: ReadonlySet, + tokenizeOptions?: TokenizeOptions, +): PageLandmarkReport { + const report: PageLandmarkReport = { + header: [], + footer: [], + nav: [], + aside: [], + form: [], + search: [], + main: landmarks.main.map(toPosition), + }; + + for (const type of ALL_LANDMARK_TYPES) { + for (const instance of landmarks[type]) { + const tokens = instance.html + ? new Set(tokenize(`${instance.html}`, tokenizeOptions).tokens) + : new Set(); + report[type].push({ + ...toPosition(instance), + isChrome: isChromeLandmarkInstance(tokens, shellTokens), + }); + } + } + + return report; +} diff --git a/packages/@d-zero/page-cluster/src/cli.spec.ts b/packages/@d-zero/page-cluster/src/cli.spec.ts index e566bbfd..78cb49f4 100644 --- a/packages/@d-zero/page-cluster/src/cli.spec.ts +++ b/packages/@d-zero/page-cluster/src/cli.spec.ts @@ -68,6 +68,12 @@ describe('parseArgs', () => { }); }); + test('--include-landmark-positions', () => { + expect(parseArgs(['--include-landmark-positions'])).toEqual({ + includeLandmarkPositions: true, + }); + }); + test('unknown flag is captured', () => { expect(parseArgs(['--nope'])).toEqual({ unknownFlag: '--nope' }); }); @@ -143,6 +149,19 @@ describe('runCli', () => { expect(stdout.read()).toMatch(/Usage:/); }); + test('--help mentions --include-landmark-positions', async () => { + const stdout = makeCollector(); + const stderr = makeCollector(); + await runCli({ + stdin: makeStdin(''), + stdout: stdout.stream, + stderr: stderr.stream, + argv: ['--help'], + version: '0.0.0', + }); + expect(stdout.read()).toMatch(/--include-landmark-positions/); + }); + test('--version prints version and exits 0', async () => { const stdout = makeCollector(); const stderr = makeCollector(); @@ -290,4 +309,60 @@ describe('runCli', () => { /\[page-cluster\] error: .*failed to parse JSONL/, ); }); + + test('--include-landmark-positions adds a landmarks field to each output line', async () => { + const input = [ + JSON.stringify({ + id: 'a', + paths: ['news', '1'], + stylesheetHrefs: [], + html: '
H
one
', + }), + JSON.stringify({ + id: 'b', + paths: ['news', '2'], + stylesheetHrefs: [], + html: '
H
two
', + }), + ].join('\n'); + const stdout = makeCollector(); + const stderr = makeCollector(); + const code = await runCli({ + stdin: makeStdin(input), + stdout: stdout.stream, + stderr: stderr.stream, + argv: ['--include-landmark-positions'], + version: '0.0.0', + }); + expect(code).toBe(0); + const lines = stdout.read().split('\n').filter(Boolean); + expect(lines).toHaveLength(2); + const parsed = lines.map( + (line) => + JSON.parse(line) as { + id: string; + clusterKey: string; + landmarks: { header: { isChrome: boolean }[]; main: object[] }; + }, + ); + expect(parsed[0]!.landmarks.header[0]!.isChrome).toBe(true); + expect(parsed[0]!.landmarks.main).toHaveLength(1); + }); + + test('without --include-landmark-positions, output lines carry no landmarks field', async () => { + const stdout = makeCollector(); + const stderr = makeCollector(); + const code = await runCli({ + stdin: makeStdin( + JSON.stringify({ id: 'a', html: '
H
' }), + ), + stdout: stdout.stream, + stderr: stderr.stream, + argv: [], + version: '0.0.0', + }); + expect(code).toBe(0); + const line = stdout.read().split('\n').find(Boolean); + expect(JSON.parse(line!)).not.toHaveProperty('landmarks'); + }); }); diff --git a/packages/@d-zero/page-cluster/src/cli.ts b/packages/@d-zero/page-cluster/src/cli.ts index 73cfea0e..c6d5e1af 100644 --- a/packages/@d-zero/page-cluster/src/cli.ts +++ b/packages/@d-zero/page-cluster/src/cli.ts @@ -6,6 +6,7 @@ // animated header on a TTY, appended `[page-cluster] …` lines otherwise. import type { + PageClusterKeyResult, PageClusterSignals, ProgressEvent, ResolvePageClusterKeysOptions, @@ -23,13 +24,14 @@ import { resolvePageClusterKeys } from './resolve-page-cluster-keys.js'; */ type CliArgs = { readonly contentBlockAttribute?: string; + readonly includeLandmarkPositions?: boolean; readonly help?: boolean; readonly version?: boolean; readonly unknownFlag?: string; }; const HELP_TEXT = `Usage: - page-cluster [--content-block-attribute ] < pages.jsonl > clusters.jsonl + page-cluster [--content-block-attribute ] [--include-landmark-positions] < pages.jsonl > clusters.jsonl Input (JSONL, one page per line): { @@ -43,10 +45,35 @@ Input (JSONL, one page per line): Output (JSONL, one line per input page, in input order): { "id": "...", "clusterKey": "..." } + With --include-landmark-positions, each line additionally carries a + \`landmarks\` field: every header/footer/nav/aside/form/search/main + instance's position (1-based line/column plus string offsets), with the + six excisable types (all but main) also carrying an \`isChrome\` verdict + against that page's final cluster: + { + "id": "...", "clusterKey": "...", + "landmarks": { + "header": [{ "startLine": 1, "startColumn": 7, "endLine": 1, + "endColumn": 30, "startOffset": 6, "endOffset": 29, + "isChrome": true }], + "footer": [...], "nav": [...], "aside": [...], "form": [...], "search": [...], + "main": [{ "startLine": 2, "startColumn": 1, "endLine": 10, + "endColumn": 8, "startOffset": 40, "endOffset": 120 }] + } + } + Options: --content-block-attribute CMS-provided attribute marking freeform content blocks that should be stripped before comparison (e.g. \`data-bgb\`). + --include-landmark-positions Add the \`landmarks\` field described + above to every output line. Not + supported for corpora over 20,000 pages + (throws instead of streaming). Disables + progress output on stderr — this option + always routes through the same + non-progress-emitting code path as a + run without progress. --help Print this help and exit. --version Print the package version and exit. @@ -84,6 +111,7 @@ const VERBOSE_HEADER = '[page-cluster]'; export function parseArgs(argv: readonly string[]): CliArgs { const out: { contentBlockAttribute?: string; + includeLandmarkPositions?: boolean; help?: boolean; version?: boolean; unknownFlag?: string; @@ -111,6 +139,10 @@ export function parseArgs(argv: readonly string[]): CliArgs { i++; break; } + case '--include-landmark-positions': { + out.includeLandmarkPositions = true; + break; + } default: { out.unknownFlag = arg; return out; @@ -406,21 +438,39 @@ export async function runCli(options: { renderProgress(lanes, useTty, formatProgressLine(event, elapsed())); }, }; - let keys: string[]; + + // `includeLandmarkPositions` always routes resolvePageClusterKeys + // through its non-progress-emitting sync path (see that option's own + // JSDoc), so the onProgress callback above is set but never invoked + // in this branch — no separate "quiet" resolveOptions variant needed. + let clusterKeys: string[]; + let landmarksByIndex: PageClusterKeyResult['landmarks'][] | undefined; try { - keys = await resolvePageClusterKeys(() => pages, resolveOptions); + if (args.includeLandmarkPositions) { + const results = await resolvePageClusterKeys(() => pages, { + ...resolveOptions, + includeLandmarkPositions: true, + }); + clusterKeys = results.map((r) => r.clusterKey); + landmarksByIndex = results.map((r) => r.landmarks); + } else { + clusterKeys = await resolvePageClusterKeys(() => pages, resolveOptions); + } } catch (error) { renderProgress(lanes, useTty, errorLine((error as Error).message)); return 1; } - const clusterCount = new Set(keys).size; + const clusterCount = new Set(clusterKeys).size; renderProgress(lanes, useTty, doneLine(pages.length, clusterCount, elapsed())); - for (const [index, key] of keys.entries()) { - options.stdout.write( - `${JSON.stringify({ id: ids[index] ?? index, clusterKey: key })}\n`, - ); + for (const [index, key] of clusterKeys.entries()) { + const row: { id: string | number; clusterKey: string; landmarks?: unknown } = { + id: ids[index] ?? index, + clusterKey: key, + }; + if (landmarksByIndex) row.landmarks = landmarksByIndex[index]; + options.stdout.write(`${JSON.stringify(row)}\n`); } return 0; } finally { diff --git a/packages/@d-zero/page-cluster/src/extract-landmarks.spec.ts b/packages/@d-zero/page-cluster/src/extract-landmarks.spec.ts index 61938b6b..d5814a8f 100644 --- a/packages/@d-zero/page-cluster/src/extract-landmarks.spec.ts +++ b/packages/@d-zero/page-cluster/src/extract-landmarks.spec.ts @@ -7,15 +7,35 @@ import { describe, expect, test } from 'vitest'; import { extractLandmarks } from './extract-landmarks.js'; import { tokenize } from './tokenize.js'; -const EMPTY_LANDMARKS = { +const EMPTY_LANDMARK_HTML = { header: [] as string[], footer: [] as string[], nav: [] as string[], aside: [] as string[], form: [] as string[], search: [] as string[], + main: [] as string[], }; +/** + * Projects an {@link ./extract-landmarks.js | ExtractLandmarksResult} down + * to just each instance's `html`, for tests that only care about content, + * not position. + * @param result + */ +function toHtmlOnly(result: ReturnType) { + return { + header: result.header.map((i) => i.html), + footer: result.footer.map((i) => i.html), + nav: result.nav.map((i) => i.html), + aside: result.aside.map((i) => i.html), + form: result.form.map((i) => i.html), + search: result.search.map((i) => i.html), + main: result.main.map((i) => i.html), + remainderHtml: result.remainderHtml, + }; +} + describe('extractLandmarks (basic extraction)', () => { test('matches the exact output documented in the JSDoc @example', () => { // Kept in sync with extractLandmarks's @example: if this ever fails, @@ -25,22 +45,56 @@ describe('extractLandmarks (basic extraction)', () => { '
H
M
F
', ); expect(result).toStrictEqual({ - ...EMPTY_LANDMARKS, - header: ['
H
'], - footer: ['
F
'], + header: [ + { + html: '
H
', + startOffset: 6, + endOffset: 24, + startLine: 1, + startColumn: 7, + endLine: 1, + endColumn: 25, + }, + ], + footer: [ + { + html: '
F
', + startOffset: 38, + endOffset: 56, + startLine: 1, + startColumn: 39, + endLine: 1, + endColumn: 57, + }, + ], + nav: [], + aside: [], + form: [], + search: [], + main: [ + { + html: '
M
', + startOffset: 24, + endOffset: 38, + startLine: 1, + startColumn: 25, + endLine: 1, + endColumn: 39, + }, + ], remainderHtml: '
M
', }); }); test('a nav is extracted and removed the same way as header/footer', () => { const result = extractLandmarks('
M
'); - expect(result.nav).toStrictEqual(['']); + expect(result.nav.map((i) => i.html)).toStrictEqual(['']); expect(result.remainderHtml).toBe('
M
'); }); test('an aside is extracted and removed the same way as header/footer', () => { const result = extractLandmarks('
M
'); - expect(result.aside).toStrictEqual(['']); + expect(result.aside.map((i) => i.html)).toStrictEqual(['']); expect(result.remainderHtml).toBe('
M
'); }); @@ -48,7 +102,9 @@ describe('extractLandmarks (basic extraction)', () => { const result = extractLandmarks( '
M
', ); - expect(result.search).toStrictEqual(['']); + expect(result.search.map((i) => i.html)).toStrictEqual([ + '', + ]); expect(result.remainderHtml).toBe('
M
'); }); @@ -56,7 +112,9 @@ describe('extractLandmarks (basic extraction)', () => { const withRole = extractLandmarks( '
M
', ); - expect(withRole.form).toStrictEqual(['
']); + expect(withRole.form.map((i) => i.html)).toStrictEqual([ + '
', + ]); expect(withRole.remainderHtml).toBe('
M
'); const bareForm = extractLandmarks('
M
'); @@ -70,22 +128,24 @@ describe('extractLandmarks (basic extraction)', () => { const result = extractLandmarks( '
M
', ); - expect(result.search).toStrictEqual(['
']); + expect(result.search.map((i) => i.html)).toStrictEqual([ + '
', + ]); expect(result.remainderHtml).toBe('
M
'); }); test('a page with no landmarks returns all fields as empty arrays and an unchanged remainderHtml', () => { - const html = '
only content
'; - expect(extractLandmarks(html)).toStrictEqual({ - ...EMPTY_LANDMARKS, + const html = 'only content'; + expect(toHtmlOnly(extractLandmarks(html))).toStrictEqual({ + ...EMPTY_LANDMARK_HTML, remainderHtml: html, }); }); test('no at all leaves remainderHtml unchanged and every field as an empty array', () => { const html = 'x'; - expect(extractLandmarks(html)).toStrictEqual({ - ...EMPTY_LANDMARKS, + expect(toHtmlOnly(extractLandmarks(html))).toStrictEqual({ + ...EMPTY_LANDMARK_HTML, remainderHtml: html, }); }); @@ -96,7 +156,9 @@ describe('extractLandmarks (role-based matching)', () => { const result = extractLandmarks( '
H
M
', ); - expect(result.header).toStrictEqual(['
H
']); + expect(result.header.map((i) => i.html)).toStrictEqual([ + '
H
', + ]); expect(result.footer).toStrictEqual([]); }); @@ -108,8 +170,12 @@ describe('extractLandmarks (role-based matching)', () => { // strictly contained by each other). const html = '
HN
M
'; const result = extractLandmarks(html); - expect(result.header).toStrictEqual(['
HN
']); - expect(result.nav).toStrictEqual(['
HN
']); + expect(result.header.map((i) => i.html)).toStrictEqual([ + '
HN
', + ]); + expect(result.nav.map((i) => i.html)).toStrictEqual([ + '
HN
', + ]); // Excising the single matched span once, not twice. expect(result.remainderHtml).toBe('
M
'); }); @@ -121,9 +187,18 @@ describe('extractLandmarks (role-based matching)', () => { // silently regress to "body's own role is never checked" again. const html = '
M
'; const result = extractLandmarks(html); - expect(result.header).toStrictEqual([html]); + expect(result.header.map((i) => i.html)).toStrictEqual([html]); expect(result.remainderHtml).toBe(''); }); + + test('role=main is recognized as a main landmark without a
tag', () => { + const result = extractLandmarks( + '
H
content
', + ); + expect(result.main.map((i) => i.html)).toStrictEqual([ + '
content
', + ]); + }); }); describe('extractLandmarks (multiple instances of the same type)', () => { @@ -136,7 +211,7 @@ describe('extractLandmarks (multiple instances of the same type)', () => { const html = '
Article H
Site H
'; const result = extractLandmarks(html); - expect(result.header).toStrictEqual([ + expect(result.header.map((i) => i.html)).toStrictEqual([ '
Article H
', '
Site H
', ]); @@ -154,8 +229,8 @@ describe('extractLandmarks (multiple instances of the same type)', () => { const html = `
outer
${siblings}`; const result = extractLandmarks(html); expect(result.header).toHaveLength(11); - expect(result.header[0]).toBe('
outer
'); - expect(result.header[10]).toBe('
sibling-9
'); + expect(result.header[0]!.html).toBe('
outer
'); + expect(result.header[10]!.html).toBe('
sibling-9
'); }); }); @@ -168,7 +243,9 @@ describe('extractLandmarks (nested landmark spans)', () => { // nav's markup as part of its own span. const html = '
Hafter-nav
M
'; const result = extractLandmarks(html); - expect(result.header).toStrictEqual(['
Hafter-nav
']); + expect(result.header.map((i) => i.html)).toStrictEqual([ + '
Hafter-nav
', + ]); expect(result.nav).toStrictEqual([]); expect(result.remainderHtml).toBe('
M
'); }); @@ -177,13 +254,117 @@ describe('extractLandmarks (nested landmark spans)', () => { const html = '
H
M
F
'; const result = extractLandmarks(html); - expect(result.header).toStrictEqual(['
H
']); - expect(result.footer).toStrictEqual(['
F
']); + expect(result.header.map((i) => i.html)).toStrictEqual(['
H
']); + expect(result.footer.map((i) => i.html)).toStrictEqual([ + '
F
', + ]); expect(result.nav).toStrictEqual([]); expect(result.remainderHtml).toBe('
M
'); }); }); +describe('extractLandmarks (main handling)', () => { + test('multiple non-nested
s are all collected', () => { + // HTML discourages more than one
, but real/malformed markup can + // still carry it — every non-nested instance is reported the same way + // the other six types are. + const html = '
one
two
'; + const result = extractLandmarks(html); + expect(result.main.map((i) => i.html)).toStrictEqual([ + '
one
', + '
two
', + ]); + }); + + test('a
nested inside another
is deduplicated to the outer one only', () => { + const html = '
outer
inner
'; + const result = extractLandmarks(html); + expect(result.main.map((i) => i.html)).toStrictEqual([ + '
outer
inner
', + ]); + }); + + test('main is never excised from remainderHtml, unlike the other six types', () => { + const html = '
H
content
F
'; + const result = extractLandmarks(html); + expect(result.remainderHtml).toBe('
content
'); + expect(result.remainderHtml).toContain('content'); + }); + + test('a section-local nav inside
is still extracted, not hidden by main containment', () => { + // This is the regression main-handling must not reintroduce: sharing + // keepOutermost's containment sweep between main and the other six + // types would make every nav/header/aside inside
look + // "contained by main" and silently vanish from those result arrays. + const html = + '' + + '
body
'; + const result = extractLandmarks(html); + expect(result.nav.map((i) => i.html)).toStrictEqual([ + '', + '', + ]); + expect(result.main.map((i) => i.html)).toStrictEqual([ + '
body
', + ]); + }); + + test('main is absent from ALL_LANDMARK_TYPES-driven chrome discovery by construction (position-only)', () => { + // extractLandmarks itself has no chrome/shell notion — this test only + // pins that main instances are reported at all, alongside the other + // six, with no isChrome-like field (that lives in + // build-page-landmark-report.ts, a separate layer downstream). + const html = '
content
'; + const result = extractLandmarks(html); + expect(result.main).toHaveLength(1); + expect(Object.keys(result.main[0]!)).toStrictEqual([ + 'html', + 'startOffset', + 'endOffset', + 'startLine', + 'startColumn', + 'endLine', + 'endColumn', + ]); + }); + + test('a main nested inside an excisable landmark is still reported, even though the excisable landmark takes its markup out of remainderHtml', () => { + // main's own keepOutermost sweep is independent of the excisable six, + // but excision itself is unaware of main: excising the outer header + // removes main's markup from remainderHtml along with it. main is + // still reported in `result.main` — only remainderHtml is affected. + const html = '
H
M
'; + const result = extractLandmarks(html); + expect(result.main.map((i) => i.html)).toStrictEqual(['
M
']); + expect(result.remainderHtml).toBe(''); + }); +}); + +describe('extractLandmarks (position: line/column)', () => { + test('a landmark on the first line reports startColumn/endColumn relative to offset 0', () => { + const html = '
H
'; + const result = extractLandmarks(html); + const [instance] = result.header; + expect(instance).toMatchObject({ startLine: 1, startColumn: 7, endLine: 1 }); + }); + + test('a landmark on a later line reports the correct line number and column reset to 1', () => { + const html = '\n
x
\n
H
\n'; + const result = extractLandmarks(html); + const [instance] = result.header; + // Line 1: ""; line 2: "
x
"; line 3: "
H
" + expect(instance).toMatchObject({ startLine: 3, startColumn: 1, endLine: 3 }); + }); + + test('a multi-line landmark reports different start/end lines', () => { + const html = '
\nH\n
'; + const result = extractLandmarks(html); + const [instance] = result.header; + expect(instance!.startLine).toBe(1); + expect(instance!.endLine).toBe(3); + }); +}); + describe('extractLandmarks (fold side effect on remainderHtml, documented and pinned)', () => { test('removing a landmark can make its class-less sibling wrapper fold-eligible once remainderHtml is tokenized', () => { // Before removal: the wrapper div has 2 element children (header, main) @@ -222,7 +403,7 @@ describe('extractLandmarks (remainderHtml tokenizes to content-only tokens)', () '' + '
body
'; const result = extractLandmarks(html); - expect(result.nav).toStrictEqual([ + expect(result.nav.map((i) => i.html)).toStrictEqual([ '', '', ]); @@ -243,7 +424,7 @@ describe('extractLandmarks (opaque tags suppress landmark detection inside them) test('a self-nested opaque tag ( inside ) does not leave stale state that blocks a later real header', () => { const html = '
H
M
'; const result = extractLandmarks(html); - expect(result.header).toStrictEqual(['
H
']); + expect(result.header.map((i) => i.html)).toStrictEqual(['
H
']); expect(result.remainderHtml).toBe( '
M
', ); @@ -261,7 +442,9 @@ describe('extractLandmarks (malformed HTML)', () => { ); const html = fs.readFileSync(fixturePath, 'utf8'); const result = extractLandmarks(html); - expect(result.header.join('')).toContain('MARKER_NESTED_BODY_TAGS_MALFORMED'); + expect(result.header.map((i) => i.html).join('')).toContain( + 'MARKER_NESTED_BODY_TAGS_MALFORMED', + ); expect(result.remainderHtml).not.toContain('MARKER_NESTED_BODY_TAGS_MALFORMED'); expect(result.remainderHtml).toContain( 'Main content that appears after multiple malformed body tags.', @@ -300,7 +483,9 @@ describe('extractLandmarks (malformed HTML)', () => { const html = 'H
M
'; expect(() => extractLandmarks(html)).not.toThrow(); const result = extractLandmarks(html); - expect(result.header).toStrictEqual(['H']); + expect(result.header.map((i) => i.html)).toStrictEqual([ + 'H', + ]); expect(result.remainderHtml).toBe('
M
'); }); @@ -312,7 +497,7 @@ describe('extractLandmarks (malformed HTML)', () => { const html = '
Outer malformed
Inner OK
'; const result = extractLandmarks(html); - expect(result.header).toStrictEqual(['
Inner OK
']); + expect(result.header.map((i) => i.html)).toStrictEqual(['
Inner OK
']); expect(result.remainderHtml).toBe( '
Outer malformed
', ); diff --git a/packages/@d-zero/page-cluster/src/extract-landmarks.ts b/packages/@d-zero/page-cluster/src/extract-landmarks.ts index fcd75c31..5c4cf6e1 100644 --- a/packages/@d-zero/page-cluster/src/extract-landmarks.ts +++ b/packages/@d-zero/page-cluster/src/extract-landmarks.ts @@ -3,6 +3,7 @@ import { findMatchingElements, type MatchingElement, } from './find-shallowest-elements.js'; +import { buildLineColumnIndex, offsetToLineColumn } from './offset-to-line-column.js'; /** * The six structural regions this module knows how to carve out of a page. @@ -17,17 +18,58 @@ import { * have no implicit landmark role under HTML-AAM unless given an accessible * name). `search` is matched via both the `` element (WHATWG * landmark shorthand) and `role="search"`. + * + * `main` is deliberately not a member of this union even though + * {@link ./extract-landmarks.js | extractLandmarks} reports it: this type is + * also the parameter type of `resolveLandmarkVariantKeys` and the vocabulary + * that chrome discovery (`computePerPageLandmarkInstances`'s + * `ALL_LANDMARK_TYPES`) iterates over. Admitting `'main'` here would let + * `resolveLandmarkVariantKeys(pages, 'main')` type-check while silently + * returning nothing (chrome discovery never looks at `main` instances) — + * exactly the kind of type-level lie this module avoids elsewhere. `main` is + * content, not chrome: it never participates in frequency-based chrome + * discovery, only in position reporting. */ export type LandmarkType = 'header' | 'footer' | 'nav' | 'aside' | 'form' | 'search'; +/** + * `extractLandmarks`'s internal scanning vocabulary: every public + * `LandmarkType` plus `'main'`. Kept private to this module so `'main'` + * never leaks into the public `LandmarkType` union (see its JSDoc). + */ +type InternalLandmarkType = LandmarkType | 'main'; + +/** + * An instance's location within the HTML string it was extracted from, in + * both string-index and 1-based line/column form. Computed once per page by + * {@link ./extract-landmarks.js | extractLandmarks} via + * {@link ./offset-to-line-column.js | buildLineColumnIndex}/`offsetToLineColumn` + * and carried downstream as plain numbers — nothing recomputes it. + */ +export type LandmarkPosition = { + readonly startOffset: number; + readonly endOffset: number; + readonly startLine: number; + readonly startColumn: number; + readonly endLine: number; + readonly endColumn: number; +}; + +/** + * One landmark instance: its raw HTML plus its {@link LandmarkPosition} + * within the page it was extracted from. + */ +export type LandmarkInstance = LandmarkPosition & { readonly html: string }; + /** * Result of {@link ./extract-landmarks.js | extractLandmarks}. Each landmark - * field holds an array of the raw HTML of every genuinely-closed instance - * of that region on the page, in document order. Empty array if the page - * has none — or if every candidate found was malformed markup - * `extractLandmarks` declined to trust (see its JSDoc's note on discarded - * candidates). `remainderHtml` is the original HTML with every extracted - * span excised, meant to be fed straight into + * field holds an array of every genuinely-closed instance of that region on + * the page, in document order. Empty array if the page has none — or if + * every candidate found was malformed markup `extractLandmarks` declined to + * trust (see its JSDoc's note on discarded candidates). `remainderHtml` is + * the original HTML with every extracted `header`/`footer`/`nav`/`aside`/ + * `form`/`search` span excised (`main` is never excised — see + * `extractLandmarks`'s "main handling" note), meant to be fed straight into * {@link ./tokenize.js | tokenize} as the page's content-only signal. * * Multiple instances per type are the norm, not the exception: real crawl @@ -40,30 +82,33 @@ export type LandmarkType = 'header' | 'footer' | 'nav' | 'aside' | 'form' | 'sea * depth or ordering rule. */ export type ExtractLandmarksResult = { - header: string[]; - footer: string[]; - nav: string[]; - aside: string[]; - form: string[]; - search: string[]; + header: LandmarkInstance[]; + footer: LandmarkInstance[]; + nav: LandmarkInstance[]; + aside: LandmarkInstance[]; + form: LandmarkInstance[]; + search: LandmarkInstance[]; + main: LandmarkInstance[]; remainderHtml: string; }; -const TAG_TO_TYPE: Readonly> = { +const TAG_TO_TYPE: Readonly> = { header: 'header', footer: 'footer', nav: 'nav', aside: 'aside', search: 'search', + main: 'main', }; -const ROLE_TO_TYPE: Readonly> = { +const ROLE_TO_TYPE: Readonly> = { banner: 'header', contentinfo: 'footer', navigation: 'nav', complementary: 'aside', form: 'form', search: 'search', + main: 'main', }; /** @@ -76,8 +121,11 @@ const ROLE_TO_TYPE: Readonly> = { * @param tagName * @param role */ -function matchLandmarkTypes(tagName: string, role: string | undefined): LandmarkType[] { - const types: LandmarkType[] = []; +function matchLandmarkTypes( + tagName: string, + role: string | undefined, +): InternalLandmarkType[] { + const types: InternalLandmarkType[] = []; const byTag = TAG_TO_TYPE[tagName]; if (byTag) { types.push(byTag); @@ -92,6 +140,19 @@ function matchLandmarkTypes(tagName: string, role: string | undefined): Landmark return types; } +/** + * Type guard splitting `findMatchingElements`' combined match list into the + * excisable six landmark types vs `main`. `main` is content, not chrome, and + * must never be mixed into the same `keepOutermost` sweep as the other six + * — see `extractLandmarks`'s "main handling" note for why. + * @param match + */ +function isMainMatch( + match: MatchingElement, +): match is MatchingElement<'main'> { + return match.type === 'main'; +} + /** * Filters out any match whose whole-element span is strictly contained by * another match's span, keeping only outermost instances. Runs across all @@ -207,21 +268,53 @@ function keepOutermost( * segment then disappears from the surviving paths, shortening them by one * level. This is inherent to "delete the matched span, use whatever's * left" and is not treated as a bug. + * + * ## Main handling + * + * `main` (the `
` tag or `role="main"`) is collected the same way as + * the other six types — one entry per genuinely-closed instance, in + * document order — but is kept out of every mechanism the other six feed: + * + * - It is **never excised**: its span is never added to `remainderHtml`'s + * excise list, because `main` is the page's actual content, not chrome. + * Removing it would gut `remainderHtml` down to whatever sits outside + * `
` (nothing, on most real pages). + * - Its `keepOutermost` nesting sweep runs **separately** from the other six + * types'. If it shared the sweep, a `
` that wraps most of the page + * (as it typically does) would make every `header`/`nav`/`aside` nested + * inside it look "contained by main" and get dropped — destroying the + * section-local chrome detection this module exists to enable (see "Why + * collect every instance" above). Only nested `
`s (an edge case — + * HTML discourages more than one) are deduplicated against each other. + * - It never contributes to chrome/shell-frequency analysis (`main` is + * absent from `computePerPageLandmarkInstances`'s `ALL_LANDMARK_TYPES`): + * its instances are reported for position purposes only, never treated as + * candidate chrome. * @param html * @example * ```ts * extractLandmarks('
H
M
F
'); * // { - * // header: ['
H
'], - * // footer: ['
F
'], + * // header: [{ html: '
H
', startOffset: 6, endOffset: 24, + * // startLine: 1, startColumn: 7, endLine: 1, endColumn: 25 }], + * // footer: [{ html: '
F
', startOffset: 38, endOffset: 56, + * // startLine: 1, startColumn: 39, endLine: 1, endColumn: 57 }], * // nav: [], aside: [], form: [], search: [], + * // main: [{ html: '
M
', startOffset: 24, endOffset: 38, + * // startLine: 1, startColumn: 25, endLine: 1, endColumn: 39 }], * // remainderHtml: '
M
', * // } * ``` */ export function extractLandmarks(html: string): ExtractLandmarksResult { const allMatches = findMatchingElements(html, matchLandmarkTypes); - const outermost = keepOutermost(allMatches); + const mainMatches = allMatches.filter(isMainMatch); + const excisableMatches = allMatches.filter( + (match): match is MatchingElement => !isMainMatch(match), + ); + + const outermost = keepOutermost(excisableMatches); + const outermostMain = keepOutermost(mainMatches); const result: ExtractLandmarksResult = { header: [], @@ -230,14 +323,36 @@ export function extractLandmarks(html: string): ExtractLandmarksResult { aside: [], form: [], search: [], + main: [], remainderHtml: html, }; const spans: { start: number; end: number }[] = []; + const lineColumnIndex = buildLineColumnIndex(html); + + const toInstance = (match: { + readonly startOffset: number; + readonly endOffset: number; + }): LandmarkInstance => { + const start = offsetToLineColumn(lineColumnIndex, match.startOffset); + const end = offsetToLineColumn(lineColumnIndex, match.endOffset); + return { + html: html.slice(match.startOffset, match.endOffset), + startOffset: match.startOffset, + endOffset: match.endOffset, + startLine: start.line, + startColumn: start.column, + endLine: end.line, + endColumn: end.column, + }; + }; for (const match of outermost) { - result[match.type].push(html.slice(match.startOffset, match.endOffset)); + result[match.type].push(toInstance(match)); spans.push({ start: match.startOffset, end: match.endOffset }); } + for (const match of outermostMain) { + result.main.push(toInstance(match)); + } result.remainderHtml = excise(html, spans); diff --git a/packages/@d-zero/page-cluster/src/is-chrome-landmark-instance.spec.ts b/packages/@d-zero/page-cluster/src/is-chrome-landmark-instance.spec.ts new file mode 100644 index 00000000..bb21773d --- /dev/null +++ b/packages/@d-zero/page-cluster/src/is-chrome-landmark-instance.spec.ts @@ -0,0 +1,43 @@ +import { describe, expect, test } from 'vitest'; + +import { isChromeLandmarkInstance } from './is-chrome-landmark-instance.js'; + +describe('isChromeLandmarkInstance', () => { + test('an instance with zero tokens is never chrome', () => { + expect(isChromeLandmarkInstance(new Set(), new Set(['a', 'b']))).toBe(false); + }); + + test('an instance fully covered by shell tokens is chrome', () => { + expect(isChromeLandmarkInstance(new Set(['a', 'b']), new Set(['a', 'b', 'c']))).toBe( + true, + ); + }); + + test('an instance with no overlap with shell tokens is not chrome', () => { + expect(isChromeLandmarkInstance(new Set(['x', 'y']), new Set(['a', 'b']))).toBe( + false, + ); + }); + + test('an instance right at the default 0.8 threshold is chrome', () => { + // 4 of 5 tokens are shell tokens: 4/5 = 0.8, meets the default clamp. + const instanceTokens = new Set(['a', 'b', 'c', 'd', 'x']); + const shellTokens = new Set(['a', 'b', 'c', 'd']); + expect(isChromeLandmarkInstance(instanceTokens, shellTokens)).toBe(true); + }); + + test('an instance just below the default 0.8 threshold is not chrome', () => { + // 3 of 4 tokens are shell tokens: 3/4 = 0.75, below the default clamp. + const instanceTokens = new Set(['a', 'b', 'c', 'x']); + const shellTokens = new Set(['a', 'b', 'c']); + expect(isChromeLandmarkInstance(instanceTokens, shellTokens)).toBe(false); + }); + + test('a custom threshold overrides the default', () => { + // 1 of 2 tokens overlap: 0.5 ratio. + const instanceTokens = new Set(['a', 'x']); + const shellTokens = new Set(['a']); + expect(isChromeLandmarkInstance(instanceTokens, shellTokens, 0.5)).toBe(true); + expect(isChromeLandmarkInstance(instanceTokens, shellTokens, 0.6)).toBe(false); + }); +}); diff --git a/packages/@d-zero/page-cluster/src/is-chrome-landmark-instance.ts b/packages/@d-zero/page-cluster/src/is-chrome-landmark-instance.ts new file mode 100644 index 00000000..b1bd0bea --- /dev/null +++ b/packages/@d-zero/page-cluster/src/is-chrome-landmark-instance.ts @@ -0,0 +1,53 @@ +/** + * Default containment threshold for {@link ./is-chrome-landmark-instance.js | isChromeLandmarkInstance}. + * Chosen to match the quorum/shell fractions used elsewhere in this pipeline + * ({@link ./merge-cross-block-clusters.js | mergeCrossBlockClusters}'s + * `QUORUM_FRACTION`, {@link ./shell-quorum.js | SHELL_QUORUM_FALLBACK_FRACTION}) — + * unlike those, this exact value has not been validated against real crawl + * data; it is a starting point carried over by convention, adjustable via + * this function's `threshold` parameter. + */ +export const DEFAULT_CHROME_OVERLAP_THRESHOLD = 0.8; + +/** + * Classifies a single landmark instance as chrome (shared site/section + * furniture) or content, given the instance's own tokens and the shell + * token set {@link ./shell-quorum.js | shellQuorum} discovered for its unit. + * + * ## Why containment (`|instance ∩ shell| / |instance|`) and not Jaccard + * + * `shellTokens` is the union of chrome tokens across an entire unit's + * landmark instances (header + nav + footer + …, corpus-wide), so it is + * usually far larger than any single instance's own token set. Jaccard's + * denominator is the *union* of both sets, which stays shell-sized even when + * the instance is 100% shell tokens — driving the score down regardless of + * how purely "shell" the instance is. Containment instead asks "of this + * instance's own tokens, how many are shell tokens", which is the question + * that actually matters for classifying one instance. + * + * An instance with zero tokens is never chrome (there is nothing to + * corroborate) — matches {@link ./per-page-landmark-signatures.js | computePerPageLandmarkInstances}'s + * own choice to drop zero-token instances before they ever reach a + * `PerPageLandmarkInstance`, kept here as a defensive default rather than an + * assumption about every caller. + * @param instanceTokens + * @param shellTokens + * @param threshold + * @example + * ```ts + * const shellTokens = shellQuorum(unitPerPageInstances); + * const isChrome = isChromeLandmarkInstance(instance.tokens, shellTokens); + * ``` + */ +export function isChromeLandmarkInstance( + instanceTokens: ReadonlySet, + shellTokens: ReadonlySet, + threshold = DEFAULT_CHROME_OVERLAP_THRESHOLD, +): boolean { + if (instanceTokens.size === 0) return false; + let hit = 0; + for (const token of instanceTokens) { + if (shellTokens.has(token)) hit++; + } + return hit / instanceTokens.size >= threshold; +} diff --git a/packages/@d-zero/page-cluster/src/merge-cross-block-clusters.spec.ts b/packages/@d-zero/page-cluster/src/merge-cross-block-clusters.spec.ts index 858a706a..d834826e 100644 --- a/packages/@d-zero/page-cluster/src/merge-cross-block-clusters.spec.ts +++ b/packages/@d-zero/page-cluster/src/merge-cross-block-clusters.spec.ts @@ -1,4 +1,8 @@ -import type { ExtractLandmarksResult } from './extract-landmarks.js'; +import type { + ExtractLandmarksResult, + LandmarkInstance, + LandmarkType, +} from './extract-landmarks.js'; import type { CrossBlockUnit } from './merge-cross-block-clusters.js'; import { describe, expect, test } from 'vitest'; @@ -19,6 +23,24 @@ function toInstances(landmarks: readonly ExtractLandmarksResult[]) { return computePerPageLandmarkInstances(landmarks); } +/** + * Builds fixture {@link LandmarkInstance}s from raw HTML strings. Position + * values are arbitrary but distinct per instance — this file's tests never + * assert on position. + * @param htmlList + */ +function toLandmarkInstances(htmlList: readonly string[]): LandmarkInstance[] { + return htmlList.map((html, i) => ({ + html, + startOffset: i, + endOffset: i + html.length, + startLine: 1, + startColumn: i + 1, + endLine: 1, + endColumn: i + html.length + 1, + })); +} + const noLandmarks: ExtractLandmarksResult = { header: [], footer: [], @@ -26,12 +48,22 @@ const noLandmarks: ExtractLandmarksResult = { aside: [], form: [], search: [], + main: [], remainderHtml: '', }; const landmarksWith = ( - overrides: Partial, -): ExtractLandmarksResult => ({ ...noLandmarks, ...overrides }); + overrides: Partial>, +): ExtractLandmarksResult => { + const result = { ...noLandmarks }; + for (const [type, htmlList] of Object.entries(overrides) as [ + LandmarkType, + readonly string[], + ][]) { + result[type] = toLandmarkInstances(htmlList); + } + return result; +}; describe('mergeCrossBlockClusters', () => { test('empty input returns empty map', () => { diff --git a/packages/@d-zero/page-cluster/src/merge-cross-block-clusters.ts b/packages/@d-zero/page-cluster/src/merge-cross-block-clusters.ts index 2a647a1a..c01437ca 100644 --- a/packages/@d-zero/page-cluster/src/merge-cross-block-clusters.ts +++ b/packages/@d-zero/page-cluster/src/merge-cross-block-clusters.ts @@ -1,7 +1,6 @@ import type { PerPageLandmarkInstance } from './per-page-landmark-signatures.js'; import { assignContainedClusters } from './assign-contained-clusters.js'; -import { autoCutThreshold } from './auto-cut-threshold.js'; import { collapseAnonymousDivs } from './collapse-anonymous-divs.js'; import { completeLinkageDendrogram, @@ -11,6 +10,7 @@ import { computeDocumentFrequency } from './compute-document-frequency.js'; import { jaccardSimilarity } from './jaccard-similarity.js'; import { reservoirSample } from './reservoir-sample.js'; import { shapeToken } from './shape-token.js'; +import { shellQuorum } from './shell-quorum.js'; import { splitTokensByFrequency } from './split-tokens-by-frequency.js'; /** @@ -53,11 +53,10 @@ const CROSS_BLOCK_THRESHOLD = 0.8; * on real crawl data. Full union is shell-dominated: 298 pages collapsed into * 4 clusters — also confirmed. 80% quorum avoids both failure modes. * - * Also reused as the fallback clamp for {@link ./auto-cut-threshold.js | - * autoCutThreshold} when running on the per-landmark-instance frequency - * distribution in {@link ./merge-cross-block-clusters.js | shellQuorum}. The - * clamp only ever *loosens* the cut relative to this floor (never tightens), - * and only fires in the degenerate cases the JSDoc there describes. + * Not shared with {@link ./shell-quorum.js | shellQuorum}'s own fallback + * clamp (`SHELL_QUORUM_FALLBACK_FRACTION`) — the two happen to be the same + * value today because both were validated against the same real crawl + * corpora, but they are independently tunable. */ const QUORUM_FRACTION = 0.8; @@ -201,101 +200,6 @@ function l2Contained(xSig: Map, ySig: Map): bool return true; } -/** - * Discovers a unit's shell tokens by auto-cutting the per-*token* page- - * frequency histogram of every landmark instance's tokens. This is the same - * max-gap primitive used for Stage A merge-height cutoffs, applied - * recursively at the landmark-token layer. - * - * ## Why per-token and not per-signature - * - * An earlier iteration ran the histogram at the level of full landmark- - * instance signatures (canonicalized token sets). That failed on a real, - * common pattern: a shared site chrome whose markup carries a per-page - * distinguishing element (a breadcrumb, a page-title element with a page- - * specific class, a "current" state). All pages have most of the same - * tokens, but every page's full signature is distinct because tokens embed - * class names. Per-signature counting saw 5 signatures at freq 0.2 each, - * autoCutThreshold on the flat distribution returned the clamp, and the - * shell collapsed to empty even though every page shared the core header - * skeleton. Per-token counting handles the same case correctly — the shared - * skeleton tokens each hit freq 1.0. - * - * ## The histogram - * - * For every member page, all its landmark instances are tokenized and - * unioned into a single per-page token set (order-agnostic, deduped: a - * token appearing in two of the page's landmarks still counts once for - * that page). The corpus histogram is then "how many pages contain each - * token". Tokens that appear on nearly every page are the unit's chrome; - * tokens that appear on only a handful are page-specific content that - * happens to be tagged as a landmark. - * - * ## Why auto-cut instead of a hard-coded quorum - * - * A fixed 80% quorum (this file's earlier implementation) baked one - * threshold in for every unit. Real corpora don't obey a universal cutoff: - * a section-local landmark token that appears on 60% of a unit's pages is - * the section's chrome under any reasonable reading, but 80% quorum - * discards it. Auto-cut looks at the *shape* of the frequency distribution - * and picks the widest gap between adjacent frequencies — if the - * distribution is `{1.00, 1.00, 0.65, 0.03, 0.02}`, the gap between 0.65 - * and 0.03 (0.62) dwarfs everything else and the cut lands mid-gap around - * 0.34, correctly grouping the 0.65 tokens with the site-wide 1.00 ones as - * "chrome for this unit". If instead the distribution is flat, the clamp - * to {@link QUORUM_FRACTION} keeps the threshold from being tighter than - * the fallback default. - * - * ## Fallbacks - * - * A single distinct token (`heights.length < 2`) or a perfectly flat - * distribution (`maxGap === 0`) returns the {@link QUORUM_FRACTION} clamp - * verbatim — exactly the same 80%-quorum behavior as before. So degenerate - * cases degrade to the old contract; only richer distributions get the - * auto-cut benefit. - * - * A page with no landmarks contributes an empty set, deliberately, so the - * shell-corroboration jaccard between two landmark-less pages is 0 rather - * than 1 (which it would be if we handed back a ``-derived - * `{body}` fallback set to both sides). - * @param perPageInstances - */ -function shellQuorum( - perPageInstances: readonly (readonly PerPageLandmarkInstance[])[], -): ReadonlySet { - const pageCount = perPageInstances.length; - if (pageCount === 0) return new Set(); - - // Union all instance token sets per page (dedupe within page: a token - // present on both header and footer of the same page still counts once - // for that page's contribution). - const tokenPageCount = new Map(); - for (const instances of perPageInstances) { - const perPageUnion = new Set(); - for (const inst of instances) { - for (const token of inst.tokens) perPageUnion.add(token); - } - for (const token of perPageUnion) { - tokenPageCount.set(token, (tokenPageCount.get(token) ?? 0) + 1); - } - } - - if (tokenPageCount.size === 0) return new Set(); - - const frequencies: number[] = []; - for (const count of tokenPageCount.values()) { - frequencies.push(count / pageCount); - } - - const cut = autoCutThreshold(frequencies, QUORUM_FRACTION); - - const shell = new Set(); - for (const [token, count] of tokenPageCount) { - if (count / pageCount >= cut) shell.add(token); - } - return shell; -} - // --------------------------------------------------------------------------- // Main function // --------------------------------------------------------------------------- diff --git a/packages/@d-zero/page-cluster/src/offset-to-line-column.spec.ts b/packages/@d-zero/page-cluster/src/offset-to-line-column.spec.ts new file mode 100644 index 00000000..b0c06e2b --- /dev/null +++ b/packages/@d-zero/page-cluster/src/offset-to-line-column.spec.ts @@ -0,0 +1,60 @@ +import { describe, expect, test } from 'vitest'; + +import { buildLineColumnIndex, offsetToLineColumn } from './offset-to-line-column.js'; + +describe('offsetToLineColumn', () => { + test('single-line string', () => { + const index = buildLineColumnIndex('abc'); + expect(offsetToLineColumn(index, 0)).toStrictEqual({ line: 1, column: 1 }); + expect(offsetToLineColumn(index, 2)).toStrictEqual({ line: 1, column: 3 }); + }); + + test('multi-line string resolves offsets on each line', () => { + const html = 'ab\ncd'; + const index = buildLineColumnIndex(html); + expect(offsetToLineColumn(index, 0)).toStrictEqual({ line: 1, column: 1 }); // 'a' + expect(offsetToLineColumn(index, 1)).toStrictEqual({ line: 1, column: 2 }); // 'b' + expect(offsetToLineColumn(index, 3)).toStrictEqual({ line: 2, column: 1 }); // 'c' + expect(offsetToLineColumn(index, 4)).toStrictEqual({ line: 2, column: 2 }); // 'd' + }); + + test('offset immediately after a newline starts the next line at column 1', () => { + const html = 'ab\ncd'; + const index = buildLineColumnIndex(html); + // offset 3 is 'c', the character right after the '\n' at offset 2 + expect(offsetToLineColumn(index, 3)).toStrictEqual({ line: 2, column: 1 }); + }); + + test('the newline character itself belongs to the line it terminates', () => { + const html = 'ab\ncd'; + const index = buildLineColumnIndex(html); + expect(offsetToLineColumn(index, 2)).toStrictEqual({ line: 1, column: 3 }); + }); + + test('offset at the end of the string (exclusive end, e.g. endOffset)', () => { + const html = 'ab\ncd'; + const index = buildLineColumnIndex(html); + expect(offsetToLineColumn(index, html.length)).toStrictEqual({ line: 2, column: 3 }); + }); + + test('empty string', () => { + const index = buildLineColumnIndex(''); + expect(offsetToLineColumn(index, 0)).toStrictEqual({ line: 1, column: 1 }); + }); + + test('CRLF: \\r is counted as the last column of its own line', () => { + const html = 'ab\r\ncd'; + const index = buildLineColumnIndex(html); + expect(offsetToLineColumn(index, 2)).toStrictEqual({ line: 1, column: 3 }); // '\r' + expect(offsetToLineColumn(index, 3)).toStrictEqual({ line: 1, column: 4 }); // '\n' + expect(offsetToLineColumn(index, 4)).toStrictEqual({ line: 2, column: 1 }); // 'c' + }); + + test('the same index can be reused across multiple offsets', () => { + const html = 'one\ntwo\nthree'; + const index = buildLineColumnIndex(html); + expect(offsetToLineColumn(index, 0)).toStrictEqual({ line: 1, column: 1 }); + expect(offsetToLineColumn(index, 4)).toStrictEqual({ line: 2, column: 1 }); + expect(offsetToLineColumn(index, 8)).toStrictEqual({ line: 3, column: 1 }); + }); +}); diff --git a/packages/@d-zero/page-cluster/src/offset-to-line-column.ts b/packages/@d-zero/page-cluster/src/offset-to-line-column.ts new file mode 100644 index 00000000..80510c8f --- /dev/null +++ b/packages/@d-zero/page-cluster/src/offset-to-line-column.ts @@ -0,0 +1,60 @@ +/** + * A 1-based line/column position within an HTML string. + */ +export type LineColumn = { readonly line: number; readonly column: number }; + +/** + * Precomputed lookup structure for {@link ./offset-to-line-column.js | offsetToLineColumn}: + * every newline's string-index offset, ascending. + */ +export type LineColumnIndex = { readonly newlineOffsets: readonly number[] }; + +/** + * Scans `html` once and records every `\n` offset, so repeated + * {@link ./offset-to-line-column.js | offsetToLineColumn} calls against the + * same HTML string can binary-search instead of re-scanning from the start + * each time. Built once per page and reused across every landmark instance's + * start/end offset — a page with dozens of landmark instances would + * otherwise pay for a fresh O(n) scan per offset instead of one O(n) scan + * total. + * @param html + */ +export function buildLineColumnIndex(html: string): LineColumnIndex { + const newlineOffsets: number[] = []; + for (let i = 0; i < html.length; i++) { + if (html.codePointAt(i) === 10) newlineOffsets.push(i); + } + return { newlineOffsets }; +} + +/** + * Converts a string-index `offset` (the same unit as `htmlparser2`'s + * `startIndex`/`endIndex`, i.e. UTF-16 code units) into a 1-based + * `{line, column}` position, using an index built by + * {@link ./offset-to-line-column.js | buildLineColumnIndex}. + * + * `\r\n` line endings are handled without special-casing: the `\r` is + * counted as the last column of its own line, matching how most editors + * report position for CRLF files. + * @param index + * @param offset + */ +export function offsetToLineColumn(index: LineColumnIndex, offset: number): LineColumn { + const { newlineOffsets } = index; + + let low = 0; + let high = newlineOffsets.length; + while (low < high) { + const mid = (low + high) >>> 1; + if (newlineOffsets[mid]! < offset) { + low = mid + 1; + } else { + high = mid; + } + } + // `low` is the count of newlines strictly before `offset`, i.e. the + // number of completed lines — so `low` newlines completed means we're on + // line `low + 1`. + const lineStart = low === 0 ? 0 : newlineOffsets[low - 1]! + 1; + return { line: low + 1, column: offset - lineStart + 1 }; +} diff --git a/packages/@d-zero/page-cluster/src/per-page-landmark-signatures.spec.ts b/packages/@d-zero/page-cluster/src/per-page-landmark-signatures.spec.ts index e4c08995..5a74e2dd 100644 --- a/packages/@d-zero/page-cluster/src/per-page-landmark-signatures.spec.ts +++ b/packages/@d-zero/page-cluster/src/per-page-landmark-signatures.spec.ts @@ -1,4 +1,8 @@ -import type { ExtractLandmarksResult } from './extract-landmarks.js'; +import type { + ExtractLandmarksResult, + LandmarkInstance, + LandmarkType, +} from './extract-landmarks.js'; import { describe, expect, test } from 'vitest'; @@ -11,15 +15,40 @@ const emptyLandmarks: ExtractLandmarksResult = { aside: [], form: [], search: [], + main: [], remainderHtml: '', }; +/** + * Builds fixture {@link LandmarkInstance}s from raw HTML strings. Position + * values are arbitrary but distinct per instance — these tests only assert + * on `type`/`tokens`/`signature`, never on position. + * @param htmlList + */ +function toInstances(htmlList: readonly string[]): LandmarkInstance[] { + return htmlList.map((html, i) => ({ + html, + startOffset: i, + endOffset: i + html.length, + startLine: 1, + startColumn: i + 1, + endLine: 1, + endColumn: i + html.length + 1, + })); +} + const landmarksOf = ( - overrides: Partial, -): ExtractLandmarksResult => ({ - ...emptyLandmarks, - ...overrides, -}); + overrides: Partial>, +): ExtractLandmarksResult => { + const result = { ...emptyLandmarks }; + for (const [type, htmlList] of Object.entries(overrides) as [ + LandmarkType, + readonly string[], + ][]) { + result[type] = toInstances(htmlList); + } + return result; +}; describe('computePerPageLandmarkInstances', () => { test('an empty landmarks array yields an empty result', () => { @@ -32,7 +61,7 @@ describe('computePerPageLandmarkInstances', () => { expect(result[0]).toStrictEqual([]); }); - test('every landmark instance across every type is returned with its tokens and signature', () => { + test('every landmark instance across every type is returned with its tokens, signature, and position', () => { const landmarks = landmarksOf({ header: ['
'], footer: [''], @@ -43,6 +72,15 @@ describe('computePerPageLandmarkInstances', () => { for (const inst of instances!) { expect(inst.tokens.size).toBeGreaterThan(0); expect(inst.signature).toMatch(/^\[.*\]$/); + const source = landmarks[inst.type][0]!; + expect(inst.position).toStrictEqual({ + startOffset: source.startOffset, + endOffset: source.endOffset, + startLine: source.startLine, + startColumn: source.startColumn, + endLine: source.endLine, + endColumn: source.endColumn, + }); } }); diff --git a/packages/@d-zero/page-cluster/src/per-page-landmark-signatures.ts b/packages/@d-zero/page-cluster/src/per-page-landmark-signatures.ts index 8f511e02..c0be4c95 100644 --- a/packages/@d-zero/page-cluster/src/per-page-landmark-signatures.ts +++ b/packages/@d-zero/page-cluster/src/per-page-landmark-signatures.ts @@ -1,4 +1,8 @@ -import type { ExtractLandmarksResult, LandmarkType } from './extract-landmarks.js'; +import type { + ExtractLandmarksResult, + LandmarkPosition, + LandmarkType, +} from './extract-landmarks.js'; import type { TokenizeOptions } from './types.js'; import { canonicalizeTokenSet } from './canonicalize-token-set.js'; @@ -25,11 +29,20 @@ export const ALL_LANDMARK_TYPES: readonly LandmarkType[] = [ * {@link ./canonicalize-token-set.js | canonicalizeTokenSet}). Signatures * are reused across consumers so two callers see the same "same instance" * verdict without independently re-canonicalizing. + * + * `position` is the instance's location in the page it came from, computed + * once by {@link ./extract-landmarks.js | extractLandmarks} and carried here + * unchanged — a back-reference for callers (e.g. + * {@link ./build-page-landmark-report.js | buildPageLandmarkReport}) that + * need to report where a chrome-classified instance actually sits, not just + * that it exists. It plays no part in `tokens`/`signature` computation or in + * the corpus-frequency logic that consumes this type. */ export type PerPageLandmarkInstance = { readonly type: LandmarkType; readonly tokens: ReadonlySet; readonly signature: string; + readonly position: LandmarkPosition; }; /** @@ -66,16 +79,28 @@ export function computePerPageLandmarkInstances( const seenSignatures = new Set(); const out: PerPageLandmarkInstance[] = []; for (const type of ALL_LANDMARK_TYPES) { - for (const instanceHtml of entry[type]) { - if (!instanceHtml) continue; + for (const instance of entry[type]) { + if (!instance.html) continue; const tokens = new Set( - tokenize(`${instanceHtml}`, tokenizeOptions).tokens, + tokenize(`${instance.html}`, tokenizeOptions).tokens, ); if (tokens.size === 0) continue; const signature = canonicalizeTokenSet(tokens); if (seenSignatures.has(signature)) continue; seenSignatures.add(signature); - out.push({ type, tokens, signature }); + out.push({ + type, + tokens, + signature, + position: { + startOffset: instance.startOffset, + endOffset: instance.endOffset, + startLine: instance.startLine, + startColumn: instance.startColumn, + endLine: instance.endLine, + endColumn: instance.endColumn, + }, + }); } } return out; diff --git a/packages/@d-zero/page-cluster/src/resolve-page-cluster-keys-streaming.spec.ts b/packages/@d-zero/page-cluster/src/resolve-page-cluster-keys-streaming.spec.ts index a0584586..a42e1fcd 100644 --- a/packages/@d-zero/page-cluster/src/resolve-page-cluster-keys-streaming.spec.ts +++ b/packages/@d-zero/page-cluster/src/resolve-page-cluster-keys-streaming.spec.ts @@ -3,6 +3,7 @@ import type { PageClusterSignals, ProgressEvent } from './resolve-page-cluster-k import { describe, expect, test } from 'vitest'; import { + CORPUS_INLINE_THRESHOLD, resolvePageClusterKeys, resolvePageClusterKeysFromArray, resolvePageClusterKeysInMemory, @@ -164,3 +165,43 @@ describe('resolvePageClusterKeys onProgress on small corpus', () => { expect(withoutProgress).toEqual(inMemory); }); }); + +describe('resolvePageClusterKeys (includeLandmarkPositions)', () => { + test('small corpus with includeLandmarkPositions matches resolvePageClusterKeysInMemory', async () => { + const pages = buildTinyCorpus(); + const streamed = await resolvePageClusterKeys(() => pages, { + includeLandmarkPositions: true, + }); + const inMemory = resolvePageClusterKeysInMemory(pages, { + includeLandmarkPositions: true, + }); + expect(streamed).toEqual(inMemory); + }); + + test('combined with onProgress, still returns reports but emits no progress events (routed through the sync path)', async () => { + const pages = buildTinyCorpus(); + const events: ProgressEvent[] = []; + const result = await resolvePageClusterKeys(() => pages, { + includeLandmarkPositions: true, + onProgress: (event) => events.push(event), + }); + expect(result).toHaveLength(pages.length); + expect(result[0]).toHaveProperty('landmarks'); + expect(events).toStrictEqual([]); + }); + + test('a corpus above CORPUS_INLINE_THRESHOLD rejects with a RangeError instead of streaming', async () => { + const bigCount = CORPUS_INLINE_THRESHOLD + 1; + /** + * @yields {PageClusterSignals} A minimal page, `bigCount` times. + */ + function* generate(): Generator { + for (let i = 0; i < bigCount; i++) { + yield { paths: ['p', String(i)], stylesheetHrefs: [], html: '' }; + } + } + await expect( + resolvePageClusterKeys(() => generate(), { includeLandmarkPositions: true }), + ).rejects.toThrow(RangeError); + }); +}); diff --git a/packages/@d-zero/page-cluster/src/resolve-page-cluster-keys.spec.ts b/packages/@d-zero/page-cluster/src/resolve-page-cluster-keys.spec.ts index 8869a817..566c22db 100644 --- a/packages/@d-zero/page-cluster/src/resolve-page-cluster-keys.spec.ts +++ b/packages/@d-zero/page-cluster/src/resolve-page-cluster-keys.spec.ts @@ -1,6 +1,10 @@ import { describe, expect, test } from 'vitest'; -import { resolvePageClusterKeysInMemory as resolvePageClusterKeys } from './resolve-page-cluster-keys.js'; +import { + assertLandmarkPositionsSupportedForPageCount, + resolvePageClusterKeysInMemory, + resolvePageClusterKeysInMemory as resolvePageClusterKeys, +} from './resolve-page-cluster-keys.js'; describe('resolvePageClusterKeys', () => { test('an empty page list returns an empty array', () => { @@ -639,3 +643,97 @@ describe('resolvePageClusterKeys (local-landmark pseudo-token injection)', () => for (let i = 1; i < 6; i++) expect(result[i]).toBe(result[0]); }); }); + +describe('resolvePageClusterKeysInMemory (includeLandmarkPositions)', () => { + test('omitting includeLandmarkPositions returns a plain string[], unchanged from before', () => { + const html = '
H
content
'; + const result = resolvePageClusterKeysInMemory([ + { paths: ['a'], stylesheetHrefs: [], html }, + ]); + expect(result).toStrictEqual(['["path:a","cluster:0"]']); + }); + + test("a header shared across the whole cluster is reported as chrome; each page's own unique nav is not", () => { + const sharedHeader = '
'; + const html1 = `${sharedHeader}
one
`; + const html2 = `${sharedHeader}
two
`; + const result = resolvePageClusterKeysInMemory( + [ + { paths: ['p', '1'], stylesheetHrefs: [], html: html1 }, + { paths: ['p', '2'], stylesheetHrefs: [], html: html2 }, + ], + { includeLandmarkPositions: true }, + ); + expect(result).toHaveLength(2); + // Both pages land in the same final cluster (identical header + same + // article structure), so shellQuorum is computed over both together. + expect(result[0]!.clusterKey).toBe(result[1]!.clusterKey); + expect(result[0]!.landmarks.header[0]!.isChrome).toBe(true); + expect(result[1]!.landmarks.header[0]!.isChrome).toBe(true); + // Each page's nav carries a page-unique class not shared by the other + // — content, not shell. + expect(result[0]!.landmarks.nav[0]!.isChrome).toBe(false); + expect(result[1]!.landmarks.nav[0]!.isChrome).toBe(false); + }); + + test('main instances are reported with position but no isChrome verdict', () => { + const html = '
H
content
'; + const result = resolvePageClusterKeysInMemory( + [{ paths: ['a'], stylesheetHrefs: [], html }], + { includeLandmarkPositions: true }, + ); + expect(result[0]!.landmarks.main).toHaveLength(1); + expect(Object.keys(result[0]!.landmarks.main[0]!)).not.toContain('isChrome'); + }); + + test('two distinct clusters each get their own shellQuorum, not a corpus-wide one', () => { + // Cluster A's header/nav tokens never appear anywhere in cluster B's + // pages, and vice versa. If shellQuorum were mistakenly computed once + // over all 6 pages instead of once per final cluster, each header's + // corpus-wide frequency would be 3/6 = 0.5 — below the single-distinct- + // signature fallback clamp (0.8) — and neither would be chrome. Per- + // cluster computation gives each header a 3/3 = 1.0 frequency within + // its own cluster, so both are correctly chrome. + const headerA = '
'; + const headerB = '
'; + const clusterAPages = Array.from({ length: 3 }, (_, i) => ({ + paths: ['team-a', `p${i}`], + stylesheetHrefs: [], + html: `${headerA}
content ${i}
`, + })); + const clusterBPages = Array.from({ length: 3 }, (_, i) => ({ + paths: ['team-b', `p${i}`], + stylesheetHrefs: [], + html: `${headerB}
content ${i}
`, + })); + const result = resolvePageClusterKeysInMemory([...clusterAPages, ...clusterBPages], { + includeLandmarkPositions: true, + }); + + // Precondition: the two groups really are in different final clusters + // (different blocking path AND different tag skeleton: article vs + // section) — otherwise this test wouldn't be exercising the + // per-cluster grouping at all. + const clusterKeys = new Set(result.map((r) => r.clusterKey)); + expect(clusterKeys.size).toBe(2); + expect(result[0]!.clusterKey).toBe(result[1]!.clusterKey); + expect(result[0]!.clusterKey).toBe(result[2]!.clusterKey); + expect(result[3]!.clusterKey).toBe(result[4]!.clusterKey); + expect(result[3]!.clusterKey).toBe(result[5]!.clusterKey); + + for (const r of result) { + expect(r.landmarks.header[0]!.isChrome).toBe(true); + } + }); +}); + +describe('assertLandmarkPositionsSupportedForPageCount', () => { + test('does not throw when pageCount is at or below threshold', () => { + expect(() => assertLandmarkPositionsSupportedForPageCount(5, 5)).not.toThrow(); + expect(() => assertLandmarkPositionsSupportedForPageCount(4, 5)).not.toThrow(); + }); + + test('throws a RangeError when pageCount exceeds threshold', () => { + expect(() => assertLandmarkPositionsSupportedForPageCount(6, 5)).toThrow(RangeError); + }); +}); diff --git a/packages/@d-zero/page-cluster/src/resolve-page-cluster-keys.ts b/packages/@d-zero/page-cluster/src/resolve-page-cluster-keys.ts index 374e66ef..1b7c1f45 100644 --- a/packages/@d-zero/page-cluster/src/resolve-page-cluster-keys.ts +++ b/packages/@d-zero/page-cluster/src/resolve-page-cluster-keys.ts @@ -1,10 +1,15 @@ import type { ExtractLandmarksResult } from './extract-landmarks.js'; import type { CrossBlockUnit } from './merge-cross-block-clusters.js'; +import type { PerPageLandmarkInstance } from './per-page-landmark-signatures.js'; import type { ResolveBlockingGroupKeysOptions } from './resolve-blocking-group-keys.js'; import type { ResolveStructuralClusterKeysOptions } from './resolve-structural-cluster-keys.js'; import type { TokenizeOptions } from './types.js'; import { autoCutThreshold } from './auto-cut-threshold.js'; +import { + type PageLandmarkReport, + buildPageLandmarkReport, +} from './build-page-landmark-report.js'; import { capContentDepth } from './cap-content-depth.js'; import { detectContentDepthCap, @@ -17,6 +22,7 @@ import { mergeCrossBlockClusters } from './merge-cross-block-clusters.js'; import { groupIndicesByBlockKey, resolveBlockKeys } from './pass0-blocking.js'; import { computePerPageLandmarkInstances } from './per-page-landmark-signatures.js'; import { removeContentBlocks } from './remove-content-blocks.js'; +import { shellQuorum } from './shell-quorum.js'; import { stageAPerBlock } from './stage-a-per-block.js'; import { tokenize } from './tokenize.js'; @@ -193,9 +199,12 @@ export function computeLocalChromeArtifacts( ): { readonly localSignatures: ReadonlySet; readonly localTokensByPage: readonly ReadonlySet[]; + readonly perPageInstances: readonly (readonly PerPageLandmarkInstance[])[]; } { const pageCount = landmarks.length; - if (pageCount === 0) return { localSignatures: new Set(), localTokensByPage: [] }; + if (pageCount === 0) { + return { localSignatures: new Set(), localTokensByPage: [], perPageInstances: [] }; + } const perPageInstances = computePerPageLandmarkInstances(landmarks, tokenizeOptions); @@ -220,6 +229,7 @@ export function computeLocalChromeArtifacts( return { localSignatures: new Set(), localTokensByPage: landmarks.map(() => new Set()), + perPageInstances, }; } @@ -240,6 +250,7 @@ export function computeLocalChromeArtifacts( return { localSignatures: new Set(), localTokensByPage: landmarks.map(() => new Set()), + perPageInstances, }; } @@ -252,7 +263,7 @@ export function computeLocalChromeArtifacts( return out; }); - return { localSignatures, localTokensByPage }; + return { localSignatures, localTokensByPage, perPageInstances }; } /** @@ -275,6 +286,44 @@ export function computeLocalLandmarkTokens( return [...computeLocalChromeArtifacts(landmarks, tokenizeOptions).localTokensByPage]; } +/** + * Builds every page's {@link PageLandmarkReport} for the `includeLandmarkPositions` + * result path. Groups pages by their *final* cluster key (post–Stage-B), runs + * {@link ./shell-quorum.js | shellQuorum} once per cluster over the pooled + * member `PerPageLandmarkInstance`s (the same "shell tokens" concept Stage B + * itself uses for L2 shell corroboration, just recomputed at the final- + * cluster granularity rather than the pre-merge unit granularity), then + * classifies each member page's own landmark instances against that + * cluster-level shell. + * @param finalKeys + * @param landmarks + * @param perPageInstances + */ +function buildLandmarkReportsByCluster( + finalKeys: readonly string[], + landmarks: readonly ExtractLandmarksResult[], + perPageInstances: readonly (readonly PerPageLandmarkInstance[])[], +): PageLandmarkReport[] { + const indicesByKey = new Map(); + for (const [i, key] of finalKeys.entries()) { + const indices = indicesByKey.get(key); + if (indices) { + indices.push(i); + } else { + indicesByKey.set(key, [i]); + } + } + + const reports: PageLandmarkReport[] = Array.from({ length: finalKeys.length }); + for (const indices of indicesByKey.values()) { + const shellTokens = shellQuorum(indices.map((i) => perPageInstances[i]!)); + for (const i of indices) { + reports[i] = buildPageLandmarkReport(landmarks[i]!, shellTokens); + } + } + return reports; +} + /** * Per-page input to {@link ./resolve-page-cluster-keys.js | resolvePageClusterKeys}: * the blocking signals {@link ./resolve-blocking-group-keys.js | resolveBlockingGroupKeys} @@ -359,11 +408,50 @@ export type ResolvePageClusterKeysOptions = TokenizeOptions & * sync helper to running a per-block async loop that emits * `pass1-block-complete` and `stage-b-start`. Omitting `onProgress` * keeps the small-corpus branch on the pre-refactor sync path with - * zero yield overhead. + * zero yield overhead. Ignored when `includeLandmarkPositions` is + * `true` — that option always routes the small-corpus branch through + * the sync helper (see `includeLandmarkPositions`'s own JSDoc), so no + * progress events fire in that combination. */ onProgress?: (event: ProgressEvent) => void; + /** + * When `true`, every result entry additionally carries a + * {@link PageLandmarkReport} — each landmark instance's position, plus + * a chrome/content verdict for the six excisable types (see + * {@link ./build-page-landmark-report.js | buildPageLandmarkReport}). + * Changes the return shape from `string[]` to + * `{@link PageClusterKeyResult}[]` (see the overloads on + * `resolvePageClusterKeysInMemory`/`resolvePageClusterKeys`/ + * `resolvePageClusterKeysFromArray`). Defaults to `false`, in which + * case every existing caller's behavior and return type are + * unchanged. + * + * Not supported on the streaming path + * (`pageCount > {@link CORPUS_INLINE_THRESHOLD}`): reservoir sampling + * and Jaccard-based non-sample assignment there have no notion of + * "this page's shell tokens" to classify chrome against, and + * retrofitting one is out of scope. `resolvePageClusterKeys` throws a + * `RangeError` up front if both apply to the same call, rather than + * silently degrading semantics. + * + * On the async factory-based `resolvePageClusterKeys`, this option + * forces the small-corpus branch through the sync + * `resolvePageClusterKeysInMemory` helper regardless of `onProgress` + * — see `onProgress`'s own JSDoc. + */ + includeLandmarkPositions?: boolean; }; +/** + * One page's clustering result when `includeLandmarkPositions` is `true`: + * the same `clusterKey` every caller already gets, plus that page's + * {@link PageLandmarkReport}. + */ +export type PageClusterKeyResult = { + readonly clusterKey: string; + readonly landmarks: PageLandmarkReport; +}; + /** * Corpus size at or below which the async factory-based * `resolvePageClusterKeys` reads the entire input into an array and delegates @@ -387,6 +475,27 @@ export type ResolvePageClusterKeysOptions = TokenizeOptions & */ export const CORPUS_INLINE_THRESHOLD = 20_000; +/** + * Throws when `includeLandmarkPositions` is combined with a corpus over + * `threshold` pages (the streaming path — see `includeLandmarkPositions`'s + * own JSDoc for why it has no sample-based equivalent there). Split out from + * its call site so tests can exercise the boundary with a small injected + * `threshold` instead of constructing a 20,001-page fixture to cross the + * real {@link CORPUS_INLINE_THRESHOLD}. + * @param pageCount + * @param threshold + */ +export function assertLandmarkPositionsSupportedForPageCount( + pageCount: number, + threshold: number, +): void { + if (pageCount > threshold) { + throw new RangeError( + `resolvePageClusterKeys: includeLandmarkPositions is not supported for corpora larger than ${threshold} pages (the streaming path) — this corpus has ${pageCount}`, + ); + } +} + /** * Reservoir-sample size per block on the streaming path. Blocks larger than * this have Stage A run on a random sample of `BLOCK_SAMPLE_SIZE` pages, @@ -440,10 +549,18 @@ export const BLOCK_SAMPLE_SIZE = 100; * @param pages * @param options */ +export function resolvePageClusterKeysInMemory( + pages: readonly PageClusterSignals[], + options: ResolvePageClusterKeysOptions & { includeLandmarkPositions: true }, +): PageClusterKeyResult[]; export function resolvePageClusterKeysInMemory( pages: readonly PageClusterSignals[], options?: ResolvePageClusterKeysOptions, -): string[] { +): string[]; +export function resolvePageClusterKeysInMemory( + pages: readonly PageClusterSignals[], + options?: ResolvePageClusterKeysOptions, +): string[] | PageClusterKeyResult[] { const excludeLandmarks = options?.excludeLandmarks ?? true; const similarityThreshold = options?.similarityThreshold ?? 0.8; @@ -460,8 +577,12 @@ export function resolvePageClusterKeysInMemory( extractLandmarks(page.html), ); - // Corpus-level chrome discovery - const localLandmarkTokensByPage = computeLocalLandmarkTokens(landmarks, options); + // Corpus-level chrome discovery. `perPageInstances` is only consumed + // below when `includeLandmarkPositions` is set — computed unconditionally + // anyway since `computeLocalChromeArtifacts` already builds it internally + // for chrome discovery, so exposing it here costs nothing extra. + const { localTokensByPage: localLandmarkTokensByPage, perPageInstances } = + computeLocalChromeArtifacts(landmarks, options); const contentBlockAttribute = options?.contentBlockAttribute; const preparedHtml = pages.map((page, index) => { @@ -518,7 +639,11 @@ export function resolvePageClusterKeysInMemory( finalKeys[i] = rootKey; } } - return finalKeys; + + if (!options?.includeLandmarkPositions) return finalKeys; + + const reports = buildLandmarkReportsByCluster(finalKeys, landmarks, perPageInstances); + return finalKeys.map((clusterKey, i) => ({ clusterKey, landmarks: reports[i]! })); } /** @@ -696,10 +821,18 @@ export type PageFactory = () => * }); * ``` */ +export async function resolvePageClusterKeys( + pages: PageFactory, + options: ResolvePageClusterKeysOptions & { includeLandmarkPositions: true }, +): Promise; export async function resolvePageClusterKeys( pages: PageFactory, options?: ResolvePageClusterKeysOptions, -): Promise { +): Promise; +export async function resolvePageClusterKeys( + pages: PageFactory, + options?: ResolvePageClusterKeysOptions, +): Promise { const onProgress = options?.onProgress; // Pass 0: HTML-free — collect blocking signals (paths, stylesheetHrefs, // host) into an array. This is the only per-page state we keep across @@ -739,14 +872,27 @@ export async function resolvePageClusterKeys( // into per-block progress, so delegate to the untouched sync path — // keeping behavior byte-for-byte identical (and yield-overhead-free) // to how library-only consumers experienced this before the CLI - // progress work landed. - if (onProgress === undefined) { + // progress work landed. `includeLandmarkPositions` always routes here + // too (see its own JSDoc): `resolveSmallCorpusWithProgress` has no + // landmark-report support, and duplicating that logic into the + // progress-emitting path for a reporting feature that has nothing to + // do with progress observability isn't worth the added surface. + if (onProgress === undefined || options?.includeLandmarkPositions) { return resolvePageClusterKeysInMemory(fullPages, options); } return resolveSmallCorpusWithProgress(fullPages, onProgress, options); } - // Large corpus: streaming path. + // Large corpus: streaming path. includeLandmarkPositions has no sample- + // based equivalent (see its own JSDoc) — fail fast rather than silently + // ignoring the option or returning a semantically-wrong report. + if (options?.includeLandmarkPositions) { + assertLandmarkPositionsSupportedForPageCount( + blockingSignals.length, + CORPUS_INLINE_THRESHOLD, + ); + } + const excludeLandmarks = options?.excludeLandmarks ?? true; const similarityThreshold = options?.similarityThreshold ?? 0.8; if (!(similarityThreshold >= 0 && similarityThreshold <= 1)) { @@ -1018,9 +1164,17 @@ export async function resolvePageClusterKeys( * ]); * ``` */ +export function resolvePageClusterKeysFromArray( + pages: readonly PageClusterSignals[], + options: ResolvePageClusterKeysOptions & { includeLandmarkPositions: true }, +): Promise; export function resolvePageClusterKeysFromArray( pages: readonly PageClusterSignals[], options?: ResolvePageClusterKeysOptions, -): Promise { +): Promise; +export function resolvePageClusterKeysFromArray( + pages: readonly PageClusterSignals[], + options?: ResolvePageClusterKeysOptions, +): Promise { return resolvePageClusterKeys(() => pages, options); } diff --git a/packages/@d-zero/page-cluster/src/shell-quorum.spec.ts b/packages/@d-zero/page-cluster/src/shell-quorum.spec.ts new file mode 100644 index 00000000..248f6eb8 --- /dev/null +++ b/packages/@d-zero/page-cluster/src/shell-quorum.spec.ts @@ -0,0 +1,81 @@ +import type { LandmarkType } from './extract-landmarks.js'; +import type { PerPageLandmarkInstance } from './per-page-landmark-signatures.js'; + +import { describe, expect, test } from 'vitest'; + +import { shellQuorum } from './shell-quorum.js'; + +const DUMMY_POSITION = { + startOffset: 0, + endOffset: 0, + startLine: 1, + startColumn: 1, + endLine: 1, + endColumn: 1, +}; + +/** + * + * @param tokens + * @param type + */ +function instance( + tokens: readonly string[], + type: LandmarkType = 'header', +): PerPageLandmarkInstance { + return { + type, + tokens: new Set(tokens), + signature: JSON.stringify(tokens.toSorted()), + position: DUMMY_POSITION, + }; +} + +describe('shellQuorum', () => { + test('empty input returns an empty shell', () => { + expect(shellQuorum([])).toStrictEqual(new Set()); + }); + + test('every page with zero landmark instances yields an empty shell', () => { + expect(shellQuorum([[], [], []])).toStrictEqual(new Set()); + }); + + test('a single distinct token present on every page passes the fallback clamp', () => { + // One distinct token → autoCutThreshold's `heights.length < 2` fallback + // returns the clamp (0.8) verbatim. freq 1.0 >= 0.8 → shell. + const perPage = Array.from({ length: 5 }, () => [instance(['a'])]); + expect(shellQuorum(perPage)).toStrictEqual(new Set(['a'])); + }); + + test('the JSDoc-documented gap distribution ({1.00,1.00,0.65,0.03,0.02}) splits shell from noise', () => { + // Largest gap is between 0.65 and 0.03 (0.62) — cut lands at the + // midpoint (0.34), grouping the 0.65 token with the two 1.00 tokens + // as shell, and excluding the 0.03/0.02 tokens as page-specific noise. + const pageCount = 100; + const perPage: PerPageLandmarkInstance[][] = []; + for (let i = 0; i < pageCount; i++) { + const tokens: string[] = ['a', 'b']; + if (i < 65) tokens.push('c'); + if (i < 3) tokens.push('d'); + if (i < 2) tokens.push('e'); + perPage.push([instance(tokens)]); + } + expect(shellQuorum(perPage)).toStrictEqual(new Set(['a', 'b', 'c'])); + }); + + test('a token shared by two landmark instances on the same page counts once for that page', () => { + // page 0 carries 'x' in both header and footer; page 1 has no + // landmarks. Without per-page dedupe, page 0 would contribute count=2 + // for 'x', pushing freq to 1.0 (shell). With correct dedupe, freq is + // 1/2 = 0.5, below the single-token fallback clamp (0.8) → not shell. + const perPage = [[instance(['x'], 'header'), instance(['x'], 'footer')], []]; + expect(shellQuorum(perPage)).toStrictEqual(new Set()); + }); + + test('a page with no landmark instances still counts toward the pageCount denominator', () => { + // freq('x') = 1/2 = 0.5 (single distinct token, clamp 0.8) → not + // shell. Proves the landmark-less page isn't skipped from the count. + const perPage = [[instance(['x'])], []]; + expect(shellQuorum(perPage)).toStrictEqual(new Set()); + }); +}); diff --git a/packages/@d-zero/page-cluster/src/shell-quorum.ts b/packages/@d-zero/page-cluster/src/shell-quorum.ts new file mode 100644 index 00000000..e62b8899 --- /dev/null +++ b/packages/@d-zero/page-cluster/src/shell-quorum.ts @@ -0,0 +1,117 @@ +import type { PerPageLandmarkInstance } from './per-page-landmark-signatures.js'; + +import { autoCutThreshold } from './auto-cut-threshold.js'; + +/** + * Fallback clamp for {@link ./auto-cut-threshold.js | autoCutThreshold} when + * run on the per-landmark-instance token-frequency distribution in + * {@link ./shell-quorum.js | shellQuorum}. Independently tunable from + * `merge-cross-block-clusters.ts`'s own `QUORUM_FRACTION` (same value today, + * 0.8, by coincidence of both having been validated against the same real + * crawl corpora — not because the two are meant to move together). + */ +const SHELL_QUORUM_FALLBACK_FRACTION = 0.8; + +/** + * Discovers a unit's shell tokens by auto-cutting the per-*token* page- + * frequency histogram of every landmark instance's tokens. This is the same + * max-gap primitive used for Stage A merge-height cutoffs, applied + * recursively at the landmark-token layer. + * + * ## Why per-token and not per-signature + * + * An earlier iteration ran the histogram at the level of full landmark- + * instance signatures (canonicalized token sets). That failed on a real, + * common pattern: a shared site chrome whose markup carries a per-page + * distinguishing element (a breadcrumb, a page-title element with a page- + * specific class, a "current" state). All pages have most of the same + * tokens, but every page's full signature is distinct because tokens embed + * class names. Per-signature counting saw 5 signatures at freq 0.2 each, + * autoCutThreshold on the flat distribution returned the clamp, and the + * shell collapsed to empty even though every page shared the core header + * skeleton. Per-token counting handles the same case correctly — the shared + * skeleton tokens each hit freq 1.0. + * + * ## The histogram + * + * For every member page, all its landmark instances are tokenized and + * unioned into a single per-page token set (order-agnostic, deduped: a + * token appearing in two of the page's landmarks still counts once for + * that page). The corpus histogram is then "how many pages contain each + * token". Tokens that appear on nearly every page are the unit's chrome; + * tokens that appear on only a handful are page-specific content that + * happens to be tagged as a landmark. + * + * ## Why auto-cut instead of a hard-coded quorum + * + * A fixed 80% quorum (this file's earlier implementation) baked one + * threshold in for every unit. Real corpora don't obey a universal cutoff: + * a section-local landmark token that appears on 60% of a unit's pages is + * the section's chrome under any reasonable reading, but 80% quorum + * discards it. Auto-cut looks at the *shape* of the frequency distribution + * and picks the widest gap between adjacent frequencies — if the + * distribution is `{1.00, 1.00, 0.65, 0.03, 0.02}`, the gap between 0.65 + * and 0.03 (0.62) dwarfs everything else and the cut lands mid-gap around + * 0.34, correctly grouping the 0.65 tokens with the site-wide 1.00 ones as + * "chrome for this unit". If instead the distribution is flat, the clamp + * to {@link SHELL_QUORUM_FALLBACK_FRACTION} keeps the threshold from being + * tighter than the fallback default. + * + * ## Fallbacks + * + * A single distinct token (`heights.length < 2`) or a perfectly flat + * distribution (`maxGap === 0`) returns the + * {@link SHELL_QUORUM_FALLBACK_FRACTION} clamp verbatim — exactly the same + * 80%-quorum behavior as before. So degenerate cases degrade to the old + * contract; only richer distributions get the auto-cut benefit. + * + * A page with no landmarks contributes an empty set, deliberately, so the + * shell-corroboration jaccard between two landmark-less pages is 0 rather + * than 1 (which it would be if we handed back a ``-derived + * `{body}` fallback set to both sides). + * + * ## Reuse + * + * Originally private to {@link ./merge-cross-block-clusters.js | mergeCrossBlockClusters}'s + * Stage B L2 corroboration; exported from its own module so + * {@link ./resolve-page-cluster-keys.js | resolvePageClusterKeysInMemory}'s + * `includeLandmarkPositions` reporting path can run it once per final + * cluster to classify individual landmark instances as chrome (see + * {@link ./is-chrome-landmark-instance.js | isChromeLandmarkInstance}). + * @param perPageInstances + */ +export function shellQuorum( + perPageInstances: readonly (readonly PerPageLandmarkInstance[])[], +): ReadonlySet { + const pageCount = perPageInstances.length; + if (pageCount === 0) return new Set(); + + // Union all instance token sets per page (dedupe within page: a token + // present on both header and footer of the same page still counts once + // for that page's contribution). + const tokenPageCount = new Map(); + for (const instances of perPageInstances) { + const perPageUnion = new Set(); + for (const inst of instances) { + for (const token of inst.tokens) perPageUnion.add(token); + } + for (const token of perPageUnion) { + tokenPageCount.set(token, (tokenPageCount.get(token) ?? 0) + 1); + } + } + + if (tokenPageCount.size === 0) return new Set(); + + const frequencies: number[] = []; + for (const count of tokenPageCount.values()) { + frequencies.push(count / pageCount); + } + + const cut = autoCutThreshold(frequencies, SHELL_QUORUM_FALLBACK_FRACTION); + + const shell = new Set(); + for (const [token, count] of tokenPageCount) { + if (count / pageCount >= cut) shell.add(token); + } + return shell; +}