From ca5abfe02349106c7b924b67f855704a38fe6301 Mon Sep 17 00:00:00 2001 From: Bret Comnes Date: Tue, 15 Sep 2026 08:44:07 -0700 Subject: [PATCH 01/14] Add incremental global-data state and docs navigation indexing --- docs/data/README.md | 134 ++++- index.js | 239 +++++--- .../global-data-state-types.test.ts | 63 +++ lib/build-pages/global-data-state.js | 134 +++++ lib/build-pages/global-data-state.test.js | 375 ++++++++++++ lib/build-pages/index.js | 31 +- lib/build-pages/resolve-vars.js | 7 +- lib/helpers/find-dependencies.js | 71 +++ lib/helpers/find-dependencies.test.js | 343 +++++++++++ lib/watch-plan.js | 166 +++++- lib/watch-plan.test.js | 363 +++++++++++- site/globals/global.data.ts | 27 +- site/layouts/docs/navigation-watch.test.js | 333 ++++++++++- site/layouts/docs/navigation.js | 100 ++-- site/layouts/docs/navigation.test.js | 33 +- test-cases/generated-pages/index.test.js | 32 +- test-cases/generated-pages/streaming.test.js | 5 +- test-cases/incremental-global-data/helpers.js | 321 +++++++++++ .../incremental-global-data/index.test.js | 535 ++++++++++++++++++ test-cases/nested-layouts/index.test.js | 23 +- test-cases/page-outputs/cache.test.js | 9 +- test-cases/page-outputs/ownership.test.js | 11 +- test-cases/page-outputs/watch.test.js | 48 +- test-cases/type-exports/index.test.ts | 3 + test-cases/watch/helpers.js | 54 ++ test-cases/watch/index.test.js | 9 +- types.ts | 6 + 27 files changed, 3270 insertions(+), 205 deletions(-) create mode 100644 lib/build-pages/global-data-state-types.test.ts create mode 100644 lib/build-pages/global-data-state.js create mode 100644 lib/build-pages/global-data-state.test.js create mode 100644 lib/helpers/find-dependencies.js create mode 100644 lib/helpers/find-dependencies.test.js create mode 100644 test-cases/incremental-global-data/helpers.js create mode 100644 test-cases/incremental-global-data/index.test.js create mode 100644 test-cases/watch/helpers.js diff --git a/docs/data/README.md b/docs/data/README.md index def72ff4..a55ca476 100644 --- a/docs/data/README.md +++ b/docs/data/README.md @@ -18,7 +18,7 @@ For output definitions, see [Generation](../generation/); for ordinary configura The `global.data.ts` file is an optional file that can live anywhere in your `src` tree. The first one found wins and duplicates warn. -It runs **once per build**, after [source-backed pages](../pages/#page-files) are initialized and before generated-page factories run. +Its callback runs **once per page build**, after [source-backed pages](../pages/#page-files) are initialized and before generated-page factories run. > [!NOTE] > `global.data.js` works too. @@ -27,6 +27,8 @@ See [Supported file types](../typescript/#supported-file-types) for all availabl For data that aggregates across multiple pages — like blog indexes, sitemaps, recent-post lists, or RSS feed content — use `global.data.ts`. It is the only public build hook that receives the source-backed `PageData[]` collection. It returns an object of named, top-level values that downstream consumers can explicitly subscribe to. +Existing synchronous or asynchronous callbacks that accept only `({ pages })` remain supported, as do static default object exports such as `export default { siteName: 'My site' }`. +Incremental state is opt-in and does not change the meaning of the returned object. ```typescript // src/global.data.ts @@ -71,6 +73,128 @@ const buildGlobalData: AsyncGlobalDataFunction = async ({ pages }) = export default buildGlobalData ``` +## Incremental global data + +In watch mode, a global-data callback can retain explicit state to avoid repeating expensive indexing work for unchanged source pages. +The callback receives `{ pages, previousState, changes, setState }`: + +- `pages` is the complete current, initialized source-backed `PageData[]` collection, not just the pages selected for output rendering. + Generated pages are not included, and source membership respects discovery and draft eligibility. +- `previousState` is an isolated clone of the state retained from the last successful build, or `undefined` on reset or when no state was retained. +- `setState(next)` immediately clones and stages an explicit state value for the next successful build. + The last call wins; later mutations to `next` do not alter that snapshot. + Mutating `previousState` alone does not stage an update: without `setState`, a delta retains the old state and a reset leaves state undefined. +- `changes` is a discriminated union describing source inputs, independently of output-render filters: + - `{ kind: 'reset', reason: string, events: WatchEvent[] }` means rebuild your state from all of `pages`. + It has no `upserted` or `removed` fields. + - `{ kind: 'delta', upserted: PageData[], removed: string[], events: WatchEvent[] }` supplies current initialized pages whose inputs were invalidated or which became newly eligible, plus source paths no longer eligible. + Upserts are conservative input invalidations, not a guarantee that rendered content changed. + +Use `page.pageInfo.pageFile.filepath`, a normalized absolute source path, as the identity for an upsert. +`changes.removed` uses the same normalized absolute paths, not URLs, output filenames, or source-relative directory paths. +Treat an upsert as replacement of that source's cached record and a removal as deletion of that key. +If an upsert stops matching your own index filter, delete its cached record too. + +Both variants expose the existing raw `WatchEvent` objects through `changes.events`, with `type`, `filepath`, `name`, and `convention` fields. +The event list preserves batch order, duplicates, and removals rather than becoming a deduplicated page-change list; the initial reset has an empty event list. +Use `changes.kind`, `upserted`, and `removed` to update an index rather than reconstructing page membership from raw events. + +### Retained state and resets + +State must be structured-cloneable: plain data, arrays, `Map`, `Set`, `Date`, and other isolated values supported by `structuredClone` are suitable. +`SharedArrayBuffer` and views backed by shared memory are rejected because cloning them does not isolate their bytes; copy them into non-shared storage first. +Store extracted records, not `PageData` instances, functions, or renderers; uncloneable state passed to `setState` throws an actionable error. +State is private to the producer and separate from its ordinary returned data, so a retained `Map` does not itself participate in public-data fingerprints or subscriptions. +State is session-local, not a persistent disk cache or retained module globals. + +Rebuild from `pages` on every reset, regardless of the diagnostic `reason` string. +Resets cover the initial build and a new or restarted watch session, changes to the global-data producer or its tracked static imports, global vars or their tracked imports, Markdown settings or their tracked imports, broader global configuration changes, unknown or unreliable events, and recovery after build or dependency-analysis failure. +A reset always supplies `previousState: undefined`, even if an earlier build had retained state. + +Only a successful page/template phase, including generated pages, followed by successful cleanup commits the staged state and source-membership baseline. +Returning from `global.data.ts` or calling `setState` does not commit it early. +A failed build does not advance that baseline; the next page build resets and recomputes instead of trusting a partial candidate. +This protects retained state, not filesystem outputs: output writes and deletions are not transactional and are not rolled back after failure. + +### Source-keyed Markdown search example + +This example retains one rendered Markdown document per source file for a search-index consumer. +On reset it indexes all current Markdown pages; on a delta it renders only upserted Markdown pages and removes records that are no longer eligible. + +```typescript +// src/global.data.ts +import type { AsyncGlobalDataFunction } from '@domstack/static/types.js' + +type SourceVars = { title?: string } +type SearchDocument = { url: string, title: string, html: string } +type SearchData = { searchDocuments: SearchDocument[] } +type SearchState = Map + +const globalData: AsyncGlobalDataFunction = async ({ + pages, previousState, changes, setState, +}) => { + const documents = changes.kind === 'reset' + ? new Map() + : previousState ?? new Map() + const upserted = changes.kind === 'reset' ? pages : changes.upserted + + if (changes.kind === 'delta') { + for (const filepath of changes.removed) documents.delete(filepath) + } + + for (const page of upserted) { + const filepath = page.pageInfo.pageFile.filepath + if (page.pageInfo.type !== 'md') { + documents.delete(filepath) + continue + } + documents.set(filepath, { + url: page.pageInfo.url, + title: page.vars.title ?? page.pageInfo.url, + html: await page.renderInnerPage(), + }) + } + + setState(documents) + return { + searchDocuments: [...documents.entries()] + .sort(([a], [b]) => a < b ? -1 : a > b ? 1 : 0) + .map(([, document]) => document), + } +} + +export default globalData +``` + +The indexed Markdown pages must not themselves subscribe to global data when `renderInnerPage()` is called here; see [Rendering page content](#rendering-page-content). +A search template or generated-page factory subscribes with `export const dataDeps = ['searchDocuments']` and reads `data.searchDocuments` as usual. +The returned array of plain records is ordinary public data with the same top-level fingerprinting rules as a full recomputation. +Stable source-key ordering avoids changing its fingerprint merely because records were inserted in a different order. +`dataDeps` remains authoritative for consumer access and data-driven invalidation; retaining state neither adds subscriptions nor replaces output dependency tracking. + +### How this documentation site uses the index + +DOMStack's own `site/globals/global.data.ts` retains a source-keyed index of documentation titles, heading anchors, and ordering/group/parent metadata. +Only upserted documentation pages are rendered and parsed; removed or newly ineligible sources are dropped from the index. +The navigation helpers in `site/layouts/docs/navigation.js` then regenerate the ordered, nested `docsNavigation` and `docsIndexHtml` from those plain records without rendering Markdown again. +Parent/child nesting is assembled on fresh entries rather than mutating cached heading arrays, so repeated builds do not accumulate children and deletions or parent changes take effect immediately. +A body-only edit updates one record, but unchanged TOC fingerprints prevent other documentation pages from rebuilding. + +### Tracking, batching, and performance limits + +Dependency tracking covers relative static ESM imports and re-exports in the non-ignored watched source tree, including imported JSON files. +It does not automatically track bare-package internals, CommonJS `require()`, dynamic imports, arbitrary filesystem reads, network responses, environment variables, or other dynamic inputs. +When those inputs change, explicitly invalidate the relevant source input (or the producer for a full state reset), or restart the watch session; do not assume a raw event or a cached record discovers the change. + +Watch events can be batched while a build is running, and subsequent builds converge on the current source tree. +A batch is not a snapshot of filesystem contents at event time, and an intermediate build may observe newer contents before queued events are processed. +Write index updates as replacements and deletions that tolerate repeated invalidations, not as exactly-once event operations. + +Incremental indexing can reduce the producer's Markdown rendering work, but it does not make the entire build O(changed pages). +All source pages are still initialized, retained state is cloned, and returned data is still fingerprinted; this example also sorts the complete index on each callback. +A counter around `renderInnerPage()` in this index measures indexing render calls only, not total source-page initialization, downstream rendering, or filesystem output writes. +Those are separate stages, and rendering an output does not necessarily write it when its bytes are unchanged. + ## Data subscriptions The returned object is not merged into `vars`. @@ -142,9 +266,9 @@ When one layout calls another layout function directly, the composing layout mus - Gives pages, layouts, templates, and page factories only their declared top-level keys through `data`. - Keeps global data separate from ordinary `vars`, so derived values cannot silently collide with page or layout configuration. - Runs inside the worker process (same as all other dynamic imports) to avoid ESM caching issues. -- Skipped entirely if no `global.data.*` file exists — zero overhead. +- No producer callback runs if no `global.data.*` file exists. - In watch mode, DOMStack fingerprints each top-level returned value and rebuilds only consumers subscribed to changed keys. -- Editing `global.data.*` or one of its statically imported helpers recomputes data; a shared helper also rebuilds its direct page, layout, template, and factory consumers. +- Editing `global.data.*` or one of its tracked statically imported helpers in the watched source tree resets retained state and recomputes data; a shared helper also rebuilds its direct page, layout, template, and factory consumers. - Values composed of JSON-safe primitives, arrays, and plain objects get stable fingerprints; opaque values such as functions, class instances, maps, sets, or cycles conservatively invalidate their subscribers on every page build. - A declaration naming a missing key fails the build, and access to an existing but undeclared key throws a focused error. @@ -180,6 +304,10 @@ export default globalData Use `AsyncGlobalDataFunction` instead when the implementation needs to await rendering, network requests, or other asynchronous work. For typed source input, use `GlobalDataFunction` or its async counterpart. Helpers can accept `GlobalDataFunctionParams['pages']` without recovering types from the full global-data result. +For retained state, add the final generic: `GlobalDataFunction`, `AsyncGlobalDataFunction`, or `GlobalDataFunctionParams`. +`State` defaults to `unknown`; existing result and source generic positions are unchanged. +`GlobalDataChanges`, `GlobalDataResetChanges`, `GlobalDataDeltaChanges`, and `WatchEvent` are also exported from `@domstack/static/types.js`. +Manually constructed `GlobalDataFunctionParams` objects must include `previousState`, `changes`, and `setState`, even though callbacks may ignore those fields. ## Global data caveats diff --git a/index.js b/index.js index 492ae6c1..2d036a43 100644 --- a/index.js +++ b/index.js @@ -2,7 +2,7 @@ /** * @import { DomStackOpts, Results, SiteData } from './lib/builder.js' - * @import { Stats } from 'node:fs' + * @import { FSWatcher } from 'chokidar' * @import { WorkerBuildStepResult } from './lib/build-pages/index.js' * @import { PageInfo, TemplateInfo, PagesFileInfo } from './lib/identify-pages.js' @@ -13,6 +13,7 @@ * @import { WatchDependencyState } from './lib/build-pages/watch-dependencies.js' * @import { PageOutputCache } from './lib/build-pages/page-builders/page-output-writer.js' * @import { WatchSnapshot, WatchEvent, WatchPlan } from './lib/watch-plan.js' + * @import { GlobalDataBaseline, GlobalDataInputChanges } from './lib/build-pages/global-data-state.js' * @typedef {{ dispose: () => Promise }} DisposableBuildContext * @typedef {{ pageFilePath: string, sourcePageFilePath?: string | undefined, pagesFilePath?: string | undefined, layoutNames: string[], outputs?: DomstackManifestRecord[] | undefined }} WatchedPageReport * @typedef {object} WatchSession @@ -20,9 +21,13 @@ * @property {AbortController} cancellation - Cancels event waits, not resource acquisition. * @property {Promise} startupWork - The current resource-acquiring startup phase; never the user callback. * @property {Promise | null} shutdown - Shared by explicit stops and startup failure cleanup. + * @property {WatchEvent[]} pendingEvents + * @property {boolean} drainScheduled + * @property {GlobalDataBaseline | null} globalDataBaseline */ import { once } from 'events' -import assert from 'node:assert' +import { setImmediate } from 'node:timers/promises' + import { lstat, mkdtemp, readFile, rm } from 'node:fs/promises' import { tmpdir } from 'node:os' import chokidar from 'chokidar' @@ -33,7 +38,7 @@ import ignore from 'ignore' import { watch as cpxWatch } from 'cpx2' import { inspect } from 'util' import { createServer } from '@domstack/sync' -import { find } from '@11ty/dependency-tree-typescript' +import { findDependencies as find } from './lib/helpers/find-dependencies.js' import { assertInsideDest } from './lib/helpers/path.js' import { getCopyGlob } from './lib/build-static/index.js' @@ -43,7 +48,7 @@ import { builder } from './lib/builder.js' import { buildEsbuildWatch } from './lib/build-esbuild/index.js' import { buildPages } from './lib/build-pages/index.js' import { identifyPages } from './lib/identify-pages.js' -import { classifyWatchEvent, planWatchEvent, planBundleChange } from './lib/watch-plan.js' +import { classifyWatchEvent, planWatchEvent, planWatchBatch, planBundleChange } from './lib/watch-plan.js' import { ensureDest } from './lib/helpers/ensure-dest.js' import { DomStackAggregateError } from './lib/helpers/domstack-aggregate-error.js' import { createDomStackLogger } from './lib/logger.js' @@ -100,8 +105,15 @@ export class DomStack { #pagesFileDepMap = new Map() /** @type {Set} Imported inputs of global.data, including its entry file. */ #globalDataDepPaths = new Set() + /** @type {Set} */ + #globalVarsDepPaths = new Set() + /** @type {Set} */ + #markdownDepPaths = new Set() + #dependencyAnalysisFailed = false /** @type {Set} absolute filepaths of esbuild entry points */ #esbuildEntryPoints = new Set() + /** @type {Set} Known browser-only helpers can skip the page phase. */ + #esbuildDepPaths = new Set() /** @type {Map>} source page or *.pages.* filepath → owned absolute output paths */ #pageOutputMap = new Map() /** @type {PageOutputCache} Successful writes, including those before an iterator failure. */ @@ -191,6 +203,9 @@ export class DomStack { cancellation: new AbortController(), startupWork: Promise.resolve(), shutdown: null, + pendingEvents: [], + drainScheduled: false, + globalDataBaseline: null, } this.#watchSession = session try { @@ -217,9 +232,9 @@ export class DomStack { */ async #startWatch (session, { serve, onInitialBuild }) { const { signal } = session.cancellation - const preparation = this.#prepareWatch(signal) + const preparation = this.#prepareWatch(session) session.startupWork = preparation - const { report, watcher, ready } = await preparation + const { report, ready } = await preparation await ready if (signal.aborted) return report @@ -234,28 +249,18 @@ export class DomStack { } session.state = 'watching' - const enqueue = (/** @type {() => Promise} */ fn) => { - this.#enqueueBuild(session, fn) - } - - watcher.on('add', path => { - enqueue(() => this.#handleWatchEvent(path, 'added')) - }) - watcher.on('change', path => { - assert(this.#src) - assert(this.#dest) - enqueue(() => this.#handleWatchEvent(path, 'change')) - }) - watcher.on('unlink', path => { - enqueue(() => this.#handleWatchEvent(path, 'removed')) - }) - watcher.on('error', err => errorLogger(err, this.#logger)) + this.#scheduleWatchBatch(session) return report } - /** @param {AbortSignal} signal */ - async #prepareWatch (signal) { + /** @param {WatchSession} session */ + async #prepareWatch (session) { + const { signal } = session.cancellation + // Establish observation before discovery. Initial scan adds are not edits; + // subsequent events stay buffered until startup and the user callback finish. + const { watcher, ready } = this.#createSourceWatcher(session) + await ready // ── Initial build (inline, not via builder()) ──────────────────────── const siteData = await identifyPages(this.#src, this.opts) @@ -302,6 +307,8 @@ export class DomStack { this.#updatePageLayoutNames(pageBuildResults.report.pages, true) this.#pageBuildFailed = false this.#watchDependencies = pageBuildResults.report.watchDependencies ?? null + session.globalDataBaseline = pageBuildResults.report.globalDataBaseline ?? null + delete pageBuildResults.report.globalDataBaseline delete pageBuildResults.report.watchDependencies delete pageBuildResults.report.rebuiltPagesFilePaths buildLogger(report, this.#logger) @@ -325,26 +332,22 @@ export class DomStack { const copyErrors = copyStartup.filter(result => result.status === 'rejected').map(result => result.reason) if (copyErrors.length) throw new AggregateError(copyErrors, 'Copy watch startup failed') - // ── Chokidar watcher ───────────────────────────────────────────────── + return { report, watcher, ready } + } + + /** @param {WatchSession} session */ + #createSourceWatcher (session) { + const { signal } = session.cancellation const ig = ignore().add(this.opts.ignore ?? []) const anymatch = (/** @type {string} */name) => ig.ignores(relname(this.#src, name)) const watcher = chokidar.watch(this.#src, { - /** - * Determines whether a given path should be ignored by the watcher. - * - * @param {string} filePath - The path to the file or directory. - * @param {Stats} [stats] - The stats object for the path (may be undefined). - * @returns {boolean} - Returns true if the path should be ignored. - */ - ignored: (filePath, stats) => { - return ( - anymatch(filePath) || - Boolean((stats?.isFile() && !isProcessedFile(filePath))) - ) - }, + // Observe non-page extensions too (for example statically imported JSON). + // Route only processed files and known dependencies after maps are ready. + ignored: filePath => anymatch(filePath), persistent: true, + ignoreInitial: true, // Increase the atomic write window so editors that do slow atomic saves // (write to a temp file then rename) emit a `change` event rather than // `unlink` + `add`, which would otherwise trigger unnecessary full rebuilds. @@ -352,13 +355,22 @@ export class DomStack { }) this.#watcher = watcher + const record = (/** @type {string} */ path, /** @type {WatchEvent['type']} */ type) => { + if (session.state === 'stopping') return + session.pendingEvents.push(classifyWatchEvent(type, path)) + this.#scheduleWatchBatch(session) + } + watcher.on('add', path => record(path, 'added')) + watcher.on('change', path => record(path, 'change')) + watcher.on('unlink', path => record(path, 'removed')) + watcher.on('error', err => errorLogger(err, this.#logger)) // Attach the listener before returning; the watcher can become ready before // the caller resumes. Cancellation settles this wait even without a ready event. const ready = once(watcher, 'ready', { signal }).catch(error => { if (!signal.aborted || error.name !== 'AbortError') throw error }) - return { report, watcher, ready } + return { watcher, ready } } /** @@ -416,7 +428,7 @@ export class DomStack { * Full rebuild: re-identify pages, restart esbuild, rebuild all pages, rebuild maps. * Used for structural changes (add/unlink), global.vars.*, esbuild.settings.*. */ - async #fullRebuild () { + async #fullRebuild (/** @type {GlobalDataInputChanges} */ inputChanges) { this.#logger.info('Triggering full rebuild...') // Dispose the old esbuild context if (this.#esbuildContext) { @@ -427,9 +439,7 @@ export class DomStack { const siteData = await identifyPages(this.#src, this.opts) if (siteData.errors.length > 0) { - this.#logger.error(`identifyPages errors: -${siteData.errors.map(err => ` ${err.message}`).join('\n')}`) - return + throw new DomStackAggregateError(siteData.errors, 'Page discovery failed.', siteData) } await ensureDest(this.#dest, siteData) @@ -438,8 +448,7 @@ ${siteData.errors.map(err => ` ${err.message}`).join('\n')}`) this.#esbuildContext = context this.#siteData = siteData - await this.#runPageBuild(siteData) - await this.#rebuildMaps(siteData) + await this.#runPageBuild(siteData, null, null, null, inputChanges) } /** @returns {WatchSnapshot | undefined} */ @@ -456,52 +465,67 @@ ${siteData.errors.map(err => ` ${err.message}`).join('\n')}`) pagesFileDepMap: this.#pagesFileDepMap, pagesFileLayoutMap: this.#pagesFileLayoutMap, globalDataDepPaths: this.#globalDataDepPaths, + globalVarsDepPaths: this.#globalVarsDepPaths, + markdownDepPaths: this.#markdownDepPaths, + dependencyAnalysisFailed: this.#dependencyAnalysisFailed, pageBuildFailed: this.#pageBuildFailed, esbuildEntryPoints: this.#esbuildEntryPoints, + esbuildDepPaths: this.#esbuildDepPaths, } } - /** - * @param {string} changedPath - * @param {'change' | 'added' | 'removed'} type - */ - async #handleWatchEvent (changedPath, type) { + /** @param {WatchEvent[]} events */ + async #handleWatchBatch (events) { const snapshot = this.#watchSnapshot() - if (!snapshot) return - const event = classifyWatchEvent(type, changedPath) - await this.#executeWatchPlan(planWatchEvent(snapshot, event), event) + events = events.filter(event => isProcessedFile(event.filepath) || this.#pageBuildFailed || this.#dependencyAnalysisFailed || + this.#globalDataDepPaths.has(event.filepath) || this.#globalVarsDepPaths.has(event.filepath) || + this.#markdownDepPaths.has(event.filepath) || this.#layoutDepMap.has(event.filepath) || + this.#pageDepMap.has(event.filepath) || this.#templateDepMap.has(event.filepath) || + this.#pagesFileDepMap.has(event.filepath) || this.#esbuildDepPaths.has(event.filepath)) + const event = events[0] + if (!snapshot || !event) return + const { plan, inputChanges } = planWatchBatch(snapshot, events) + // Keep the existing precise bundle-membership path for a single event. + // Mixed batches use the conservative rediscovery plan instead. + const singlePlan = events.length === 1 ? planWatchEvent(snapshot, event) : null + await this.#executeWatchPlan( + snapshot.pageBuildFailed + ? { kind: 'full', message: 'Rediscovering and retrying all pages after the previous build failure...' } + : singlePlan?.kind === 'restart' ? singlePlan : plan, + event, inputChanges + ) } /** * Keep resource ownership and successful-build state updates in the executor. * @param {WatchPlan} plan * @param {WatchEvent} event + * @param {GlobalDataInputChanges} inputChanges * @returns {Promise} */ - async #executeWatchPlan (plan, event) { + async #executeWatchPlan (plan, event, inputChanges) { if (plan.message) this.#logger.info(plan.message) if (plan.kind === 'skip') return if (plan.kind === 'full') { - await this.#fullRebuild() + await this.#fullRebuild(inputChanges) return } if (plan.kind === 'restart') { - await this.#restartEsbuildForEvent(event) + await this.#restartEsbuildForEvent(event, inputChanges) return } if (!this.#siteData) return if (plan.pages || plan.templates) { logRebuildTree(event.name, this.#logger, new Set(plan.pages), new Set(plan.templates)) } - await this.#runPageBuild(this.#siteData, plan.pageFilterPaths, plan.templateFilterPaths, plan.pagesFileFilterPaths) + await this.#runPageBuild(this.#siteData, plan.pageFilterPaths, plan.templateFilterPaths, plan.pagesFileFilterPaths, inputChanges) } - /** @param {WatchEvent} event */ - async #restartEsbuildForEvent (event) { + /** @param {WatchEvent} event @param {GlobalDataInputChanges} inputChanges */ + async #restartEsbuildForEvent (event, inputChanges) { const siteData = await identifyPages(this.#src, this.opts) if (siteData.errors.length > 0) { - this.#logger.error(`identifyPages errors:\n${siteData.errors.map(err => ` ${err.message}`).join('\n')}`) - return + throw new DomStackAggregateError(siteData.errors, 'Page discovery failed.', siteData) } await ensureDest(this.#dest, siteData) if (this.#esbuildContext) { @@ -517,7 +541,7 @@ ${siteData.errors.map(err => ` ${err.message}`).join('\n')}`) // Successful page builds refresh their own maps. Service workers have no // HTML consumers, but their entry map still changes. if (plan.kind === 'skip') await this.#rebuildMaps(siteData) - await this.#executeWatchPlan(plan, event) + await this.#executeWatchPlan(plan, event, inputChanges) } /** @@ -527,8 +551,9 @@ ${siteData.errors.map(err => ` ${err.message}`).join('\n')}`) * @param {string[] | null} [pageFilterPaths] * @param {string[] | null} [templateFilterPaths] * @param {string[] | null} [pagesFileFilterPaths] + * @param {GlobalDataInputChanges} [inputChanges] */ - async #runPageBuild (siteData, pageFilterPaths = null, templateFilterPaths = null, pagesFileFilterPaths = null) { + async #runPageBuild (siteData, pageFilterPaths = null, templateFilterPaths = null, pagesFileFilterPaths = null, inputChanges) { // Retry the complete page phase after a failure: neither subscriptions nor // layout routing from a failed build can safely drive an incremental retry. if (this.#pageBuildFailed) pageFilterPaths = templateFilterPaths = pagesFileFilterPaths = null @@ -538,6 +563,8 @@ ${siteData.errors.map(err => ` ${err.message}`).join('\n')}`) ...(pageFilterPaths ? { pageFilterPaths } : {}), ...(templateFilterPaths ? { templateFilterPaths } : {}), ...(pagesFileFilterPaths ? { pagesFileFilterPaths } : {}), + previousGlobalDataBaseline: this.#watchSession?.globalDataBaseline, + globalDataInputChanges: inputChanges, previousWatchDependencies: this.#watchDependencies, previousPageOutputCache: this.#pageOutputCache, trackWatchDependencies: true, @@ -552,17 +579,26 @@ ${siteData.errors.map(err => ` ${err.message}`).join('\n')}`) }) } const isFiltered = pageFilterPaths !== null || templateFilterPaths !== null || pagesFileFilterPaths !== null - await this.#removeObsoletePageOutputs(pageBuildResults, isFiltered) + try { + await this.#removeObsoletePageOutputs(pageBuildResults, isFiltered) + } catch (error) { + // The worker's writes succeeded even if cleanup did not. Keep their + // ownership for recovery without committing producer/subscriber state. + this.#rememberPartialPageOutputs(pageBuildResults) + throw error + } this.#updatePageLayoutNames(pageBuildResults.report.pages, !isFiltered) if (!isFiltered) { this.#pagesFileLayoutMap = getPagesFileLayoutMap(pageBuildResults.report.pages) } else { updatePagesFileLayoutMap(this.#pagesFileLayoutMap, pageBuildResults.report.rebuiltPagesFilePaths ?? [], pageBuildResults.report.pages) } + await this.#rebuildMaps(siteData) this.#watchDependencies = pageBuildResults.report.watchDependencies ?? this.#watchDependencies + if (this.#watchSession) this.#watchSession.globalDataBaseline = pageBuildResults.report.globalDataBaseline ?? null + delete pageBuildResults.report.globalDataBaseline delete pageBuildResults.report.watchDependencies delete pageBuildResults.report.rebuiltPagesFilePaths - await this.#rebuildMaps(siteData) this.#pageBuildFailed = false buildLogger( isFiltered ? pageBuildResults : { warnings: pageBuildResults.warnings, siteData, pageBuildResults }, @@ -632,18 +668,27 @@ ${siteData.errors.map(err => ` ${err.message}`).join('\n')}`) this.#templateOutputMap = templates } - /** - * @param {WatchSession} session - * @param {() => Promise} fn - */ - #enqueueBuild (session, fn) { - if (session.state !== 'watching') return + /** @param {WatchSession} session */ + #scheduleWatchBatch (session) { + if (session.state !== 'watching' || session.drainScheduled || !session.pendingEvents.length) return + session.drainScheduled = true this.#buildLock = this.#buildLock.then(async () => { - if (session.state !== 'watching') return try { - await fn() - } catch (err) { - errorLogger(err, this.#logger) + while (session.state === 'watching' && session.pendingEvents.length) { + // Coalesce the current event-loop turn, then detach. Events observed + // during asynchronous build work belong to the next batch, never this one. + await setImmediate() + if (session.state !== 'watching') break + const events = session.pendingEvents.splice(0) + try { + await this.#handleWatchBatch(events) + } catch (err) { + this.#pageBuildFailed = true + errorLogger(err, this.#logger) + } + } + } finally { + session.drainScheduled = false } }) } @@ -675,15 +720,22 @@ ${siteData.errors.map(err => ` ${err.message}`).join('\n')}`) const pageDepMap = /** @type {Map>} */ (new Map()) const templateDepMap = /** @type {Map>} */ (new Map()) const pagesFileDepMap = /** @type {Map>} */ (new Map()) - const globalDataDepPaths = new Set() - if (siteData.globalData) { - globalDataDepPaths.add(siteData.globalData.filepath) + let dependencyAnalysisFailed = false + /** @param {string | undefined} filepath */ + const rootDependencies = async filepath => { + const paths = new Set(/** @type {string[]} */ ([])) + if (!filepath) return paths + paths.add(resolve(filepath)) try { - for (const dep of await find(siteData.globalData.filepath)) globalDataDepPaths.add(resolve(dep)) + for (const dep of await find(filepath)) paths.add(resolve(dep)) } catch { - // Static import analysis is best-effort, as for page and layout helpers. + dependencyAnalysisFailed = true } + return paths } + const globalDataDepPaths = await rootDependencies(siteData.globalData?.filepath) + const globalVarsDepPaths = await rootDependencies(siteData.globalVars?.filepath) + const markdownDepPaths = await rootDependencies(siteData.markdownItSettings?.filepath) // layoutFileMap: layout filepath → layoutName for (const layout of Object.values(siteData.layouts)) { @@ -700,7 +752,7 @@ ${siteData.errors.map(err => ` ${err.message}`).join('\n')}`) layoutDepMap.get(absPath)?.add(layout.layoutName) } } catch { - // dep analysis is best-effort + dependencyAnalysisFailed = true } } @@ -720,7 +772,7 @@ ${siteData.errors.map(err => ` ${err.message}`).join('\n')}`) // pageDepMap: dep filepath → Set for (const pageInfo of siteData.pages) { - const filesToTrack = [pageInfo.pageFile.filepath] + const filesToTrack = /\.[cm]?[jt]sx?$/.test(pageInfo.pageFile.filepath) ? [pageInfo.pageFile.filepath] : [] if (pageInfo.pageVars) filesToTrack.push(pageInfo.pageVars.filepath) for (const file of filesToTrack) { try { @@ -731,7 +783,7 @@ ${siteData.errors.map(err => ` ${err.message}`).join('\n')}`) pageDepMap.get(absPath)?.add(pageInfo) } } catch { - // best-effort + dependencyAnalysisFailed = true } } } @@ -746,7 +798,7 @@ ${siteData.errors.map(err => ` ${err.message}`).join('\n')}`) templateDepMap.get(absPath)?.add(templateInfo) } } catch { - // best-effort + dependencyAnalysisFailed = true } } @@ -760,6 +812,7 @@ ${siteData.errors.map(err => ` ${err.message}`).join('\n')}`) pagesFileDepMap.get(absPath)?.add(pagesFileInfo) } } catch (err) { + dependencyAnalysisFailed = true const message = err instanceof Error ? err.message : String(err) this.#logger.debug(`Could not analyze dependencies for pages file "${pagesFileInfo.pagesFile.relname}": ${message}`) } @@ -776,6 +829,16 @@ ${siteData.errors.map(err => ` ${err.message}`).join('\n')}`) for (const asset of layoutBundleAssets(layout)) esbuildEntryPoints.add(resolve(asset.filepath)) } + const esbuildDepPaths = new Set(/** @type {string[]} */ ([])) + for (const filepath of esbuildEntryPoints) { + if (!/\.[cm]?[jt]sx?$/.test(filepath)) continue + try { + for (const dep of await find(filepath)) esbuildDepPaths.add(resolve(dep)) + } catch { + // Unknown browser helpers still take the conservative reset path. + } + } + this.#layoutDepMap = layoutDepMap this.#layoutPageMap = layoutPageMap this.#pageFileMap = pageFileMap @@ -784,7 +847,11 @@ ${siteData.errors.map(err => ` ${err.message}`).join('\n')}`) this.#templateDepMap = templateDepMap this.#pagesFileDepMap = pagesFileDepMap this.#globalDataDepPaths = globalDataDepPaths + this.#globalVarsDepPaths = globalVarsDepPaths + this.#markdownDepPaths = markdownDepPaths + this.#dependencyAnalysisFailed = dependencyAnalysisFailed this.#esbuildEntryPoints = esbuildEntryPoints + this.#esbuildDepPaths = esbuildDepPaths } /** @@ -830,6 +897,10 @@ ${siteData.errors.map(err => ` ${err.message}`).join('\n')}`) Promise.resolve().then(() => this.#esbuildContext?.dispose()), Promise.resolve().then(() => this.#syncServer?.exit()), ])) + session.pendingEvents = [] + session.globalDataBaseline = null + this.#watchDependencies = null + this.#pageBuildFailed = false this.#watcher = null this.#cpxWatchers = [] this.#esbuildContext = null diff --git a/lib/build-pages/global-data-state-types.test.ts b/lib/build-pages/global-data-state-types.test.ts new file mode 100644 index 00000000..429f245e --- /dev/null +++ b/lib/build-pages/global-data-state-types.test.ts @@ -0,0 +1,63 @@ +import type { + AsyncGlobalDataFunction, + GlobalDataFunction, + GlobalDataFunctionParams, + GlobalDataChanges, + GlobalDataDeltaChanges, + GlobalDataResetChanges, + PageData, + WatchEvent, +} from '../../types.ts' +import type { BuildPagesOptions, PageBuilderReport } from './index.js' +import type { GlobalDataBaseline, GlobalDataInputChanges } from './global-data-state.js' + +type Vars = { title: string } +type Data = { titles: string[] } +type State = { titles: Map } + +export function checkGlobalDataStateTypes (pages: PageData[], events: WatchEvent[]) { + const reset: GlobalDataResetChanges = { kind: 'reset', reason: 'initial', events } + const delta: GlobalDataDeltaChanges = { kind: 'delta', upserted: pages, removed: ['/removed.md'], events } + const changes: GlobalDataChanges = delta + const params: GlobalDataFunctionParams = { + pages, + previousState: undefined, + changes, + setState (next) { next.titles.set('/page.md', 'Title') }, + } + const stateful: GlobalDataFunction = ({ pages, changes, previousState, setState }) => { + const titles = previousState?.titles ?? new Map() + if (changes.kind === 'reset') { + const reason: string = changes.reason + titles.set(reason, pages[0]?.vars.title ?? '') + // @ts-expect-error Reset changes do not contain a partial upsert list. + String(changes.upserted) + } else { + changes.upserted.forEach(page => titles.set(page.pageInfo.pageFile.filepath, page.vars.title)) + changes.removed.forEach(path => titles.delete(path)) + // @ts-expect-error Delta changes do not have a reset reason. + String(changes.reason) + } + setState({ titles }) + // @ts-expect-error State generic constrains updates independently of returned data. + setState({ titles: ['wrong'] }) + return { titles: [...titles.values()] } + } + const asyncStateful: AsyncGlobalDataFunction = async context => stateful(context) + // Existing generic positions and pages-only callback implementations remain valid. + const legacy: GlobalDataFunction = ({ pages }) => ({ titles: pages.map(page => page.vars.title) }) + const asyncLegacy: AsyncGlobalDataFunction = async ({ pages }) => ({ titles: pages.map(page => page.vars.title) }) + const defaultState: GlobalDataFunction = ({ previousState, setState }) => { + const state: unknown = previousState + // @ts-expect-error Default state is unknown until narrowed or explicitly parameterized. + setState(previousState.titles) + setState(state) + return {} + } + const baseline: GlobalDataBaseline = { state: { titles: new Map() }, sourcePaths: [] } + const input: GlobalDataInputChanges = { resetReason: undefined, upsertedPaths: [], events } + const options: BuildPagesOptions = { previousGlobalDataBaseline: baseline, globalDataInputChanges: input } + const absent: BuildPagesOptions = { previousGlobalDataBaseline: null, globalDataInputChanges: undefined } + const report: PageBuilderReport = { pages: [], templates: [], globalDataBaseline: baseline } + return { reset, delta, params, stateful, asyncStateful, legacy, asyncLegacy, defaultState, options, absent, report } +} diff --git a/lib/build-pages/global-data-state.js b/lib/build-pages/global-data-state.js new file mode 100644 index 00000000..4729f75f --- /dev/null +++ b/lib/build-pages/global-data-state.js @@ -0,0 +1,134 @@ +/** + * @import { PageData } from './page-data.js' + * @import { GlobalDataFunctionParams } from './index.js' + * @import { WatchEvent } from '../watch-plan.js' + */ +import { resolve } from 'node:path' + +/** + * Internal candidate committed by the main build only after successful cleanup. + * @typedef {object} GlobalDataBaseline + * @property {unknown} state + * @property {string[]} sourcePaths + */ + +/** + * Internal input changes supplied by the watch planner, not output filters. + * @typedef {object} GlobalDataInputChanges + * @property {string | undefined} [resetReason] + * @property {string[]} upsertedPaths + * @property {WatchEvent[]} events + */ + +/** + * @typedef {object} GlobalDataResetChanges + * @property {'reset'} kind + * @property {string} reason + * @property {WatchEvent[]} events + */ + +/** + * @template {Record} [T=any] - Source-page vars. + * @template [U=any] - Source-page render values. + * @typedef {object} GlobalDataDeltaChanges + * @property {'delta'} kind + * @property {PageData[]} upserted - Current initialized pages, identified by their normalized absolute pageInfo.pageFile.filepath. + * @property {string[]} removed - Normalized absolute source-page paths no longer eligible. + * @property {WatchEvent[]} events + */ + +/** + * @template {Record} [T=any] + * @template [U=any] + * @typedef {GlobalDataResetChanges | GlobalDataDeltaChanges} GlobalDataChanges + */ + +/** + * @param {unknown} state + * @param {string} operation + * @returns {unknown} + */ +function cloneState (state, operation) { + let cloned + try { + cloned = structuredClone(state) + } catch (cause) { + throw new Error(`global.data ${operation} requires structured-cloneable state. Store plain data, Maps, Sets, or other structured-cloneable values instead of functions, PageData instances, or renderers.`, { cause }) + } + // Inspect only retained values: cloning evaluates source getters once and drops + // private fields, but SharedArrayBuffer still aliases its original memory. + rejectSharedMemory(cloned, operation) + return cloned +} + +/** + * @param {unknown} value + * @param {string} operation + * @param {Set} [seen] + */ +function rejectSharedMemory (value, operation, seen = new Set()) { + if (value === null || typeof value !== 'object' || seen.has(value)) return + seen.add(value) + if (value instanceof SharedArrayBuffer || (ArrayBuffer.isView(value) && value.buffer instanceof SharedArrayBuffer)) { + throw new Error(`global.data ${operation} cannot retain shared-memory state (SharedArrayBuffer or views backed by it): structuredClone shares their memory. Copy the bytes into a non-shared ArrayBuffer or typed array before storing state.`) + } + // Cloned views have no custom properties; avoid walking every byte or element. + if (ArrayBuffer.isView(value)) return + if (value instanceof Map) { + for (const [key, entry] of value) { + rejectSharedMemory(key, operation, seen) + rejectSharedMemory(entry, operation, seen) + } + } else if (value instanceof Set) { + for (const entry of value) rejectSharedMemory(entry, operation, seen) + } + // Include non-enumerable retained properties such as Error.cause. + for (const descriptor of Object.values(Object.getOwnPropertyDescriptors(value))) { + if ('value' in descriptor) rejectSharedMemory(descriptor.value, operation, seen) + } +} + +/** + * Keep the callback's mutable previousState separate from the candidate so that + * only setState can commit changes. Snapshot membership before calling user code. + * + * @param {object} params + * @param {PageData[]} params.pages + * @param {GlobalDataBaseline | null | undefined} [params.previousGlobalDataBaseline] + * @param {GlobalDataInputChanges | undefined} [params.globalDataInputChanges] + */ +export function createGlobalDataState ({ pages, previousGlobalDataBaseline, globalDataInputChanges }) { + const sourcePaths = [...new Set(pages.map(page => resolve(page.pageInfo.pageFile.filepath)))] + const sourcePathSet = new Set(sourcePaths) + const reset = !previousGlobalDataBaseline || globalDataInputChanges?.resetReason !== undefined + let state = reset ? undefined : cloneState(previousGlobalDataBaseline.state, 'previous state') + const events = structuredClone(globalDataInputChanges?.events ?? []) + const previousPaths = new Set(previousGlobalDataBaseline?.sourcePaths.map(path => resolve(path))) + const upsertedPaths = new Set(globalDataInputChanges?.upsertedPaths.map(path => resolve(path))) + + /** @type {GlobalDataFunctionParams} */ + const context = { + pages, + previousState: reset ? undefined : cloneState(state, 'previous state'), + changes: reset + ? { kind: 'reset', reason: globalDataInputChanges?.resetReason ?? 'initial', events } + : { + kind: 'delta', + upserted: pages.filter(page => { + const path = resolve(page.pageInfo.pageFile.filepath) + return upsertedPaths.has(path) || !previousPaths.has(path) + }), + removed: [...previousPaths].filter(path => !sourcePathSet.has(path)), + events, + }, + setState (next) { + state = cloneState(next, 'setState(next)') + }, + } + + return { + context, + /** @returns {GlobalDataBaseline} */ + getBaseline: () => ({ state, sourcePaths }), + } +} diff --git a/lib/build-pages/global-data-state.test.js b/lib/build-pages/global-data-state.test.js new file mode 100644 index 00000000..3f2b525d --- /dev/null +++ b/lib/build-pages/global-data-state.test.js @@ -0,0 +1,375 @@ +/** + * @import { TestContext } from 'node:test' + * @import { BuildPagesFilterOptions } from './index.js' + */ +import assert from 'node:assert/strict' +import { test } from 'node:test' +import { mkdtemp, mkdir, writeFile, readFile, rm } from 'node:fs/promises' +import { tmpdir } from 'node:os' +import { join, resolve } from 'node:path' +import { createGlobalDataState } from './global-data-state.js' +import { buildPages, buildPagesDirect } from './index.js' +import { identifyPages } from '../identify-pages.js' +import { classifyWatchEvent } from '../watch-plan.js' +import { resolveGlobalData } from './resolve-vars.js' + +/** @param {TestContext} t */ +async function fixture (t) { + const dir = await mkdtemp(join(tmpdir(), 'domstack-global-state-')) + t.after(() => rm(dir, { recursive: true, force: true })) + const src = join(dir, 'src') + const dest = join(dir, 'public') + await mkdir(src) + await writeFile(join(src, 'package.json'), '{"type":"module"}') + await writeFile(join(src, 'root.layout.js'), 'export default ({ children }) => children') + await writeFile(join(src, 'page.js'), `export const vars = { title: 'Home', dataDeps: ['summary'] } + export default ({ data }) => JSON.stringify(data.summary)`) + return { src, dest } +} + +test('candidate and previousState are isolated, setState snapshots immediately and last call wins', () => { + const state = { count: 1, map: new Map([['a', new Set([1])]]), date: new Date(0) } + const previousGlobalDataBaseline = { state, sourcePaths: [] } + const transaction = createGlobalDataState({ pages: [], previousGlobalDataBaseline }) + const previous = /** @type {typeof state} */ (transaction.context.previousState) + previous.count = 2 + previous.map.get('a')?.add(2) + previous.date.setTime(100) + assert.deepEqual(transaction.getBaseline().state, state) + assert.notEqual(transaction.getBaseline().state, state) + transaction.context.setState(previous) + previous.count = 3 + assert.equal(Reflect.get(/** @type {object} */ (transaction.getBaseline().state), 'count'), 2) + transaction.context.setState({ final: true }) + assert.deepEqual(transaction.getBaseline().state, { final: true }) + transaction.context.setState(undefined) + assert.equal(transaction.getBaseline().state, undefined) + assert.equal(state.count, 1) + assert.deepEqual(state.map.get('a'), new Set([1])) + assert.equal(state.date.getTime(), 0) +}) + +test('state supports cycles and uncloneable updates fail actionably without replacing the candidate', () => { + const transaction = createGlobalDataState({ pages: [] }) + const state = { self: /** @type {unknown} */ (null), value: 1n } + state.self = state + transaction.context.setState(state) + const candidate = /** @type {typeof state} */ (transaction.getBaseline().state) + assert.equal(candidate.self, candidate) + assert.notEqual(candidate, state) + for (const invalid of [() => {}, { renderer () {} }, new WeakMap(), Symbol('state')]) { + assert.throws(() => transaction.context.setState(invalid), /global\.data setState\(next\).*structured-cloneable.*PageData/) + assert.equal(transaction.getBaseline().state, candidate) + } +}) + +/** @param {SharedArrayBuffer} buffer */ +function sharedMemoryStates (buffer) { + const cycle = { self: /** @type {unknown} */ (null), nested: new Set([new Map([['view', new DataView(buffer)]])]) } + cycle.self = cycle + return [ + buffer, + { nested: [buffer] }, + new Uint8Array(buffer), + { nested: new DataView(buffer) }, + new Map([[buffer, 'key']]), + new Map([['value', buffer]]), + new Set([buffer]), + cycle, + new Error('nested cause', { cause: { buffer } }), + ] +} + +test('setState rejects nested shared memory without replacing or sharing the retained candidate', () => { + const buffer = new SharedArrayBuffer(8) + const bytes = new Uint8Array(buffer) + bytes[0] = 1 + const transaction = createGlobalDataState({ pages: [] }) + transaction.context.setState({ bytes: Uint8Array.from(bytes) }) + const candidate = /** @type {{ bytes: Uint8Array }} */ (transaction.getBaseline().state) + for (const state of sharedMemoryStates(buffer)) { + assert.throws(() => transaction.context.setState(state), /global\.data setState\(next\).*SharedArrayBuffer.*non-shared ArrayBuffer/) + assert.equal(transaction.getBaseline().state, candidate) + } + bytes[0] = 99 + assert.equal(candidate.bytes[0], 1, 'mutating rejected shared memory cannot mutate retained state') +}) + +test('previous baselines containing shared memory are rejected instead of exposing aliased state', () => { + const buffer = new SharedArrayBuffer(8) + for (const state of sharedMemoryStates(buffer)) { + const previousGlobalDataBaseline = { state, sourcePaths: [] } + assert.throws(() => createGlobalDataState({ pages: [], previousGlobalDataBaseline }), /global\.data previous state.*SharedArrayBuffer.*non-shared ArrayBuffer/) + assert.equal(previousGlobalDataBaseline.state, state) + } + const reset = createGlobalDataState({ + pages: [], + previousGlobalDataBaseline: { state: buffer, sourcePaths: [] }, + globalDataInputChanges: { resetReason: 'recovery', upsertedPaths: [], events: [] }, + }) + assert.equal(reset.context.previousState, undefined) + assert.equal(reset.getBaseline().state, undefined) +}) + +test('shared-memory validation inspects only cloned values and does not reread source getters', () => { + const transaction = createGlobalDataState({ pages: [] }) + let reads = 0 + const safe = { get value () { reads++; return new Uint8Array([1]) } } + transaction.context.setState({ first: safe, second: safe }) + assert.equal(reads, 1) + const unsafe = { get value () { reads++; return new SharedArrayBuffer(8) } } + assert.throws(() => transaction.context.setState(unsafe), /SharedArrayBuffer/) + assert.equal(reads, 2) + assert.throws(() => createGlobalDataState({ pages: [], previousGlobalDataBaseline: { state: unsafe, sourcePaths: [] } }), /SharedArrayBuffer/) + assert.equal(reads, 3) + createGlobalDataState({ pages: [], previousGlobalDataBaseline: { state: safe, sourcePaths: [] } }) + assert.equal(reads, 4, 'isolating both previousState and the candidate only reads the source getter once') + + class PrivateState { + #buffer = new SharedArrayBuffer(8) + value = 1 + read () { return this.#buffer } + } + transaction.context.setState(new PrivateState()) + assert.deepEqual(transaction.getBaseline().state, { value: 1 }, 'private fields discarded by structuredClone are not retained state') +}) + +test('non-shared buffers and views remain isolated across previousState and setState', () => { + const bytes = new Uint8Array([1, 2]) + const state = { bytes, view: new DataView(bytes.buffer) } + const transaction = createGlobalDataState({ pages: [], previousGlobalDataBaseline: { state, sourcePaths: [] } }) + const previous = /** @type {typeof state} */ (transaction.context.previousState) + previous.view.setUint8(0, 3) + assert.equal(previous.bytes[0], 3) + assert.equal(bytes[0], 1) + assert.equal(/** @type {typeof state} */ (transaction.getBaseline().state).bytes[0], 1) + transaction.context.setState(previous) + previous.bytes[0] = 4 + assert.equal(/** @type {typeof state} */ (transaction.getBaseline().state).view.getUint8(0), 3) +}) + +test('resets discard state even without setState and do not try to clone invalid old state', () => { + for (const previousGlobalDataBaseline of [undefined, null, { state: () => {}, sourcePaths: ['/old.md'] }]) { + const transaction = createGlobalDataState({ + pages: [], + previousGlobalDataBaseline, + globalDataInputChanges: { resetReason: 'global-data-changed', upsertedPaths: [], events: [] }, + }) + assert.equal(transaction.context.previousState, undefined) + assert.deepEqual(transaction.context.changes, { kind: 'reset', reason: 'global-data-changed', events: [] }) + assert.deepEqual(transaction.getBaseline(), { state: undefined, sourcePaths: [] }) + } + const initial = createGlobalDataState({ pages: [] }) + assert.deepEqual(initial.context.changes, { kind: 'reset', reason: 'initial', events: [] }) + const delta = createGlobalDataState({ pages: [], previousGlobalDataBaseline: initial.getBaseline() }) + assert.deepEqual(delta.context.changes, { kind: 'delta', upserted: [], removed: [], events: [] }) +}) + +for (const [mode, build] of /** @type {const} */ ([['direct', buildPagesDirect], ['worker', buildPages]])) { + test(`${mode}: reset and delta use eligible source membership, not output filters or generated pages`, async t => { + const { src, dest } = await fixture(t) + await writeFile(join(src, 'other.md'), '---\ntitle: Other\n---\nOther') + await writeFile(join(src, 'hidden.draft.md'), 'Draft') + await writeFile(join(src, 'archive.pages.js'), 'export default () => ({ children: \'Generated\' })') + await writeFile(join(src, 'global.data.js'), `export default ({ pages, previousState, changes, setState }) => { + const summary = { + pages: pages.map(page => ({ path: page.pageInfo.pageFile.filepath, title: page.vars.title })), + previousState, + changes: changes.kind === 'reset' ? changes : { + ...changes, upserted: changes.upserted.map(page => page.pageInfo.pageFile.filepath), + }, + } + const next = { count: (previousState?.count ?? 0) + 1 } + setState(next) + next.count = 999 + return { summary } + }`) + const site = await identifyPages(src) + const home = site.pages.find(page => page.pageFile.basename === 'page.js') + assert.ok(home) + home.pageFile.filepath = src + '/./page.js' + const initial = await build(src, dest, site, { trackWatchDependencies: true }) + assert.equal(home.pageFile.filepath, src + '/./page.js', 'normalizing public identities does not mutate discovery metadata') + assert.deepEqual(initial.errors, []) + const first = JSON.parse(await readFile(join(dest, 'index.html'), 'utf8')) + assert.equal(first.changes.kind, 'reset') + assert.equal(first.changes.reason, 'initial') + assert.equal(first.previousState, undefined) + assert.deepEqual(first.pages.map((/** @type {{path: string}} */ p) => p.path).sort(), [join(src, 'other.md'), join(src, 'page.js')]) + assert.equal(first.pages.find((/** @type {{title: string}} */ p) => p.title === 'Home').title, 'Home') + assert.deepEqual(initial.report.globalDataBaseline, { state: { count: 1 }, sourcePaths: site.pages.map(p => resolve(p.pageFile.filepath)) }) + assert.ok(initial.report.pages.some(page => page.pagesFilePath === join(src, 'archive.pages.js'))) + + await rm(join(src, 'other.md')) + await writeFile(join(src, 'new.md'), 'New') + const current = await identifyPages(src, { buildDrafts: true }) + const event = classifyWatchEvent('change', join(src, 'page.js')) + /** @type {BuildPagesFilterOptions} */ + const opts = { + trackWatchDependencies: true, + previousWatchDependencies: initial.report.watchDependencies, + previousGlobalDataBaseline: initial.report.globalDataBaseline, + pageFilterPaths: [], + templateFilterPaths: [], + pagesFileFilterPaths: [], + globalDataInputChanges: { + upsertedPaths: [src + '/./page.js', join(src, 'page.js'), join(src, 'other.md'), join(src, 'archive.pages.js')], + events: [event], + }, + } + const before = structuredClone(opts) + const next = await build(src, dest, current, opts) + assert.deepEqual(next.errors, []) + assert.deepEqual(structuredClone(opts), before, 'worker and direct builds leave caller baselines and input changes untouched') + const second = JSON.parse(await readFile(join(dest, 'index.html'), 'utf8')) + assert.deepEqual(second.previousState, { count: 1 }) + assert.equal(second.changes.kind, 'delta') + assert.deepEqual(second.changes.upserted.sort(), [join(src, 'hidden.draft.md'), join(src, 'new.md'), join(src, 'page.js')]) + assert.deepEqual(second.changes.removed, [join(src, 'other.md')]) + assert.deepEqual(second.changes.events, [event]) + assert.deepEqual(next.report.globalDataBaseline?.state, { count: 2 }) + assert.deepEqual(next.report.pages.map(page => page.sourcePageFilePath), [join(src, 'page.js')], 'changed public data still invalidates its subscriber') + + const reset = await build(src, dest, current, { + ...opts, + previousGlobalDataBaseline: next.report.globalDataBaseline, + globalDataInputChanges: { resetReason: 'global-data-changed', upsertedPaths: [], events: [event] }, + }) + assert.deepEqual(reset.errors, []) + const third = JSON.parse(await readFile(join(dest, 'index.html'), 'utf8')) + assert.equal(third.previousState, undefined) + assert.deepEqual(third.changes, { kind: 'reset', reason: 'global-data-changed', events: [event] }) + assert.deepEqual(reset.report.globalDataBaseline?.state, { count: 1 }) + }) + + test(`${mode}: state does not leak into public data fingerprints or commit previousState mutations`, async t => { + const { src, dest } = await fixture(t) + await writeFile(join(src, 'global.data.js'), `export default ({ previousState, setState, changes }) => { + if (!previousState) setState({ count: 1 }) + else { + previousState.count++ + if (changes.events.length) setState(previousState) + } + return { summary: 'stable' } + }`) + const site = await identifyPages(src) + const initial = await build(src, dest, site, { trackWatchDependencies: true }) + assert.deepEqual(initial.errors, []) + const next = await build(src, dest, site, { + trackWatchDependencies: true, + previousGlobalDataBaseline: initial.report.globalDataBaseline, + previousWatchDependencies: initial.report.watchDependencies, + pageFilterPaths: [], + templateFilterPaths: [], + pagesFileFilterPaths: [], + }) + assert.deepEqual(next.errors, []) + assert.deepEqual(next.report.globalDataBaseline?.state, { count: 1 }) + assert.deepEqual(initial.report.globalDataBaseline?.state, { count: 1 }) + assert.deepEqual(next.report.pages, []) + assert.deepEqual(next.report.watchDependencies?.globalDataFingerprints, initial.report.watchDependencies?.globalDataFingerprints) + assert.deepEqual(Object.keys(next.report.watchDependencies?.globalDataFingerprints ?? {}), ['summary']) + const stateOnly = await build(src, dest, site, { + trackWatchDependencies: true, + previousGlobalDataBaseline: next.report.globalDataBaseline, + previousWatchDependencies: next.report.watchDependencies, + pageFilterPaths: [], + templateFilterPaths: [], + pagesFileFilterPaths: [], + globalDataInputChanges: { upsertedPaths: [], events: [classifyWatchEvent('change', join(src, 'global.data.js'))] }, + }) + assert.deepEqual(stateOnly.errors, []) + assert.deepEqual(stateOnly.report.globalDataBaseline?.state, { count: 2 }) + assert.deepEqual(stateOnly.report.pages, []) + assert.deepEqual(stateOnly.report.watchDependencies?.globalDataFingerprints, next.report.watchDependencies?.globalDataFingerprints) + }) + + test(`${mode}: successful watch builds emit candidates while cold reports keep state private`, async t => { + const { src, dest } = await fixture(t) + await writeFile(join(src, 'global.data.js'), 'export default ({ setState }) => { setState({ candidate: true }); return { summary: \'\' } }') + const site = await identifyPages(src) + const cold = await build(src, dest, site, {}) + assert.deepEqual(cold.errors, []) + assert.equal(Object.hasOwn(cold.report, 'globalDataBaseline'), false) + const watch = await build(src, dest, site, { trackWatchDependencies: true }) + assert.deepEqual(watch.errors, []) + assert.deepEqual(watch.report.globalDataBaseline, { state: { candidate: true }, sourcePaths: [join(src, 'page.js')] }) + }) + + test(`${mode}: failed page, template, factory, and initialization phases never emit candidate baselines`, async t => { + for (const [filename, content] of [ + ['page.js', 'export default () => { throw new Error(\'render failed\') }'], + ['feed.template.js', 'export default () => { throw new Error(\'template failed\') }'], + ['archive.pages.js', 'export default () => { throw new Error(\'factory failed\') }'], + ['page.vars.js', 'export default () => { throw new Error(\'vars failed\') }'], + ]) { + const { src, dest } = await fixture(t) + assert.ok(filename && content) + await writeFile(join(src, filename), content) + await writeFile(join(src, 'global.data.js'), 'export default ({ setState }) => { setState({ candidate: true }); return { summary: \'\' } }') + const previousGlobalDataBaseline = { state: { previous: true }, sourcePaths: [] } + const result = await build(src, dest, await identifyPages(src), { previousGlobalDataBaseline, trackWatchDependencies: true }) + assert.ok(result.errors.length > 0, filename) + assert.equal(Object.hasOwn(result.report, 'globalDataBaseline'), false, filename) + assert.deepEqual(previousGlobalDataBaseline, { state: { previous: true }, sourcePaths: [] }) + } + }) + + test(`${mode}: shared-memory updates and previous baselines fail without emitting candidates`, async t => { + for (const previous of [false, true]) { + const { src, dest } = await fixture(t) + await writeFile(join(src, 'global.data.js'), previous + ? 'export default () => { throw new Error(\'shared baseline must be rejected before callback\') }' + : 'export default ({ setState }) => { setState({ nested: new Map([[\'bytes\', new DataView(new SharedArrayBuffer(8))]]) }); return { summary: \'\' } }') + const site = await identifyPages(src) + /** @type {BuildPagesFilterOptions} */ + const opts = { + trackWatchDependencies: true, + previousGlobalDataBaseline: previous ? { state: new Set([new Uint8Array(new SharedArrayBuffer(8))]), sourcePaths: [] } : undefined, + } + const message = previous ? /global\.data previous state.*SharedArrayBuffer/ : /global\.data setState\(next\).*SharedArrayBuffer/ + if (mode === 'direct') { + await assert.rejects(build(src, dest, site, opts), message) + } else { + const result = await build(src, dest, site, opts) + assert.equal(result.errors.length, 1) + const failure = result.errors[0] + assert.ok(failure instanceof Error) + assert.match(failure.message, message) + assert.equal(Object.hasOwn(result.report, 'globalDataBaseline'), false) + } + } + }) + + test(`${mode}: uncloneable setState and global-data failures are actionable without candidates`, async t => { + for (const body of [ + 'setState({ renderer () {} })', + 'setState({ candidate: true }); throw new Error(\'global data failed\')', + ]) { + const { src, dest } = await fixture(t) + await writeFile(join(src, 'global.data.js'), `export default ({ setState }) => { ${body}; return { summary: '' } }`) + const site = await identifyPages(src) + if (mode === 'direct') { + await assert.rejects(build(src, dest, site, { trackWatchDependencies: true }), /global\.data setState\(next\).*structured-cloneable|global data failed/) + } else { + const result = await build(src, dest, site, { trackWatchDependencies: true }) + assert.equal(result.errors.length, 1) + const failure = result.errors[0] + assert.ok(failure instanceof Error) + assert.match(failure.message, /global\.data setState\(next\).*structured-cloneable|global data failed/) + assert.equal(Object.hasOwn(result.report, 'globalDataBaseline'), false) + } + } + }) +} + +test('resolveGlobalData retains object and non-serializable public return values unchanged', async t => { + const { src } = await fixture(t) + const globalDataPath = join(src, 'global.data.js') + await writeFile(globalDataPath, 'export default { render: () => \'public function\', value: 1 }') + const imported = await import(globalDataPath) + const result = await resolveGlobalData({ globalDataPath, pages: [] }) + assert.equal(result, imported.default) + assert.equal(Reflect.get(result, 'render')(), 'public function') + assert.deepEqual(await resolveGlobalData({ pages: [] }), {}) +}) diff --git a/lib/build-pages/index.js b/lib/build-pages/index.js index d8ba54fb..31fcd415 100644 --- a/lib/build-pages/index.js +++ b/lib/build-pages/index.js @@ -7,6 +7,7 @@ * @import { DomstackManifestRecord } from '../domstack-manifest/index.js' * @import { WatchDependencyState, WatchConsumer, WatchDependencyTracker } from './watch-dependencies.js' * @import { PageOutputCache } from './page-builders/page-output-writer.js' + * @import { GlobalDataBaseline, GlobalDataInputChanges, GlobalDataChanges } from './global-data-state.js' */ import { Worker } from 'worker_threads' @@ -25,6 +26,7 @@ import { isAsyncIterable, isPlainObject } from '../helpers/type-guards.js' import { createSubscribedData, resolveDataDeps } from './data-deps.js' import { WatchDependencyTracker as WatchDependencyTrackerClass } from './watch-dependencies.js' import { outputWarnings } from '../helpers/output-warnings.js' +import { createGlobalDataState } from './global-data-state.js' const MAX_CONCURRENCY = Math.min(cpus().length, 24) @@ -47,14 +49,19 @@ const __dirname = import.meta.dirname * @property {WatchDependencyState | undefined} [watchDependencies] * @property {PageOutputCache | undefined} [pageOutputCache] * @property {string[] | undefined} [rebuiltPagesFilePaths] + * @property {GlobalDataBaseline | undefined} [globalDataBaseline] - Candidate state and membership, emitted only after a successful page phase. */ /** * Parameters passed to a global.data.js default export function. * @template {Record} [T=any] - Source-page vars. * @template [U=any] - Source-page render values. + * @template [S=unknown] - Explicitly retained, structured-cloneable state. * @typedef {object} GlobalDataFunctionParams - * @property {PageData[]} pages - Fully initialized source-backed pages, before generated pages are created. + * @property {PageData[]} pages - Fully initialized source-backed pages, before generated pages are created. Identities are normalized absolute pageInfo.pageFile.filepath values. + * @property {S | undefined} previousState - Isolated state from the last successful build, or undefined on reset. + * @property {GlobalDataChanges} changes - Reset information or source-page input deltas, independent of output filters. + * @property {(next: S) => void} setState - Immediately snapshot state for the next successful build. Mutating previousState alone does not commit changes. */ /** @@ -64,8 +71,9 @@ const __dirname = import.meta.dirname * @template {Record} [T=Record] - The shape of the derived vars object returned. * @template {Record} [V=any] - Source-page vars. * @template [U=any] - Source-page render values. + * @template [S=unknown] - Explicitly retained, structured-cloneable state. * @callback GlobalDataFunction - * @param {GlobalDataFunctionParams} params + * @param {GlobalDataFunctionParams} params * @returns {T | Promise} */ @@ -76,8 +84,9 @@ const __dirname = import.meta.dirname * @template {Record} [T=Record] - The shape of the derived vars object returned. * @template {Record} [V=any] - Source-page vars. * @template [U=any] - Source-page render values. + * @template [S=unknown] - Explicitly retained, structured-cloneable state. * @callback AsyncGlobalDataFunction - * @param {GlobalDataFunctionParams} params + * @param {GlobalDataFunctionParams} params * @returns {Promise} */ @@ -93,6 +102,8 @@ const __dirname = import.meta.dirname * @property {WatchDependencyState | null | undefined} [previousWatchDependencies] - Dependency state from the previous successful watch build. * @property {boolean | undefined} [trackWatchDependencies] - Collect subscriptions for incremental watch builds. * @property {PageOutputCache | undefined} [previousPageOutputCache] - Successful output hashes and metadata retained across watch workers. + * @property {GlobalDataBaseline | null | undefined} [previousGlobalDataBaseline] + * @property {GlobalDataInputChanges | undefined} [globalDataInputChanges] */ /** @@ -465,6 +476,8 @@ export function buildPages (src, dest, siteData, opts) { previousWatchDependencies: opts?.previousWatchDependencies, trackWatchDependencies: opts?.trackWatchDependencies, previousPageOutputCache: opts?.previousPageOutputCache, + previousGlobalDataBaseline: opts?.previousGlobalDataBaseline, + globalDataInputChanges: opts?.globalDataInputChanges, } return new Promise((resolve, reject) => { @@ -609,15 +622,24 @@ export async function buildPagesDirect (_src, dest, siteData, opts) { } // Mix in resolveVars, renderInnerPage and renderFullPage methods for concrete pages. - const concretePages = await pMap(siteData.pages, initPageData, { concurrency: MAX_CONCURRENCY }) + const concretePages = await pMap(siteData.pages, pageInfo => initPageData({ + ...pageInfo, + pageFile: { ...pageInfo.pageFile, filepath: resolve(pageInfo.pageFile.filepath) }, + }), { concurrency: MAX_CONCURRENCY }) if (result.errors.length > 0) return result // Derive collection data from source-backed pages before generated-page factories run. // This keeps generated pages downstream while making shared data available to them. + const globalDataState = createGlobalDataState({ + pages: concretePages, + previousGlobalDataBaseline: opts?.previousGlobalDataBaseline, + globalDataInputChanges: opts?.globalDataInputChanges, + }) const globalData = /** @type {Record} */ (await resolveGlobalData({ globalDataPath: siteData.globalData?.filepath, pages: concretePages, + context: globalDataState.context, })) const changedGlobalDataKeys = watchDependencyTracker.updateGlobalDataFingerprints( globalData, @@ -766,6 +788,7 @@ export async function buildPagesDirect (_src, dest, siteData, opts) { ) result.report.watchDependencies = watchDependencyTracker.state } + if (opts?.trackWatchDependencies && result.errors.length === 0) result.report.globalDataBaseline = globalDataState.getBaseline() return result } diff --git a/lib/build-pages/resolve-vars.js b/lib/build-pages/resolve-vars.js index 904e095c..a5e05bef 100644 --- a/lib/build-pages/resolve-vars.js +++ b/lib/build-pages/resolve-vars.js @@ -1,8 +1,10 @@ /** * @import { PageData } from './page-data.js' + * @import { GlobalDataFunctionParams } from './index.js' */ import { isFunction, isObject, isPlainObject } from '../helpers/type-guards.js' +import { createGlobalDataState } from './global-data-state.js' /** * Resolve an object-or-function vars export. @@ -54,16 +56,17 @@ export async function resolveVars ({ * @param {object} params * @param {string | undefined} [params.globalDataPath] - Path to the global.data file. * @param {PageData[]} params.pages - Initialized source-backed PageData array. + * @param {GlobalDataFunctionParams} [params.context] - State context prepared by the page phase. * @returns {Promise} */ -export async function resolveGlobalData ({ globalDataPath, pages }) { +export async function resolveGlobalData ({ globalDataPath, pages, context }) { if (!globalDataPath) return {} const imported = await import(globalDataPath) const maybeGlobalData = imported.default if (isFunction(maybeGlobalData)) { - const result = await maybeGlobalData({ pages }) + const result = await maybeGlobalData(context ?? createGlobalDataState({ pages }).context) if (isObject(result)) return result throw new Error('global.data default export function must return an object') } else if (isObject(maybeGlobalData)) { diff --git a/lib/helpers/find-dependencies.js b/lib/helpers/find-dependencies.js new file mode 100644 index 00000000..7397dd60 --- /dev/null +++ b/lib/helpers/find-dependencies.js @@ -0,0 +1,71 @@ +import { stat } from 'node:fs/promises' +import { createRequire } from 'node:module' +import { extname, resolve } from 'node:path' + +// Resolve the analyzer's own runtime dependencies, even when they are not hoisted. +// Its TS wrapper does not expose the ESM finder's parserOverride option. +const require = createRequire(import.meta.resolve('@11ty/dependency-tree-typescript')) +/** @type {{ find: (filepath: string, options: { parserOverride: typeof parserOverride }) => Promise }} */ +const { find } = require(require.resolve('@11ty/dependency-tree-esm')) +const { Parser } = require('acorn') +const { tsPlugin } = require('@sveltejs/acorn-typescript') +const parser = Parser.extend(tsPlugin()) + +const parserOverride = { + /** @param {string} contents */ + parse (contents) { + const ast = parser.parse(contents, { sourceType: 'module', ecmaVersion: 'latest' }) + return { + ...ast, + body: ast.body.filter(node => { + // Node erases declaration-level types only. Inline type specifiers leave + // an import/export (possibly empty) that still executes the source module. + if (node.type === 'ImportDeclaration') { + return !('importKind' in node && node.importKind === 'type') + } + if (node.type === 'ExportNamedDeclaration' || node.type === 'ExportAllDeclaration') { + return !('exportKind' in node && node.exportKind === 'type') + } + return true + }).map(node => { + if ((node.type === 'ExportNamedDeclaration' || node.type === 'ExportAllDeclaration') && node.source) { + // Only the dependency walk sees this projection. Keep source and import + // attributes intact so re-exported JSON/CSS remain attributed leaves. + return { ...node, type: 'ImportDeclaration' } + } + return node + }), + } + }, +} + +/** @param {string} filepath */ +async function assertFile (filepath) { + if (!(await stat(filepath)).isFile()) { + throw new Error(`Cannot analyze dependency "${filepath}": expected a file`) + } +} + +/** + * Find transitive static ESM imports and re-exports in JS/TS, using the existing + * analyzer's cwd-relative path strings and bare-specifier exclusion semantics. + * Declaration-level type-only imports/exports are excluded; inline type-only + * specifier lists retain runtime edges. JSON roots have no dependencies; + * attributed imports/re-exports remain leaves. + * Missing files and parse errors reject so callers can invalidate conservatively + * instead of installing an incomplete dependency index. No results are cached. + * + * @param {string} filepath + * @returns {Promise} + */ +export async function findDependencies (filepath) { + const root = resolve(filepath) + await assertFile(root) + if (extname(root) === '.json') return [] + + const dependencies = await find(root, { parserOverride }) + // The upstream finder silently accepts missing imports (including attributed + // leaves). Validate every edge before returning anything to the caller. + await Promise.all(dependencies.map(assertFile)) + return [...new Set(dependencies)].filter(dependency => resolve(dependency) !== root) +} diff --git a/lib/helpers/find-dependencies.test.js b/lib/helpers/find-dependencies.test.js new file mode 100644 index 00000000..83963394 --- /dev/null +++ b/lib/helpers/find-dependencies.test.js @@ -0,0 +1,343 @@ +/** + * @import { TestContext } from 'node:test' + * @import { WatchSnapshot } from '../watch-plan.js' + */ +import assert from 'node:assert/strict' +import { mkdir, mkdtemp, readFile, rm, writeFile } from 'node:fs/promises' +import { stripTypeScriptTypes } from 'node:module' +import { tmpdir } from 'node:os' +import { basename, dirname, join, relative, resolve } from 'node:path' +import { test } from 'node:test' +import { pathToFileURL } from 'node:url' +import { find } from '@11ty/dependency-tree-typescript' +import { classifyWatchEvent, planWatchBatch } from '../watch-plan.js' +import { findDependencies } from './find-dependencies.js' + +/** @param {TestContext} t @param {Record} files */ +async function fixture (t, files) { + const directory = await mkdtemp(join(tmpdir(), 'domstack-dependencies-')) + t.after(() => rm(directory, { recursive: true, force: true })) + for (const [name, source] of Object.entries(files)) { + const filepath = join(directory, name) + await mkdir(dirname(filepath), { recursive: true }) + await writeFile(filepath, source) + } + return directory +} + +/** @param {string[]} paths */ +function absolutePaths (paths) { + assert.ok(paths.every(path => typeof path === 'string')) + return paths.map(path => resolve(path)).sort() +} + +test('follows mixed JS/TS imports and named, star, and namespace re-exports transitively', async t => { + const directory = await fixture(t, { + 'root.js': 'import { value } from "./barrel.ts"; export default value;', + 'barrel.ts': 'export { value } from "./nested/named.mjs"; export * as namespace from "./namespace.mts";', + 'nested/named.mjs': 'export * from "../leaf.cts";', + 'namespace.mts': 'export { value } from "./leaf.cts";', + 'leaf.cts': 'import "./side-effect.js"; export const value: number = 1;', + 'side-effect.js': 'export const loaded = true;', + }) + const root = join(directory, 'root.js') + const expected = ['barrel.ts', 'nested/named.mjs', 'namespace.mts', 'leaf.cts', 'side-effect.js'].map(name => join(directory, name)).sort() + assert.deepEqual(absolutePaths(await findDependencies(root)), expected) + assert.deepEqual(absolutePaths(await findDependencies(relative(process.cwd(), root))), expected) + assert.equal((await findDependencies(root)).length, expected.length, 'shared leaves are deduplicated') +}) + +test('handles cycles and self-re-exports without returning the root', async t => { + const directory = await fixture(t, { + 'root.ts': 'export * from "./barrel.ts"; export * from "./root.ts";', + 'barrel.ts': 'export * from "./root.ts"; import "./leaf.js";', + 'leaf.js': 'import "./barrel.ts";', + }) + assert.deepEqual(absolutePaths(await findDependencies(join(directory, 'root.ts'))), [join(directory, 'barrel.ts'), join(directory, 'leaf.js')]) +}) + +test('uses parsed syntax, not import-like comments, strings, templates, or regular expressions', async t => { + const directory = await fixture(t, { + 'root.ts': [ + '// export * from "./missing-comment.js";', + '/* import "./missing-block.js"; */', + 'const text: string = \'export { x } from "./missing-string.js"\';', + 'const template = `export * from "./missing-template.js"`;', + 'const regex = /export .* from "missing-regex"/;', + 'export {', + ' value as renamed // a multiline re-export', + '} from "./le\\u0061f.js";', + 'export { text, template, regex };', + ].join('\n'), + 'leaf.js': 'export const value = 1;', + }) + assert.deepEqual(absolutePaths(await findDependencies(join(directory, 'root.ts'))), [join(directory, 'leaf.js')]) +}) + +test('ignores missing declaration-level type-only imports and re-exports erased by Node', async t => { + const directory = await fixture(t, { + 'root.ts': 'export * from "./barrel.ts";', + 'barrel.ts': '', + }) + for (const source of [ + 'import type { A } from "./missing.ts";', + 'import type Default from "./missing.ts";', + 'import type * as Types from "./missing.ts";', + + 'export type { A } from "./missing.ts";', + 'export type * from "./missing.ts";', + 'export type * as Types from "./missing.ts";', + + 'import type {} from "./missing.ts";', + 'export type {} from "./missing.ts";', + ]) { + assert.equal(stripTypeScriptTypes(source).trim(), '', source) + await writeFile(join(directory, 'barrel.ts'), source) + assert.deepEqual(await findDependencies(join(directory, 'barrel.ts')), [], source) + assert.deepEqual(absolutePaths(await findDependencies(join(directory, 'root.ts'))), [join(directory, 'barrel.ts')], source) + } +}) + +test('all-type specifier lists retain side effects and transitive runtime dependencies', async t => { + for (const source of [ + 'import { type A } from "./registration.ts";', + 'import { type A, type B as C } from "./registration.ts";', + 'export { type A } from "./registration.ts";', + 'export { type A, type B as C } from "./registration.ts";', + ]) { + await t.test(source, async t => { + const directory = await fixture(t, { + 'root.ts': source, + 'stripped.mjs': stripTypeScriptTypes(source), + 'registration.ts': 'import "./side-effect.ts"; export interface A { name: string } export type B = number;', + 'side-effect.ts': 'import { writeFileSync } from "node:fs"; writeFileSync(new URL("./executed.txt", import.meta.url), "registered");', + }) + const root = join(directory, 'root.ts') + const marker = join(directory, 'executed.txt') + const expected = ['registration.ts', 'side-effect.ts'].map(name => join(directory, name)) + assert.deepEqual(absolutePaths(await findDependencies(root)), expected) + assert.deepEqual(absolutePaths(await findDependencies(join(directory, 'stripped.mjs'))), expected, 'matches Node type stripping') + await assert.rejects(readFile(marker), { code: 'ENOENT' }, 'analysis must not execute the modules') + await import(pathToFileURL(root).href) + assert.equal(await readFile(marker, 'utf8'), 'registered', 'Node executes the transitive side effect') + await rm(join(directory, 'side-effect.ts')) + await assert.rejects(findDependencies(root), { code: 'ENOENT' }, 'missing transitive runtime edges still fail conservatively') + }) + } +}) + +test('does not traverse declaration-level type-only imports/re-exports into runtime dependencies', async t => { + const directory = await fixture(t, { + 'root.ts': 'import type { A } from "./types.ts"; export type { A } from "./types.ts";', + 'types.ts': 'import "./missing-library.js"; export interface A { name: string }', + }) + assert.deepEqual(await findDependencies(join(directory, 'root.ts')), []) +}) + +test('mixed value/type imports and re-exports retain the source and its transitive runtime edges', async t => { + const directory = await fixture(t, { + 'root.ts': '', + 'values.ts': 'import "./leaf.js"; export interface Shape { name: string } export const value = 1; export default value;', + 'leaf.js': 'export const loaded = true;', + }) + for (const source of [ + 'import { type Shape, value } from "./values.ts";', + 'import { value, type Shape } from "./values.ts";', + 'import value, { type Shape } from "./values.ts";', + 'export { type Shape, value } from "./values.ts";', + 'export { value, type Shape } from "./values.ts";', + ]) { + await writeFile(join(directory, 'root.ts'), source) + assert.deepEqual(absolutePaths(await findDependencies(join(directory, 'root.ts'))), ['leaf.js', 'values.ts'].map(name => join(directory, name)), source) + } + await rm(join(directory, 'values.ts')) + await assert.rejects(findDependencies(join(directory, 'root.ts')), { code: 'ENOENT' }) +}) + +test('preserves side-effect imports and empty named imports/re-exports as runtime edges', async t => { + const directory = await fixture(t, { + 'root.ts': '', + 'runtime.ts': 'import "./leaf.js"; export const value = 1;', + 'leaf.js': 'export const loaded = true;', + }) + const sources = [ + 'import "./runtime.ts";', + 'import {} from "./runtime.ts";', + 'export {} from "./runtime.ts";', + ] + for (const source of sources) { + await writeFile(join(directory, 'root.ts'), source) + assert.deepEqual(absolutePaths(await findDependencies(join(directory, 'root.ts'))), ['leaf.js', 'runtime.ts'].map(name => join(directory, name)), source) + } + await rm(join(directory, 'runtime.ts')) + for (const source of sources) { + await writeFile(join(directory, 'root.ts'), source) + await assert.rejects(findDependencies(join(directory, 'root.ts')), { code: 'ENOENT' }, source) + } +}) + +test('does not confuse value bindings named type with type-only specifiers', async t => { + const directory = await fixture(t, { + 'root.ts': 'import { type as value } from "./values.ts"; export { type } from "./values.ts";', + 'values.ts': 'export const type = 1;', + }) + assert.deepEqual(absolutePaths(await findDependencies(join(directory, 'root.ts'))), [join(directory, 'values.ts')]) +}) + +test('preserves existing import path strings and bare-import exclusion semantics', async t => { + const directory = await fixture(t, { + 'root.js': 'import "node:fs"; import "uninstalled-package"; import "#alias"; import "./leaf.js";', + 'barrel.js': 'export * from "uninstalled-package"; export { readFile } from "node:fs"; export * from "./leaf.js";', + 'leaf.js': 'export const value = 1;', + }) + const expected = await find(join(directory, 'root.js')) + assert.deepEqual(await findDependencies(join(directory, 'root.js')), expected) + assert.deepEqual(await findDependencies(join(directory, 'barrel.js')), expected) + assert.deepEqual(await find(join(directory, 'barrel.js')), [], 'the original analyzer is not patched globally') +}) + +test('returns JSON and other attributed leaves as path strings, without parsing them as modules', async t => { + const directory = await fixture(t, { + 'root.js': 'import data from "./data.json" with { type: "json" }; export * from "./barrel.ts";', + 'barrel.ts': 'export { default as more } from "./more.json" with { type: "json" }; export { default as css } from "./style.css" with { type: "css" };', + 'data.json': '{ "value": 1, "text": "export * from missing" }', + 'more.json': '{ "value": 2 }', + 'style.css': 'body { color: red; }', + }) + const dependencies = await findDependencies(join(directory, 'root.js')) + assert.deepEqual(absolutePaths(dependencies), ['barrel.ts', 'data.json', 'more.json', 'style.css'].map(name => join(directory, name))) + assert.deepEqual(await findDependencies(join(directory, 'data.json')), []) +}) + +test('rejects missing roots and unresolved direct, transitive, and attributed dependencies', async t => { + const directory = await fixture(t, { + 'root.js': 'import "./barrel.ts";', + 'barrel.ts': '', + }) + await assert.rejects(findDependencies(join(directory, 'missing.js')), { code: 'ENOENT' }) + await assert.rejects(findDependencies(join(directory, 'missing.json')), { code: 'ENOENT' }) + for (const source of [ + 'import "./missing.js";', + 'import { type A } from "./missing.ts";', + 'import { type A, type B as C } from "./missing.ts";', + 'export { type A } from "./missing.ts";', + 'export { type A, type B as C } from "./missing.ts";', + 'export { value } from "./missing.js";', + 'export * from "./missing.js";', + 'export * as values from "./missing.js";', + 'import data from "./missing.json" with { type: "json" };', + 'export { default as data } from "./missing.json" with { type: "json" };', + ]) { + await writeFile(join(directory, 'barrel.ts'), source) + await assert.rejects(findDependencies(join(directory, 'barrel.ts')), { code: 'ENOENT' }, source) + await assert.rejects(findDependencies(join(directory, 'root.js')), { code: 'ENOENT' }, source) + } +}) + +test('propagates parser errors and rejects directories rather than returning incomplete results', async t => { + const directory = await fixture(t, { + 'root.js': 'export * from "./broken.ts";', + 'broken.ts': 'export const value: = ;', + 'directory.js': 'export { default as data } from "./folder" with { type: "json" };', + 'folder/leaf.js': '', + }) + await assert.rejects(findDependencies(join(directory, 'root.js')), SyntaxError) + await assert.rejects(findDependencies(directory), /expected a file/) + await assert.rejects(findDependencies(join(directory, 'directory.js')), /expected a file/) +}) + +test('reanalyzes edited barrel edges and notices removed leaves on later calls', async t => { + const directory = await fixture(t, { + 'root.js': 'export * from "./barrel.js";', + 'barrel.js': 'export * from "./first.js";', + 'first.js': 'export const value = 1;', + 'second.js': 'export const value = 2;', + }) + const root = join(directory, 'root.js') + assert.deepEqual(absolutePaths(await findDependencies(root)), ['barrel.js', 'first.js'].map(name => join(directory, name))) + await writeFile(join(directory, 'barrel.js'), 'export * from "./second.js";') + assert.deepEqual(absolutePaths(await findDependencies(root)), ['barrel.js', 'second.js'].map(name => join(directory, name))) + await rm(join(directory, 'second.js')) + await assert.rejects(findDependencies(root), { code: 'ENOENT' }) +}) + +/** @param {string} directory */ +function watchFixture (directory) { + const filepath = join(directory, 'feed.template.js') + const template = { + templateFile: { root: directory, filepath, relname: basename(filepath), basename: basename(filepath), parentName: '.' }, + path: '', + outputName: 'feed.xml', + } + const leaf = join(directory, 'leaf.js') + /** @type {WatchSnapshot} */ + const state = { + siteData: { pages: [], templates: [template], pagesFiles: [], layouts: {} }, + layoutDepMap: new Map(), + layoutPageMap: new Map(), + pageFileMap: new Map(), + layoutFileMap: new Map(), + pageDepMap: new Map(), + templateDepMap: new Map([[leaf, new Set([template])]]), + pagesFileDepMap: new Map(), + pagesFileLayoutMap: new Map(), + globalDataDepPaths: new Set(), + globalVarsDepPaths: new Set(), + markdownDepPaths: new Set(), + dependencyAnalysisFailed: false, + pageBuildFailed: false, + esbuildEntryPoints: new Set([leaf]), + } + return { state, leaf, template } +} + +test('a barrel leaf shared with a template and browser role still invalidates each producer index', async t => { + const directory = await fixture(t, { + 'producer.js': 'import { value } from "./barrel.ts"; export default value;', + 'barrel.ts': 'export * from "./named.js";', + 'named.js': 'export { value } from "./leaf.js";', + 'leaf.js': 'export const value = 1;', + }) + const root = join(directory, 'producer.js') + const original = new Set(absolutePaths(await find(root))) + const complete = new Set(absolutePaths(await findDependencies(root))) + const cases = /** @type {const} */ ([ + ['globalDataDepPaths', 'global-data-changed'], + ['globalVarsDepPaths', 'global-vars-changed'], + ['markdownDepPaths', 'markdown-settings-changed'], + ]) + for (const [role, reason] of cases) { + const { state, leaf, template } = watchFixture(directory) + const event = classifyWatchEvent('change', leaf) + state[role] = original + const stale = planWatchBatch(state, [event]) + assert.equal(stale.inputChanges.resetReason, undefined, 'another known role masks the missing producer edge') + state[role] = complete + const batch = planWatchBatch(state, [event]) + assert.equal(batch.inputChanges.resetReason, reason) + if (role === 'globalVarsDepPaths') { + assert.equal(batch.plan.kind, 'full') + } else { + assert.equal(batch.plan.kind, 'pages') + if (batch.plan.kind !== 'pages') throw new Error('Expected a page plan') + assert.deepEqual(batch.plan.templateFilterPaths, [template.templateFile.filepath]) + } + } +}) + +test('analysis errors let the caller select conservative recovery even for a known shared role', async t => { + const directory = await fixture(t, { 'producer.js': 'export * from "./missing.js";' }) + const { state, leaf } = watchFixture(directory) + try { + state.globalDataDepPaths = new Set(absolutePaths(await findDependencies(join(directory, 'producer.js')))) + } catch { + state.dependencyAnalysisFailed = true + } + const batch = planWatchBatch(state, [classifyWatchEvent('change', leaf)]) + assert.equal(batch.inputChanges.resetReason, 'dependency-analysis-failed') + assert.equal(batch.plan.kind, 'pages') + if (batch.plan.kind !== 'pages') throw new Error('Expected a page plan') + assert.equal(batch.plan.pageFilterPaths, null) + assert.equal(batch.plan.templateFilterPaths, null) + assert.equal(batch.plan.pagesFileFilterPaths, null) +}) diff --git a/lib/watch-plan.js b/lib/watch-plan.js index 42207622..a48aaeb9 100644 --- a/lib/watch-plan.js +++ b/lib/watch-plan.js @@ -12,8 +12,12 @@ * @property {ReadonlyMap>} pagesFileDepMap * @property {ReadonlyMap>} pagesFileLayoutMap * @property {ReadonlySet} globalDataDepPaths + * @property {ReadonlySet} [globalVarsDepPaths] + * @property {ReadonlySet} [markdownDepPaths] + * @property {boolean} [dependencyAnalysisFailed] * @property {boolean} pageBuildFailed * @property {ReadonlySet} esbuildEntryPoints + * @property {ReadonlySet} [esbuildDepPaths] Known imported browser inputs; server roles still take precedence. * @typedef {ReturnType} WatchEvent * @typedef {object} PagePlan * @property {'pages'} kind @@ -24,15 +28,21 @@ * @property {PageInfo[]} [pages] * @property {TemplateInfo[]} [templates] * @typedef {PagePlan | {kind: 'skip', message: string} | {kind: 'full', message: string} | {kind: 'restart', message: string}} WatchPlan + * @typedef {object} WatchInputChanges + * @property {string} [resetReason] Recompute all producer inputs instead of applying the delta. + * @property {string[]} upsertedPaths Source-page candidates; reconcile against rediscovered membership before reading. + * @property {WatchEvent[]} events Original events in order, including duplicates and removals. + * @typedef {{ plan: WatchPlan, inputChanges: WatchInputChanges }} WatchBatchPlan */ -import { basename, dirname, relative } from 'node:path' -import { classifyFile } from './file-conventions.js' +import { basename, dirname, relative, resolve } from 'node:path' +import { classifyFile, fileConventions } from './file-conventions.js' /** * @param {'change' | 'added' | 'removed'} type - * @param {string} filepath Absolute source path. + * @param {string} filepath Source path, resolved against the current working directory. */ export function classifyWatchEvent (type, filepath) { + filepath = resolve(filepath) const name = basename(filepath) return { type, filepath, name, convention: classifyFile(name) } } @@ -51,15 +61,11 @@ export function planWatchEvent (state, event) { ? { kind: 'restart', message: `"${name}" ${type}, restarting esbuild...` } : { kind: 'full', message: `"${name}" ${type}, triggering full rebuild...` } } - if (convention?.change === 'full') return { kind: 'full', message: `"${name}" changed, triggering full rebuild...` } - if (convention?.change === 'manifest') { + if (convention?.change === 'full' || state.globalVarsDepPaths?.has(filepath)) return { kind: 'full', message: `"${name}" changed, triggering full rebuild...` } + if (convention?.change === 'manifest' && !hasWatchRole(state, filepath)) { return { kind: 'skip', message: `"${name}" changed but domstack manifests are disabled in watch mode, skipping.` } } if (state.pageBuildFailed) return retryAllPages(event) - if (convention?.change === 'markdown') { - const pages = siteData.pages.filter(page => page.type === 'md') - return selectedPages(pages) - } // A module can have several roles, including both browser and server uses. // Union every direct and imported consumer before deciding to skip it. @@ -68,6 +74,12 @@ export function planWatchEvent (state, event) { if (directLayout) layouts.add(directLayout) const affected = layoutConsumers(state, layouts) const pages = new Set(state.pageDepMap.get(filepath)) + const markdownChanged = convention?.change === 'markdown' || state.markdownDepPaths?.has(filepath) + if (markdownChanged) { + for (const page of siteData.pages) { + if (page.type === 'md') pages.add(page) + } + } const directPage = state.pageFileMap.get(filepath) if (directPage) pages.add(directPage) for (const page of affected.pages) pages.add(page) @@ -86,8 +98,8 @@ export function planWatchEvent (state, event) { // Even without direct consumers, a producer input must recompute global data. // The page worker selects subscribers after comparing the resulting values. - const globalDataChanged = state.globalDataDepPaths.has(filepath) - if (globalDataChanged || pages.size || templates.size || owners.size) { + const globalDataChanged = convention?.change === 'pages' || state.globalDataDepPaths.has(filepath) + if (globalDataChanged || markdownChanged || pages.size || templates.size || owners.size) { return { ...selectedPages([...pages]), templateFilterPaths: [...templates].map(template => template.templateFile.filepath), @@ -96,12 +108,142 @@ export function planWatchEvent (state, event) { ...(globalDataChanged ? { message: `"${name}" changed, rebuilding data subscribers...` } : {}), } } - if (state.esbuildEntryPoints.has(filepath)) { + if (state.esbuildEntryPoints.has(filepath) || state.esbuildDepPaths?.has(filepath)) { return { kind: 'skip', message: `"${name}" changed, esbuild will handle rebundling.` } } return { kind: 'skip', message: `"${name}" changed but did not match any rebuild rule, skipping.` } } +/** + * Union output selections independently of producer-input invalidation. Null + * filters mean all; full dominates pages/skip. Batch restarts become full because + * their HTML scope cannot be resolved until discovery and esbuild have run. + * A page plan paired with skip is preserved exactly, including logging metadata. + * Page-plan unions also union pages/templates by filepath and distinct message + * lines in event order so logging retains every selected consumer and reason. + * + * A full output plan requires rediscovery, but does not itself reset inputs. + * Structural source events upsert the change-plan's source consumers plus the + * event path (including removed paths). Other structural events upsert all + * current source pages. The worker must reconcile membership after rediscovery; + * these candidates are not instructions to read deleted files. Templates and + * generated-page owners are output selections, not source-input rows. + * + * A reset supersedes the delta. Reasons are page-build-failed, then + * dependency-analysis-failed, then the first applicable event reason, using + * this per-event priority: + * unreliable-event, global-data-changed, global-vars-changed, + * markdown-settings-changed, global-config-changed, or unknown-event. + * Unknown changes reset; unknown adds/removes can use conservative deltas. + * Empty batches skip without requesting recovery. Inputs are never mutated. + * + * @param {WatchSnapshot} state + * @param {ReadonlyArray} events Events from classifyWatchEvent. + * @returns {WatchBatchPlan} + */ +export function planWatchBatch (state, events) { + /** @type {WatchPlan} */ + let plan = { kind: 'skip', message: 'No watch events to rebuild.' } + /** @type {WatchInputChanges} */ + const inputChanges = { upsertedPaths: [], events: [...events] } + if (!events.length) return { plan, inputChanges } + + let resetReason = state.pageBuildFailed + ? 'page-build-failed' + : state.dependencyAnalysisFailed ? 'dependency-analysis-failed' : undefined + const sourcePaths = new Set(pagePaths(state.siteData.pages.filter(page => !page.generated))) + const upsertedPaths = new Set(/** @type {string[]} */ ([])) + for (const event of events) { + const eventPlan = planWatchEvent(state, event) + const structural = event.type === 'added' || event.type === 'removed' + const changePlan = structural ? planWatchEvent(state, { ...event, type: 'change' }) : eventPlan + const reason = inputResetReason(state, event, changePlan) + resetReason ??= reason + plan = unionWatchPlans(plan, eventPlan) + if (reason === 'unknown-event' || reason === 'unreliable-event') { + plan = { kind: 'full', message: `"${event.name}" cannot be routed reliably, triggering full rebuild...` } + } + + const sourceEvent = event.name.endsWith('.md') || + (event.convention?.change === 'page' && !fileConventions.pageVars.names.includes(event.name)) || + sourcePaths.has(event.filepath) + const allInputs = changePlan.kind === 'full' || + (structural && !sourceEvent) || reason === 'unreliable-event' + const selected = changePlan.kind === 'pages' ? changePlan.pageFilterPaths : [] + for (const path of allInputs || selected === null ? sourcePaths : selected) { + if (sourcePaths.has(path)) upsertedPaths.add(path) + } + if (structural && sourceEvent) upsertedPaths.add(event.filepath) + } + if (state.pageBuildFailed || state.dependencyAnalysisFailed) { + plan = unionWatchPlans(plan, allPages()) + } + if (resetReason) inputChanges.resetReason = resetReason + inputChanges.upsertedPaths = [...upsertedPaths] + return { plan, inputChanges } +} + +/** + * @param {WatchSnapshot} state + * @param {WatchEvent} event + * @param {WatchPlan} changePlan + */ +function inputResetReason (state, { type, filepath, name, convention }, changePlan) { + if (type !== 'change' && type !== 'added' && type !== 'removed') return 'unreliable-event' + if (convention?.change === 'pages' || state.globalDataDepPaths.has(filepath)) return 'global-data-changed' + if (fileConventions.globalVars.names.includes(name) || state.globalVarsDepPaths?.has(filepath)) return 'global-vars-changed' + if (convention?.change === 'markdown' || state.markdownDepPaths?.has(filepath)) return 'markdown-settings-changed' + if (convention?.change === 'full') return 'global-config-changed' + if (type === 'change' && changePlan.kind === 'skip' && + convention?.change !== 'manifest' && !state.esbuildEntryPoints.has(filepath) && + !state.esbuildDepPaths?.has(filepath) && !hasWatchRole(state, filepath)) { + return 'unknown-event' + } +} + +/** @param {WatchSnapshot} state @param {string} filepath */ +function hasWatchRole (state, filepath) { + return state.globalDataDepPaths.has(filepath) || state.globalVarsDepPaths?.has(filepath) || state.markdownDepPaths?.has(filepath) || + state.pageFileMap.has(filepath) || state.layoutFileMap.has(filepath) || state.layoutDepMap.has(filepath) || + state.pageDepMap.has(filepath) || state.templateDepMap.has(filepath) || state.pagesFileDepMap.has(filepath) || + state.siteData.templates.some(template => template.templateFile.filepath === filepath) || + state.siteData.pagesFiles?.some(owner => owner.pagesFile.filepath === filepath) +} + +/** @param {WatchPlan} left @param {WatchPlan} right @returns {WatchPlan} */ +function unionWatchPlans (left, right) { + if (left.kind === 'full') return left + if (right.kind === 'full') return right + if (left.kind === 'restart' || right.kind === 'restart') { + return { kind: 'full', message: 'Bundle membership changed, triggering full rebuild...' } + } + if (right.kind === 'skip') return left.kind === 'skip' ? right : left + if (left.kind === 'skip') return right + const messages = new Set([...(left.message?.split('\n') ?? []), ...(right.message?.split('\n') ?? [])]) + return { + kind: 'pages', + pageFilterPaths: unionFilterPaths(left.pageFilterPaths, right.pageFilterPaths), + templateFilterPaths: unionFilterPaths(left.templateFilterPaths, right.templateFilterPaths), + pagesFileFilterPaths: unionFilterPaths(left.pagesFileFilterPaths, right.pagesFileFilterPaths), + ...(left.pages || right.pages + ? { + pages: [...new Map([...(left.pages ?? []), ...(right.pages ?? [])].map(page => [page.pageFile.filepath, page])).values()], + } + : {}), + ...(left.templates || right.templates + ? { + templates: [...new Map([...(left.templates ?? []), ...(right.templates ?? [])].map(template => [template.templateFile.filepath, template])).values()], + } + : {}), + ...(messages.size ? { message: [...messages].join('\n') } : {}), + } +} + +/** @param {string[] | null} left @param {string[] | null} right */ +function unionFilterPaths (left, right) { + return left === null || right === null ? null : [...new Set([...left, ...right])] +} + /** * Select HTML consumers after discovery and esbuild have refreshed bundle metadata. * The reverse maps still describe the last successful map refresh, including diff --git a/lib/watch-plan.test.js b/lib/watch-plan.test.js index ba5b8886..3b2ccc22 100644 --- a/lib/watch-plan.test.js +++ b/lib/watch-plan.test.js @@ -4,8 +4,8 @@ */ import { test } from 'node:test' import assert from 'node:assert/strict' -import { basename, dirname, join } from 'node:path' -import { classifyWatchEvent, planWatchEvent, planBundleChange } from './watch-plan.js' +import { basename, dirname, join, resolve } from 'node:path' +import { classifyWatchEvent, planWatchEvent, planWatchBatch, planBundleChange } from './watch-plan.js' const src = '/site' @@ -36,8 +36,12 @@ function fixture () { pagesFileDepMap: new Map([['/site/archive-helper.js', new Set([owner])]]), pagesFileLayoutMap: new Map([[owner.pagesFile.filepath, new Set(['root'])]]), globalDataDepPaths: new Set(['/site/global.data.js']), + globalVarsDepPaths: new Set(), + markdownDepPaths: new Set(), + dependencyAnalysisFailed: false, pageBuildFailed: false, esbuildEntryPoints: new Set(['/site/client.jsx', '/site/root.layout.css']), + esbuildDepPaths: new Set(), }) return { state, home, other, template, owner } } @@ -187,3 +191,358 @@ test('bundle plans retry the full page phase after failure but preserve service- } assert.deepEqual(failed, before) }) + +test('classification resolves paths without changing the public event shape', () => { + const event = classifyWatchEvent('added', './nested/../page.md') + assert.deepEqual(event, { + type: 'added', + filepath: resolve('page.md'), + name: 'page.md', + convention: classifyWatchEvent('change', '/site/page.md').convention, + }) + assert.equal(classifyWatchEvent('change', '/site/nested/../page.md').filepath, '/site/page.md') +}) + +test('snapshots without optional dependency analysis fields remain compatible', () => { + const { state, home } = fixture() + const { globalVarsDepPaths, markdownDepPaths, dependencyAnalysisFailed, esbuildDepPaths, ...legacy } = state + assert.deepEqual(scope(planWatchEvent(legacy, classifyWatchEvent('change', '/site/page.md'))), [[home.pageFile.filepath], [], []]) + assert.equal(planWatchBatch(legacy, [classifyWatchEvent('change', '/site/page.md')]).inputChanges.resetReason, undefined) + assert.equal(planWatchBatch(legacy, [classifyWatchEvent('change', '/site/unknown.js')]).inputChanges.resetReason, 'unknown-event') +}) + +test('markdown roots and imports union server roles before browser or manifest skips', () => { + for (const name of ['markdown-it.settings.js', 'markdown-helper.js', 'client.jsx', 'domstack-manifest.settings.js']) { + const { state, home, other, template, owner } = fixture() + const filepath = `/site/${name}` + if (name !== 'markdown-it.settings.js') state.markdownDepPaths.add(filepath) + state.pageDepMap.set(filepath, new Set([other])) + state.layoutDepMap.set(filepath, new Set(['root'])) + state.templateDepMap.set(filepath, new Set([template])) + state.pagesFileDepMap.set(filepath, new Set([owner])) + state.esbuildEntryPoints.add(filepath) + const event = classifyWatchEvent('change', filepath) + assert.deepEqual(scope(planWatchEvent(state, event)), [[other.pageFile.filepath, home.pageFile.filepath], [template.templateFile.filepath], [owner.pagesFile.filepath]], name) + const batch = planWatchBatch(state, [event]) + assert.deepEqual(scope(batch.plan), scope(planWatchEvent(state, event))) + assert.equal(batch.inputChanges.resetReason, 'markdown-settings-changed') + } +}) + +test('global vars imports dominate every shared role and global data roots need no map entry', () => { + for (const name of ['vars-helper.js', 'client.jsx', 'markdown-it.settings.js', 'domstack-manifest.settings.js']) { + const { state, other } = fixture() + const filepath = `/site/${name}` + state.globalVarsDepPaths.add(filepath) + state.markdownDepPaths.add(filepath) + state.pageDepMap.set(filepath, new Set([other])) + const event = classifyWatchEvent('change', filepath) + assert.equal(planWatchEvent(state, event).kind, 'full', name) + assert.equal(planWatchBatch(state, [event]).inputChanges.resetReason, 'global-vars-changed', name) + } + const { state } = fixture() + state.globalDataDepPaths.clear() + assert.deepEqual(scope(planWatchEvent(state, classifyWatchEvent('change', '/site/global.data.js'))), [[], [], []]) + state.siteData.pages = [] + assert.deepEqual(scope(planWatchEvent(state, classifyWatchEvent('change', '/site/markdown-it.settings.js'))), [[], [], []]) +}) + +test('manifest settings imported by server consumers do not hide those roles', () => { + const { state, other } = fixture() + const filepath = '/site/domstack-manifest.settings.js' + state.pageDepMap.set(filepath, new Set([other])) + state.globalDataDepPaths.add(filepath) + const event = classifyWatchEvent('change', filepath) + assert.deepEqual(scope(planWatchEvent(state, event)), [[other.pageFile.filepath], [], []]) + assert.equal(planWatchBatch(state, [event]).inputChanges.resetReason, 'global-data-changed') +}) + +test('batch unions filtered output scopes and source inputs, retaining event order and duplicates', () => { + const { state, home, other, template, owner } = fixture() + const events = ['page-helper.js', 'layout-helper.js', 'template-helper.js', 'archive-helper.js', 'page.md', 'client.jsx', 'page-helper.js'] + .map(name => classifyWatchEvent('change', `/site/${name}`)) + const before = structuredClone({ state, events }) + events.forEach(Object.freeze) + Object.freeze(events) + const batch = planWatchBatch(state, events) + assert.deepEqual(scope(batch.plan), [[other.pageFile.filepath, home.pageFile.filepath], [template.templateFile.filepath], [owner.pagesFile.filepath]]) + assert.equal(batch.plan.kind, 'pages') + if (batch.plan.kind !== 'pages') throw new Error('Expected a page plan') + assert.deepEqual(batch.plan.pages, [other, home]) + assert.deepEqual(batch.plan.templates, [template]) + assert.equal(batch.plan.message, undefined, 'browser skip messages do not replace rebuild logging') + assert.deepEqual(batch.inputChanges, { + upsertedPaths: [other.pageFile.filepath, home.pageFile.filepath], + events, + }) + assert.notEqual(batch.inputChanges.events, events) + assert.deepEqual(planWatchBatch(state, events), batch) + assert.deepEqual({ state, events }, before) +}) + +test('single page plans preserve exact logging metadata and messages even when surrounded by skips', () => { + const { state } = fixture() + const skipped = classifyWatchEvent('change', '/site/client.jsx') + for (const name of ['page.md', 'layout-helper.js', 'feed.template.js', 'global.data.js', 'markdown-it.settings.js']) { + const event = classifyWatchEvent('change', `/site/${name}`) + const expected = planWatchEvent(state, event) + assert.equal(expected.kind, 'pages') + for (const events of [[event], [skipped, event], [event, skipped], [skipped, event, skipped]]) { + assert.deepEqual(planWatchBatch(state, events).plan, expected, name) + } + const failed = { ...state, pageBuildFailed: true } + assert.deepEqual(planWatchBatch(failed, [event]).plan, planWatchEvent(failed, event), 'recovery preserves the single-event message') + } +}) + +test('page-plan unions retain distinct messages in event order without duplicate lines', () => { + const { state, home, template } = fixture() + state.globalDataDepPaths.add('/site/data-helper.js') + const data = classifyWatchEvent('change', '/site/global.data.js') + const helper = classifyWatchEvent('change', '/site/data-helper.js') + const pageEvent = classifyWatchEvent('change', home.pageFile.filepath) + const templateEvent = classifyWatchEvent('change', template.templateFile.filepath) + for (const [first, second] of [[data, helper], [helper, data]]) { + assert.ok(first && second) + const batch = planWatchBatch(state, [first, pageEvent, second, templateEvent, first, second]) + assert.equal(batch.plan.kind, 'pages') + if (batch.plan.kind !== 'pages') throw new Error('Expected a page plan') + assert.deepEqual(batch.plan.pages, [home]) + assert.deepEqual(batch.plan.templates, [template]) + assert.equal(batch.plan.message, [planWatchEvent(state, first).message, planWatchEvent(state, second).message].join('\n')) + } +}) + +test('known browser-only dependencies skip page rebuilding in both planners without resetting inputs', () => { + const { state } = fixture() + for (const filepath of ['/site/browser-helper.js', '/site/nested/client.js']) { + state.esbuildDepPaths.add(filepath) + assert.equal(state.esbuildEntryPoints.has(filepath), false) + const event = classifyWatchEvent('change', filepath) + const single = planWatchEvent(state, event) + assert.equal(single.kind, 'skip') + assert.ok(single.message?.includes('esbuild will handle rebundling')) + const batch = planWatchBatch(state, [event]) + assert.deepEqual(batch.plan, single) + assert.deepEqual(batch.inputChanges, { upsertedPaths: [], events: [event] }) + } + const unknown = planWatchBatch(state, [classifyWatchEvent('change', '/site/unknown.js')]) + assert.equal(unknown.plan.kind, 'full') + assert.equal(unknown.inputChanges.resetReason, 'unknown-event') +}) + +test('browser dependency membership cannot hide mapped server consumers or global reset reasons', () => { + const { state } = fixture() + state.globalDataDepPaths.add('/site/data-helper.js') + state.markdownDepPaths.add('/site/markdown-helper.js') + state.globalVarsDepPaths.add('/site/vars-helper.js') + for (const name of ['page.md', 'page.vars.js', 'root.layout.js', 'layout-helper.js', 'page-helper.js', 'template-helper.js', 'archive-helper.js', 'data-helper.js', 'markdown-helper.js', 'vars-helper.js']) { + const filepath = `/site/${name}` + const event = classifyWatchEvent('change', filepath) + const single = planWatchEvent(state, event) + const batch = planWatchBatch(state, [event]) + assert.notEqual(single.kind, 'skip', name) + state.esbuildDepPaths.add(filepath) + assert.deepEqual(planWatchEvent(state, event), single, name) + assert.deepEqual(planWatchBatch(state, [event]), batch, name) + } +}) + +test('known browser dependencies do not suppress recovery after build or dependency-analysis failures', () => { + const { state } = fixture() + const event = classifyWatchEvent('change', '/site/browser-helper.js') + state.esbuildDepPaths.add(event.filepath) + for (const flag of /** @type {const} */ (['pageBuildFailed', 'dependencyAnalysisFailed'])) { + const batch = planWatchBatch({ ...state, [flag]: true }, [event]) + assert.deepEqual(scope(batch.plan), [null, null, null]) + assert.equal(batch.inputChanges.resetReason, flag === 'pageBuildFailed' ? 'page-build-failed' : 'dependency-analysis-failed') + } +}) + +test('template and generated-page owner changes do not invent source inputs', () => { + const { state, template, owner } = fixture() + const batch = planWatchBatch(state, ['feed.template.js', 'archive.pages.js', 'archive-helper.js'] + .map(name => classifyWatchEvent('change', `/site/${name}`))) + assert.deepEqual(scope(batch.plan), [[], [template.templateFile.filepath], [owner.pagesFile.filepath]]) + assert.deepEqual(batch.inputChanges.upsertedPaths, []) + assert.equal(batch.inputChanges.resetReason, undefined) +}) + +test('null filters dominate filtered scopes and failure resets override event reasons', () => { + for (const flag of /** @type {const} */ (['pageBuildFailed', 'dependencyAnalysisFailed'])) { + const { state: snapshot } = fixture() + const state = { ...snapshot, pageBuildFailed: flag === 'pageBuildFailed', dependencyAnalysisFailed: flag === 'dependencyAnalysisFailed' } + const events = ['page.md', 'feed.template.js', 'global.data.js'].map(name => classifyWatchEvent('change', `/site/${name}`)) + const batch = planWatchBatch(state, events) + assert.deepEqual(scope(batch.plan), [null, null, null]) + assert.equal(batch.inputChanges.resetReason, flag === 'pageBuildFailed' ? 'page-build-failed' : 'dependency-analysis-failed') + const skipped = planWatchBatch(state, [classifyWatchEvent('change', '/site/domstack-manifest.settings.js')]) + assert.deepEqual(scope(skipped.plan), [null, null, null], 'a nonempty recovery batch cannot skip') + state.pageBuildFailed = state.dependencyAnalysisFailed = true + assert.equal(planWatchBatch(state, events).inputChanges.resetReason, 'page-build-failed') + } +}) + +test('empty and intentionally skipped batches have no input changes', () => { + const { state } = fixture() + for (const events of [[], ['client.jsx', 'domstack-manifest.settings.js'].map(name => classifyWatchEvent('change', `/site/${name}`))]) { + const batch = planWatchBatch(state, events) + assert.equal(batch.plan.kind, 'skip') + assert.deepEqual(batch.inputChanges, { upsertedPaths: [], events }) + } + const failed = { ...state, pageBuildFailed: true, dependencyAnalysisFailed: true } + assert.deepEqual(planWatchBatch(failed, []).inputChanges, { upsertedPaths: [], events: [] }) + assert.equal(planWatchBatch(failed, []).plan.kind, 'skip') +}) + +test('structural source pages carry membership candidates and change-plan consumers without resetting', () => { + for (const type of /** @type {const} */ (['added', 'removed'])) { + for (const name of ['page.md', 'new/page.html', 'new/page.js', 'new/page.draft.mjs', 'new/README.md', 'new/README.draft.md', 'new/post.md', 'new/post.draft.md']) { + const { state, home, other } = fixture() + const filepath = `/site/${name}` + state.pageDepMap.set(filepath, new Set([other])) + const event = classifyWatchEvent(type, filepath) + const batch = planWatchBatch(state, [event]) + assert.equal(batch.plan.kind, 'full', 'rediscovery is required') + const expected = [other.pageFile.filepath] + if (name === 'page.md') expected.push(home.pageFile.filepath) + expected.push(filepath) + assert.deepEqual(batch.inputChanges, { upsertedPaths: [...new Set(expected)], events: [event] }, `${type} ${name}`) + } + } +}) + +test('structural events preserve replacement and removal order for membership reconciliation', () => { + const { state } = fixture() + const events = [ + classifyWatchEvent('removed', '/site/page.md'), + classifyWatchEvent('added', '/site/page.html'), + classifyWatchEvent('added', '/site/new/post.md'), + classifyWatchEvent('removed', '/site/new/post.md'), + ] + const batch = planWatchBatch(state, events) + assert.equal(batch.plan.kind, 'full') + assert.deepEqual(batch.inputChanges, { upsertedPaths: ['/site/page.md', '/site/page.html', '/site/new/post.md'], events }) +}) + +test('structural layout, vars, unknown, template, and owner events conservatively upsert only source pages', () => { + for (const type of /** @type {const} */ (['added', 'removed'])) { + for (const name of ['page.vars.js', 'new/page.vars.mjs', 'root.layout.js', 'new.layout.js', 'helper.js', 'feed.template.js', 'archive.pages.js']) { + const { state, home, other, owner } = fixture() + state.siteData.pages.push({ ...page('generated'), generated: { pagesFile: owner } }) + const event = classifyWatchEvent(type, `/site/${name}`) + const batch = planWatchBatch(state, [event]) + assert.equal(batch.plan.kind, 'full') + assert.deepEqual(batch.inputChanges, { upsertedPaths: [home.pageFile.filepath, other.pageFile.filepath], events: [event] }, `${type} ${name}`) + } + } +}) + +test('batch bundle restarts become full without resetting, including when mixed with page or full plans', () => { + for (const name of ['client.jsx', 'root.layout.css', 'global.css', 'service-worker.js']) { + const { state, home, other } = fixture() + const restart = classifyWatchEvent('removed', `/site/${name}`) + assert.equal(planWatchEvent(state, restart).kind, 'restart') + for (const events of [[restart], [restart, classifyWatchEvent('change', '/site/page.md')], [classifyWatchEvent('added', '/site/new/page.md'), restart], [restart, classifyWatchEvent('added', '/site/new/page.md')]]) { + const batch = planWatchBatch(state, events) + assert.equal(batch.plan.kind, 'full') + assert.equal(batch.inputChanges.resetReason, undefined) + assert.ok(batch.inputChanges.upsertedPaths.includes(home.pageFile.filepath)) + assert.ok(batch.inputChanges.upsertedPaths.includes(other.pageFile.filepath)) + } + } +}) + +test('global roots and imports reset on changes, additions, and removals regardless of shared roles', () => { + const cases = /** @type {const} */ ([ + ['global.data.js', 'globalDataDepPaths', 'global-data-changed'], + ['global.vars.js', 'globalVarsDepPaths', 'global-vars-changed'], + ['markdown-it.settings.js', 'markdownDepPaths', 'markdown-settings-changed'], + ['esbuild.settings.js', null, 'global-config-changed'], + ]) + for (const type of /** @type {const} */ (['change', 'added', 'removed'])) { + for (const [name, depSet, reason] of cases) { + const { state, other } = fixture() + state.globalDataDepPaths.clear() + const root = classifyWatchEvent(type, `/site/${name}`) + assert.equal(planWatchBatch(state, [root]).inputChanges.resetReason, reason, `${type} ${name}`) + if (!depSet) continue + for (const sharedName of ['helper.js', 'client.jsx', 'page.md', 'domstack-manifest.settings.js']) { + const filepath = `/site/${sharedName}` + state[depSet].add(filepath) + state.pageDepMap.set(filepath, new Set([other])) + const batch = planWatchBatch(state, [classifyWatchEvent(type, filepath)]) + assert.equal(batch.inputChanges.resetReason, reason, `${type} ${sharedName}`) + } + } + } +}) + +test('unknown changes reset and rebuild but known unused layouts and browser entries can skip', () => { + const { state } = fixture() + const unknown = classifyWatchEvent('change', '/site/unknown.js') + assert.equal(planWatchEvent(state, unknown).kind, 'skip', 'single-event behavior is unchanged') + const batch = planWatchBatch(state, [unknown]) + assert.equal(batch.plan.kind, 'full') + assert.equal(batch.inputChanges.resetReason, 'unknown-event') + state.layoutPageMap.clear() + state.pagesFileLayoutMap.clear() + const unusedLayout = planWatchBatch(state, [classifyWatchEvent('change', '/site/root.layout.js')]) + assert.equal(unusedLayout.plan.kind, 'skip') + assert.equal(unusedLayout.inputChanges.resetReason, undefined) + + const unreliable = { ...unknown, type: 'rename' } + // @ts-expect-error Runtime callers with unsupported event types must reset, without widening WatchEvent. + const invalidBatch = planWatchBatch(state, [unreliable]) + assert.equal(invalidBatch.plan.kind, 'full') + assert.equal(invalidBatch.inputChanges.resetReason, 'unreliable-event') +}) + +test('page-plan unions deduplicate paths and logging metadata for distinct objects representing one consumer', () => { + const { state, home, template, owner } = fixture() + state.pageDepMap.set('/site/shared.js', new Set([home, { ...home }])) + state.templateDepMap.set('/site/shared.js', new Set([template, { ...template }])) + state.pagesFileDepMap.set('/site/shared.js', new Set([owner, { ...owner }])) + const event = classifyWatchEvent('change', '/site/shared.js') + assert.deepEqual(planWatchBatch(state, [event]).plan, planWatchEvent(state, event), 'a single plan is preserved exactly') + const batch = planWatchBatch(state, [event, event]) + assert.deepEqual(scope(batch.plan), [[home.pageFile.filepath], [template.templateFile.filepath], [owner.pagesFile.filepath]]) + assert.equal(batch.plan.kind, 'pages') + if (batch.plan.kind !== 'pages') throw new Error('Expected a page plan') + assert.deepEqual(batch.plan.pages, [home]) + assert.deepEqual(batch.plan.templates, [template]) + assert.deepEqual(batch.inputChanges.upsertedPaths, [home.pageFile.filepath]) +}) + +test('markdown-only imports select only markdown pages and reset producer inputs', () => { + const { state, home } = fixture() + state.markdownDepPaths.add('/site/markdown-helper.js') + const event = classifyWatchEvent('change', '/site/markdown-helper.js') + assert.deepEqual(scope(planWatchEvent(state, event)), [[home.pageFile.filepath], [], []]) + const batch = planWatchBatch(state, [event]) + assert.deepEqual(scope(batch.plan), [[home.pageFile.filepath], [], []]) + assert.equal(batch.inputChanges.resetReason, 'markdown-settings-changed') +}) + +test('a full plan never short-circuits later input deltas or reset reasons', () => { + const { state } = fixture() + const restart = classifyWatchEvent('added', '/site/client.jsx') + const config = classifyWatchEvent('change', '/site/global.vars.js') + const added = classifyWatchEvent('added', '/site/new/page.md') + for (const events of [[restart, config, added], [config, added, restart], [added, restart, config]]) { + const batch = planWatchBatch(state, events) + assert.equal(batch.plan.kind, 'full') + assert.equal(batch.inputChanges.resetReason, 'global-vars-changed') + assert.ok(batch.inputChanges.upsertedPaths.includes(added.filepath)) + assert.deepEqual(batch.inputChanges.events, events) + } +}) + +test('reset reason uses the first applicable event after snapshot failure precedence', () => { + const { state } = fixture() + const data = classifyWatchEvent('change', '/site/global.data.js') + const markdown = classifyWatchEvent('change', '/site/markdown-it.settings.js') + assert.equal(planWatchBatch(state, [data, markdown]).inputChanges.resetReason, 'global-data-changed') + assert.equal(planWatchBatch(state, [markdown, data]).inputChanges.resetReason, 'markdown-settings-changed') + assert.deepEqual(scope(planWatchBatch(state, [data, markdown]).plan), [['/site/page.md'], [], []], 'reset does not force full output selection') +}) diff --git a/site/globals/global.data.ts b/site/globals/global.data.ts index 053ac730..54127367 100644 --- a/site/globals/global.data.ts +++ b/site/globals/global.data.ts @@ -1,10 +1,25 @@ import type { GlobalDataFunctionParams } from '../../types.ts' +import type { DocsNavigationIndex, DocsPageVars } from '../layouts/docs/navigation.js' import { render } from 'fragtml' -import { collectDocsNavigation, docsIndex } from '../layouts/docs/navigation.js' +import { readDocsNavigationPage, projectDocsNavigation, docsIndex } from '../layouts/docs/navigation.js' -// The collector excludes the index, whose Markdown consumes docsIndexHtml. -// Other source pages are rendered without their data-subscribing layouts. -export default async function ({ pages }: GlobalDataFunctionParams) { - const docsNavigation = await collectDocsNavigation(pages) - return { docsNavigation, docsIndexHtml: render(docsIndex(docsNavigation)) } +export default async function ({ pages, previousState, changes, setState }: GlobalDataFunctionParams) { + const reset = changes.kind === 'reset' || previousState === undefined + const index: DocsNavigationIndex = reset ? new Map() : new Map(previousState) + if (changes.kind === 'delta') { + for (const filepath of changes.removed) index.delete(filepath) + } + const inputs = changes.kind === 'delta' && !reset ? changes.upserted : pages + await Promise.all(inputs.map(async page => { + const filepath = page.pageInfo.pageFile.filepath + // Excludes the data-dependent index and renders docs without their layouts. + const record = await readDocsNavigationPage(page) + if (record) index.set(filepath, record) + else index.delete(filepath) + })) + + const docsNavigation = projectDocsNavigation(index.values()) + const docsIndexHtml = render(docsIndex(docsNavigation)) + setState(index) + return { docsNavigation, docsIndexHtml } } diff --git a/site/layouts/docs/navigation-watch.test.js b/site/layouts/docs/navigation-watch.test.js index 6b97c1dc..4c89a3e4 100644 --- a/site/layouts/docs/navigation-watch.test.js +++ b/site/layouts/docs/navigation-watch.test.js @@ -1,10 +1,22 @@ +/** + * @import { TestContext } from 'node:test' + * @import { FSWatcher } from 'chokidar' + * @import { NavigationEntry } from './navigation.js' + * @typedef {{ docsNavigation: NavigationEntry[], docsIndexHtml: string }} DocsData + * @typedef {{ kind: 'reset' | 'delta', rendered: string[], threadId: number, invocation: number, publicData: DocsData }} ProducerCall + * @typedef {{ type: 'add' | 'unlink' | 'change', file: string }} SourceEvent + */ import assert from 'node:assert/strict' +import { EventEmitter } from 'node:events' import { test } from 'node:test' -import { cp, mkdir, mkdtemp, readFile, rm, stat, writeFile } from 'node:fs/promises' +import { cp, mkdir, mkdtemp, readFile, readdir, rename, rm, stat, utimes, writeFile } from 'node:fs/promises' import { dirname, join, resolve } from 'node:path' +import { setImmediate as nextTurn } from 'node:timers/promises' +import chokidar from 'chokidar' import pino from 'pino' import { DomStack } from '../../../index.js' +import { startWatch } from '../../../test-cases/watch/helpers.js' test('heading changes refresh shared navigation; body edits leave other pages alone', { timeout: 30_000 }, async t => { const root = resolve(import.meta.dirname, '../../..') @@ -35,7 +47,7 @@ test('heading changes refresh shared navigation; body edits leave other pages al await write('docs/README.md', index) await write('docs/first/README.md', first) await write('docs/second/README.md', '---\nlayout: docs\ndocsOrder: 20\n---\n# Second\n\n## Other') - const result = await domstack.watch({ serve: false }) + const result = await startWatch(t, domstack, src) assert.equal(result.pageBuildResults?.errors.length, 0) const output = join(dest, 'docs/second/index.html') assert.match(await readFile(output, 'utf8'), /first\/#original/) @@ -57,3 +69,320 @@ test('heading changes refresh shared navigation; body edits leave other pages al assert.match(await readFile(join(dest, 'docs/first/index.html'), 'utf8'), /Changed body only/) assert.equal((await stat(output)).mtimeMs, mtime, 'unchanged navigation does not invalidate another page') }) + +const eligibleDocs = [ + 'docs/first/README.md', + 'docs/first/child/README.md', + 'docs/first/child/leaf/README.md', + 'docs/second/README.md', +] +const firstDoc = doc('First', 'docsGroup: Guides\ndocsOrder: 10\n') +const secondDoc = doc('Second', 'docsGroup: Reference\ndocsOrder: 20\n') +const leafDoc = doc('Leaf', 'docsGroup: Guides\ndocsParent: /docs/first/child/\n') +const options = { timeout: 30_000 } + +/** @param {string} title @param {string} [vars] */ +function doc (title, vars = '') { + return `---\nlayout: docs\n${vars}---\n# ${title}\n\n## Original\n\nBody\n` +} + +/** @param {ProducerCall} call @param {'reset' | 'delta'} kind @param {string[]} rendered */ +function assertRenders (call, kind, rendered) { + assert.equal(call.kind, kind) + assert.deepEqual(call.rendered, [...rendered].sort(), 'actual producer renderInnerPage calls, including duplicates') +} + +/** @param {string} directory @returns {Promise>} */ +async function docsSnapshot (directory) { + /** @type {Record} */ + const snapshot = {} + for (const entry of await readdir(directory, { withFileTypes: true })) { + const path = join(directory, entry.name) + if (entry.isDirectory()) { + for (const [name, content] of Object.entries(await docsSnapshot(path))) snapshot[join(entry.name, name)] = content + } else { + snapshot[entry.name] = await readFile(path, 'utf8') + } + } + return snapshot +} + +/** + * Mock only source notifications; discovery, dependency tracking, worker isolation, + * rendering and output writes still run through the real watch/build implementation. + * @param {TestContext} t + */ +async function instrumentedFixture (t) { + const root = resolve(import.meta.dirname, '../../..') + const temp = await mkdtemp(join(root, '.tmp-docs-navigation-')) + const src = join(temp, 'src') + const dest = join(temp, 'public') + const log = join(temp, 'producer.jsonl') + const buildOptions = { logger: pino({ level: 'silent' }), metafile: false, domstackManifest: false } + const domstack = new DomStack(src, dest, buildOptions) + t.after(async () => { + try { + if (domstack.watching) await domstack.stopWatching() + } finally { + await rm(temp, { recursive: true, force: true }) + } + }) + for (const file of [ + 'site/globals/global.data.ts', + 'site/layouts/docs/navigation.js', + 'site/layouts/docs/docs.layout.js', + 'site/layouts/root/root.layout.js', + ]) { + await mkdir(dirname(join(src, file)), { recursive: true }) + await cp(join(root, file), join(src, file)) + } + /** @param {string} file @param {string} text */ + async function write (file, text) { + await mkdir(dirname(join(src, file)), { recursive: true }) + await writeFile(join(src, file), text) + } + + // Keep the real producer's relative imports, but only the wrapper is a global root. + await rename(join(src, 'site/globals/global.data.ts'), join(src, 'site/globals/producer.ts')) + await write('site/globals/global.data.ts', ` +import { appendFileSync } from 'node:fs' +import { relative } from 'node:path' +import { threadId } from 'node:worker_threads' +import produce from './producer.ts' +let invocation = 0 +export default async function (params) { + const rendered = [] + const originals = params.pages.map(page => [page, page.renderInnerPage]) + for (const [page, render] of originals) { + page.renderInnerPage = function (...args) { + rendered.push(relative(${JSON.stringify(src)}, page.pageInfo.pageFile.filepath)) + return Reflect.apply(render, this, args) + } + } + let publicData + try { + publicData = await Reflect.apply(produce, this, [params]) + } finally { + for (const [page, render] of originals) page.renderInnerPage = render + } + appendFileSync(${JSON.stringify(log)}, JSON.stringify({ + kind: params.changes.kind, rendered: rendered.sort(), + threadId, invocation: ++invocation, publicData, + }) + '\\n') + return publicData +} +`) + await write('docs/README.md', '---\nlayout: docs\ndataDeps: [docsIndexHtml]\n---\n# Docs\n\n{{{ data.docsIndexHtml }}}') + await write('docs/first/README.md', firstDoc) + await write('docs/second/README.md', secondDoc) + await write('docs/first/child/README.md', doc('Child', 'docsGroup: Guides\ndocsParent: /docs/first/\n')) + await write('docs/first/child/leaf/README.md', leafDoc) + await write('outside/README.md', '---\nlayout: root\n---\n# Not documentation') + await write('docs/script/page.js', "export const vars = { layout: 'docs' }; export default () => '

Not Markdown

'\n") + await write('docs/hidden/page.draft.md', doc('Unpublished')) + await write('docs/navigation.json.template.js', ` +export const dataDeps = ['docsNavigation', 'docsIndexHtml'] +export default ({ data }) => JSON.stringify({ docsNavigation: data.docsNavigation, docsIndexHtml: data.docsIndexHtml }) +`) + await write('site/globals/markdown-it.settings.js', 'export default md => md.set({ typographer: false })\n') + + /** @type {FSWatcher | undefined} */ + let watcher + const watch = chokidar.watch + /** @param {Parameters} args */ + const sourceWatch = (...args) => { + if (args[0] !== src) return watch(...args) + const fake = Object.assign(new EventEmitter(), { + closed: false, + async close () { fake.closed = true }, + }) + watcher = /** @type {FSWatcher} */ (/** @type {unknown} */ (fake)) + setImmediate(() => { if (!fake.closed) fake.emit('ready') }) + return watcher + } + t.mock.method(chokidar, 'watch', sourceWatch) + + /** @returns {Promise} */ + async function calls () { + const text = await readFile(log, 'utf8') + return text.trim().split('\n').map(line => JSON.parse(line)) + } + /** @type {Set} */ + const workers = new Set() + /** @param {number} before */ + async function newCall (before) { + const added = (await calls()).slice(before) + assert.equal(added.length, 1, 'one producer invocation per build, not one per output') + const call = added[0] + assert.ok(call) + assert.equal(call.invocation, 1, 'the producer module is loaded anew for every build') + assert.ok(call.threadId > 0, 'producer runs in a real worker') + assert.ok(!workers.has(call.threadId), 'each build uses a fresh worker, not module-local state') + workers.add(call.threadId) + assert.deepEqual(Object.keys(call.publicData).sort(), ['docsIndexHtml', 'docsNavigation'], 'public keys remain unchanged') + return call + } + /** @param {SourceEvent[]} events */ + async function rebuild (events) { + const before = (await calls()).length + assert.ok(watcher) + for (const { type, file } of events) watcher.emit(type, join(src, file)) + await nextTurn() + await domstack.settled() + return newCall(before) + } + /** @param {string} file @param {string} text */ + async function edit (file, text) { + await write(file, text) + return rebuild([{ type: 'change', file }]) + } + /** @param {string} file */ + const output = file => readFile(join(dest, 'docs', file), 'utf8') + /** @param {ProducerCall} watched @param {string[]} [eligible] */ + async function matchesFresh (watched, eligible = eligibleDocs) { + const snapshot = await docsSnapshot(join(dest, 'docs')) + assert.deepEqual(JSON.parse(await output('navigation.json')), watched.publicData, 'published data matches the producer return value') + const fresh = join(temp, 'fresh') + await rm(fresh, { recursive: true, force: true }) + const before = (await calls()).length + const report = await new DomStack(src, fresh, buildOptions).build() + assert.deepEqual(report.pageBuildResults?.errors, [], 'fresh build succeeded') + const call = await newCall(before) + assertRenders(call, 'reset', eligible) + assert.deepEqual(watched.publicData, call.publicData, 'incremental public data equals a full build') + assert.deepEqual(snapshot, await docsSnapshot(join(fresh, 'docs')), 'docs output paths and bytes equal a full build') + } + return { + src, + dest, + write, + rebuild, + edit, + output, + matchesFresh, + async start () { + const report = await domstack.watch({ serve: false }) + assert.deepEqual(report.pageBuildResults?.errors, [], 'initial watch build succeeded') + const call = await newCall(0) + assertRenders(call, 'reset', eligibleDocs) + return call + }, + } +} + +test('docs index resets render every eligible Markdown source in fresh workers', options, async t => { + const site = await instrumentedFixture(t) + const initial = await site.start() + assert.deepEqual(initial.publicData.docsNavigation.map(entry => entry.url), ['/docs/first/', '/docs/second/']) + assert.doesNotMatch(JSON.stringify(initial.publicData), /Not documentation|Not Markdown|Unpublished/) + await site.matchesFresh(initial) + await site.matchesFresh(initial) +}) + +test('body-only docs edits render one producer input and preserve public data and sibling mtimes', options, async t => { + const site = await instrumentedFixture(t) + const initial = await site.start() + const untouched = ['index.html', 'second/index.html', 'first/child/index.html', 'first/child/leaf/index.html', 'navigation.json'] + .map(file => join(site.dest, 'docs', file)) + const sentinel = new Date('2000-01-01T00:00:00Z') + for (const path of untouched) await utimes(path, sentinel, sentinel) + const mtimes = await Promise.all(untouched.map(async path => (await stat(path, { bigint: true })).mtimeNs)) + const call = await site.edit('docs/first/README.md', firstDoc.replace('Body', 'Only this body changed')) + assert.deepEqual(call.publicData, initial.publicData, 'both navigation and index HTML are unchanged') + assert.match(await site.output('first/index.html'), /Only this body changed/) + assert.deepEqual(await Promise.all(untouched.map(async path => (await stat(path, { bigint: true })).mtimeNs)), mtimes) + await site.matchesFresh(call) + assertRenders(call, 'delta', ['docs/first/README.md']) +}) + +test('heading edits render one producer input and refresh sibling navigation and the docs index', options, async t => { + const site = await instrumentedFixture(t) + const initial = await site.start() + const call = await site.edit('docs/first/README.md', firstDoc.replace('## Original', '## Renamed')) + assert.notDeepEqual(call.publicData.docsNavigation, initial.publicData.docsNavigation) + assert.notEqual(call.publicData.docsIndexHtml, initial.publicData.docsIndexHtml) + for (const file of ['index.html', 'second/index.html']) { + const html = await site.output(file) + assert.match(html, /first\/#renamed/) + assert.doesNotMatch(html, /first\/#original/) + } + await site.matchesFresh(call) + assertRenders(call, 'delta', ['docs/first/README.md']) +}) + +test('docs adds, renames and deletes reconcile navigation and output ownership like fresh builds', options, async t => { + const site = await instrumentedFixture(t) + await site.start() + const added = 'docs/third/README.md' + const renamed = 'docs/third/renamed.md' + await t.test('add', async () => { + await site.write(added, doc('Third', 'docsGroup: Guides\ndocsOrder: 5\n')) + const call = await site.rebuild([{ type: 'add', file: added }]) + assert.deepEqual(call.publicData.docsNavigation.map(entry => entry.url), ['/docs/third/', '/docs/first/', '/docs/second/']) + await site.matchesFresh(call, [...eligibleDocs, added]) + assertRenders(call, 'delta', [added]) + }) + await t.test('rename a directory index to a flat page', async () => { + await rename(join(site.src, added), join(site.src, renamed)) + const call = await site.rebuild([{ type: 'unlink', file: added }, { type: 'add', file: renamed }]) + assert.deepEqual(call.publicData.docsNavigation.map(entry => entry.url), ['/docs/third/renamed.html', '/docs/first/', '/docs/second/']) + await assert.rejects(site.output('third/index.html'), { code: 'ENOENT' }) + assert.match(await site.output('index.html'), /third\/renamed\.html#original/) + await site.matchesFresh(call, [...eligibleDocs, renamed]) + assertRenders(call, 'delta', [renamed]) + }) + await t.test('delete without rendering surviving docs', async () => { + await rm(join(site.src, renamed)) + const call = await site.rebuild([{ type: 'unlink', file: renamed }]) + assert.deepEqual(call.publicData.docsNavigation.map(entry => entry.url), ['/docs/first/', '/docs/second/']) + await assert.rejects(site.output('third/renamed.html'), { code: 'ENOENT' }) + assert.doesNotMatch(await site.output('index.html'), /Third/) + await site.matchesFresh(call) + assertRenders(call, 'delta', []) + }) +}) + +test('group, order and parent edits reproject cached docs without accumulating nested entries', options, async t => { + const site = await instrumentedFixture(t) + await site.start() + for (const [name, text, titles] of /** @type {const} */ ([ + ['group', secondDoc.replace('Reference', 'Basics'), ['Second', 'First']], + ['order', secondDoc.replace('Reference', 'Guides').replace('20', '5'), ['Second', 'First']], + ['reorder', secondDoc.replace('Reference', 'Guides').replace('20', '30'), ['First', 'Second']], + ])) { + await t.test(name, async () => { + const call = await site.edit('docs/second/README.md', text) + assert.deepEqual(call.publicData.docsNavigation.map(entry => entry.title), titles) + assert.equal(call.publicData.docsNavigation.find(entry => entry.title === 'Second')?.group, name === 'group' ? 'Basics' : 'Guides') + await site.matchesFresh(call) + assertRenders(call, 'delta', ['docs/second/README.md']) + }) + } + for (const parent of ['/docs/first/', undefined, '/docs/first/child/']) { + await t.test(`parent: ${parent ?? 'none'}`, async () => { + const text = leafDoc.replace('docsParent: /docs/first/child/\n', parent ? `docsParent: ${parent}\n` : '') + const call = await site.edit('docs/first/child/leaf/README.md', text) + const first = call.publicData.docsNavigation.find(entry => entry.url === '/docs/first/') + const child = first?.sections.find(entry => entry.url === '/docs/first/child/') + assert.ok(first && child) + const container = parent === first.url ? first.sections : parent === child.url ? child.sections : call.publicData.docsNavigation + assert.equal(container.filter(entry => entry.url === '/docs/first/child/leaf/').length, 1, 'leaf appears once beneath its current parent') + await site.matchesFresh(call) + assertRenders(call, 'delta', ['docs/first/child/leaf/README.md']) + }) + } +}) + +test('producer, navigation helper and Markdown settings edits reset all eligible docs', options, async t => { + const site = await instrumentedFixture(t) + const initial = await site.start() + for (const file of ['site/globals/producer.ts', 'site/layouts/docs/navigation.js', 'site/globals/markdown-it.settings.js']) { + await t.test(file, async () => { + const text = await readFile(join(site.src, file), 'utf8') + const call = await site.edit(file, text + '\n// Trigger a dependency reset.\n') + assert.deepEqual(call.publicData, initial.publicData) + await site.matchesFresh(call) + assertRenders(call, 'reset', eligibleDocs) + }) + } +}) diff --git a/site/layouts/docs/navigation.js b/site/layouts/docs/navigation.js index cbb86c26..9ddbd8a9 100644 --- a/site/layouts/docs/navigation.js +++ b/site/layouts/docs/navigation.js @@ -29,6 +29,16 @@ import { html, raw } from 'fragtml' * @property {string} [group] */ +/** + * Per-source derived data, before child pages are attached to their parents. + * @typedef {object} DocsNavigationRecord + * @property {NavigationEntry} entry + * @property {number} order + * @property {string | undefined} parent + */ + +/** @typedef {Map} DocsNavigationIndex */ + export const docsIndexUrl = '/docs/' const siteOrigin = 'https://docs.invalid' @@ -40,49 +50,69 @@ const siteOrigin = 'https://docs.invalid' * @returns {Promise} */ export async function collectDocsNavigation (pages) { - const docs = pages - .filter(page => page.pageInfo.type === 'md' && page.pageInfo.url.startsWith(docsIndexUrl) && page.pageInfo.url !== docsIndexUrl) - .sort((a, b) => { - const group = (a.vars.docsGroup ?? '').localeCompare(b.vars.docsGroup ?? '') - if (group) return group - const order = (a.vars.docsOrder ?? Infinity) - (b.vars.docsOrder ?? Infinity) - return order || a.pageInfo.url.localeCompare(b.pageInfo.url) - }) + const records = await Promise.all(pages.map(readDocsNavigationPage)) + return projectDocsNavigation(records.filter(record => record !== undefined)) +} +/** + * Render and parse just one eligible source page. Never retain PageData, parsed + * documents, or renderer functions in the cross-worker application index. + * @param {NavigationPage} page + * @returns {Promise} + */ +export async function readDocsNavigationPage (page) { + const url = page.pageInfo.url + if (page.pageInfo.type !== 'md' || !url.startsWith(docsIndexUrl) || url === docsIndexUrl) return + const document = load(String(await page.renderInnerPage())) /** @type {NavigationEntry[]} */ - const entries = await Promise.all(docs.map(async page => { - const url = page.pageInfo.url - const document = load(String(await page.renderInnerPage())) - /** @type {NavigationEntry[]} */ - const sections = [] - /** @type {NavigationEntry | undefined} */ - let parent - const headings = page.vars.docsPageOnly ? document([]) : document('h2[id], h3[id]') - headings.each((_, heading) => { - const title = document(heading).text().trim() - if (title.toLowerCase() === 'table of contents') return - const section = { - title, - url: `${url}#${encodeURIComponent(document(heading).attr('id') ?? '')}`, - sections: [], - } - if (heading.tagName === 'h3' && parent) { - parent.sections.push(section) - } else { - sections.push(section) - } - if (heading.tagName === 'h2') parent = section - }) - return { + const sections = [] + /** @type {NavigationEntry | undefined} */ + let parent + const headings = page.vars.docsPageOnly ? document([]) : document('h2[id], h3[id]') + headings.each((_, heading) => { + const title = document(heading).text().trim() + if (title.toLowerCase() === 'table of contents') return + const section = { + title, + url: `${url}#${encodeURIComponent(document(heading).attr('id') ?? '')}`, + sections: [], + } + if (heading.tagName === 'h3' && parent) { + parent.sections.push(section) + } else { + sections.push(section) + } + if (heading.tagName === 'h2') parent = section + }) + return { + entry: { title: document('h1').first().text().trim() || page.vars.title || url, url, sections, ...(page.vars.docsGroup ? { group: page.vars.docsGroup } : {}), - } - })) + }, + order: page.vars.docsOrder ?? Infinity, + parent: page.vars.docsParent, + } +} +/** + * Regenerate public ordering and nesting without reading or rendering sources. + * @param {Iterable} records + * @returns {NavigationEntry[]} + */ +export function projectDocsNavigation (records) { + const docs = [...records].sort((a, b) => { + const group = (a.entry.group ?? '').localeCompare(b.entry.group ?? '') + if (group) return group + const order = a.order - b.order + return order || a.entry.url.localeCompare(b.entry.url) + }) + // Nest only fresh entries: attaching child pages to cached heading arrays would + // duplicate children on the next build and keep deleted/reparented docs alive. + const entries = docs.map(record => structuredClone(record.entry)) const entriesByUrl = new Map(entries.map(entry => [entry.url, entry])) - const parentsByUrl = new Map(docs.map(page => [page.pageInfo.url, page.vars.docsParent])) + const parentsByUrl = new Map(docs.map(record => [record.entry.url, record.parent])) /** @type {NavigationEntry[]} */ const roots = [] for (const entry of entries) { diff --git a/site/layouts/docs/navigation.test.js b/site/layouts/docs/navigation.test.js index e33c75ce..60ec201c 100644 --- a/site/layouts/docs/navigation.test.js +++ b/site/layouts/docs/navigation.test.js @@ -4,7 +4,7 @@ import { test } from 'node:test' import { load } from 'cheerio' import { render } from 'fragtml' import { renderMd } from '../../../lib/build-pages/page-builders/md/get-md.js' -import { collectDocsNavigation, docsIndex, navigationHref } from './navigation.js' +import { collectDocsNavigation, readDocsNavigationPage, projectDocsNavigation, docsIndex, navigationHref } from './navigation.js' import { documentationContent, navigation } from './docs.layout.js' /** @param {string} url @param {string} markdown @param {DocsPageVars} [vars] @returns {NavigationPage} */ @@ -112,6 +112,37 @@ test('navigation parents must be existing ancestor pages', async () => { } }) +test('navigation projections reuse cloneable records without mutating cached headings or nesting', async () => { + let renders = 0 + const pages = [ + page('/docs/guide/', '# Guide\n\n## Overview\n\n### Details'), + page('/docs/guide/child/', '# Child\n\n## Child section', { docsParent: '/docs/guide/' }), + page('/docs/guide/child/grandchild/', '# Grandchild', { docsParent: '/docs/guide/child/' }), + ] + for (const source of pages) { + const render = source.renderInnerPage + source.renderInnerPage = (...args) => { renders++; return render(...args) } + } + const records = (await Promise.all(pages.map(readDocsNavigationPage))).filter(record => record !== undefined) + const retained = structuredClone(records) + const expected = projectDocsNavigation(retained) + for (let i = 0; i < 3; i++) { + assert.deepEqual(projectDocsNavigation(retained), expected) + assert.deepEqual(retained, records, 'projection must not attach child pages to retained heading arrays') + } + const changed = projectDocsNavigation(retained) + assert.ok(changed[0]?.sections[0]) + changed[0].sections[0].title = 'Consumer mutation' + assert.deepEqual(projectDocsNavigation(retained), expected, 'public entries do not alias retained records') + assert.equal(renders, 3, 'only initial record extraction renders source pages') + + const survivors = retained.filter(record => record.entry.url !== '/docs/guide/child/grandchild/') + assert.deepEqual(projectDocsNavigation(survivors), await collectDocsNavigation(pages.slice(0, 2))) + const orphans = retained.filter(record => record.entry.url !== '/docs/guide/') + assert.throws(() => projectDocsNavigation(orphans), /Invalid documentation parent/) + assert.deepEqual(retained, records, 'even a failed projection leaves the retained records unchanged') +}) + test('navigation links preserve deployment prefixes for directory and flat pages', () => { for (const base of ['', '/domstack']) { for (const from of ['/docs/', '/docs/pages/', '/docs/migrations/v12-migration.html']) { diff --git a/test-cases/generated-pages/index.test.js b/test-cases/generated-pages/index.test.js index 270b293b..9e3cfe01 100644 --- a/test-cases/generated-pages/index.test.js +++ b/test-cases/generated-pages/index.test.js @@ -6,6 +6,7 @@ import * as cheerio from 'cheerio' import { DomStack, testBuild } from '../../index.js' import globalData from './src/global.data.js' import { DomStackDataError } from '../../lib/helpers/domstack-error.js' +import { startWatch } from '../watch/helpers.js' const __dirname = import.meta.dirname const fixturePrefix = '.tmp-' @@ -594,7 +595,7 @@ export default function indexesPages ({ data }) { } }) - test('rebuilds declared subscribers when a global-data key changes', { timeout: 15_000 }, async () => { + test('rebuilds declared subscribers when a global-data key changes', { timeout: 15_000 }, async t => { await withTempFixture({ 'root.layout.js': minimalRootLayout, 'global.vars.js': minimalGlobalVars, @@ -630,14 +631,19 @@ export default function unrelatedPages () { `, }, async ({ src, dest }) => { const domstack = new DomStack(src, dest) + const factoryRuns = join(src, '../unrelated-factory-runs') try { - await domstack.watch({ serve: false }) + await startWatch(t, domstack, src, { + serve: false, + async onInitialBuild () { + assert.equal(await readFile(factoryRuns, 'utf8'), 'run\n', 'the unrelated factory ran during the initial build') + }, + }) const initialOutputPath = join(dest, 'watch-first/index.html') const updatedOutputPath = join(dest, 'watch-updated/index.html') assert.match(await readFile(initialOutputPath, 'utf8'), /First title/) assert.equal(await readFile(join(dest, 'summary'), 'utf8'), 'watch-first/index.html:First title') - const factoryRuns = join(src, '../unrelated-factory-runs') - assert.equal(await readFile(factoryRuns, 'utf8'), 'run\n', 'the unrelated factory ran during the initial build') + const startupFactoryRuns = await readFile(factoryRuns, 'utf8') await writeFile(join(src, 'page.vars.js'), "export default { title: 'Updated title' }\n") await new Promise(resolve => setTimeout(resolve, 800)) @@ -648,14 +654,14 @@ export default function unrelatedPages () { assert.doesNotMatch(updatedOutput, /First title/) assert.equal(await readFile(join(dest, 'summary'), 'utf8'), 'watch-updated/index.html:Updated title') await assert.rejects(() => stat(initialOutputPath), { code: 'ENOENT' }, 'obsolete dependency-driven output is removed') - assert.equal(await readFile(factoryRuns, 'utf8'), 'run\n', 'an unrelated factory is not executed during a subscriber rebuild') + assert.equal(await readFile(factoryRuns, 'utf8'), startupFactoryRuns, 'an unrelated factory is not executed during a subscriber rebuild') } finally { if (domstack.watching) await domstack.stopWatching() } }) }) - test('rebuilds generated pages when Markdown settings change in watch mode', { timeout: 15_000 }, async () => { + test('rebuilds generated pages when Markdown settings change in watch mode', { timeout: 15_000 }, async t => { const markdownSettings = (/** @type {string} */ version) => `export default function (md) { md.renderer.rules.paragraph_open = () => '

' return md @@ -683,7 +689,7 @@ export default function ({ data }) { const outputPath = join(dest, 'summary/index.html') try { - await domstack.watch({ serve: false }) + await startWatch(t, domstack, src) assert.match(await readFile(outputPath, 'utf8'), /data-version="first"/) await writeFile(join(src, 'markdown-it.settings.js'), markdownSettings('second')) @@ -699,7 +705,7 @@ export default function ({ data }) { }) }) - test('rebuilds generated pages when layout assets are added or removed in watch mode', { timeout: 25_000 }, async () => { + test('rebuilds generated pages when layout assets are added or removed in watch mode', { timeout: 25_000 }, async t => { await withTempFixture({ 'root.layout.js': assetAwareRootLayout, 'global.vars.js': minimalGlobalVars, @@ -733,7 +739,7 @@ export default function ({ data }) { } try { - await domstack.watch({ serve: false }) + await startWatch(t, domstack, src) await assertAssetReference('root.layout.css', false) await assertAssetReference('root.layout.client.js', false) @@ -758,7 +764,7 @@ export default function ({ data }) { }) }) - test('removes obsolete regular and generated page outputs in watch mode', { timeout: 20_000 }, async () => { + test('removes obsolete regular and generated page outputs in watch mode', { timeout: 20_000 }, async t => { await withTempFixture({ 'root.layout.js': minimalRootLayout, 'global.vars.js': minimalGlobalVars, @@ -772,7 +778,7 @@ export default function ({ data }) { }, async ({ src, dest }) => { const domstack = new DomStack(src, dest) try { - await domstack.watch({ serve: false }) + await startWatch(t, domstack, src) const oldOutputPath = join(dest, 'old/index.html') const newOutputPath = join(dest, 'new/index.html') const removedOutputPath = join(dest, 'removed/index.html') @@ -812,7 +818,7 @@ export default function ({ data }) { }) }) - test('refreshes pages-file dependency trees in watch mode', { timeout: 15_000 }, async () => { + test('refreshes pages-file dependency trees in watch mode', { timeout: 15_000 }, async t => { await withTempFixture({ 'root.layout.js': minimalRootLayout, 'global.vars.js': minimalGlobalVars, @@ -825,7 +831,7 @@ export default function ({ data }) { }, async ({ src, dest }) => { const domstack = new DomStack(src, dest) try { - await domstack.watch({ serve: false }) + await startWatch(t, domstack, src) const outputPath = join(dest, 'watched/index.html') assert.match(await readFile(outputPath, 'utf8'), /Initial value/) diff --git a/test-cases/generated-pages/streaming.test.js b/test-cases/generated-pages/streaming.test.js index 69e7c611..9646f42a 100644 --- a/test-cases/generated-pages/streaming.test.js +++ b/test-cases/generated-pages/streaming.test.js @@ -11,6 +11,7 @@ import { DomStack } from '../../index.js' import { builder } from '../../lib/builder.js' import { DomStackAggregateError } from '../../lib/helpers/domstack-aggregate-error.js' import { errorText, settle, writeFiles } from '../page-outputs/helpers.js' +import { startWatch } from '../watch/helpers.js' /** @param {TestContext} t @param {Record} files @param {boolean} [buildDrafts] */ async function setup (t, files, buildDrafts = false) { @@ -275,7 +276,7 @@ for (const change of ['recovery', 'deletion', 'empty result']) { 'stream.pages.js': watchFactory(['old', 'stale']), 'sibling.pages.js': watchFactory(['sibling']), }) - await site.watch({ serve: false }) + await startWatch(t, site, src) const sibling = await read('sibling.html') for (const name of ['partial', 'second-partial']) { await settle(site, logs, async () => { @@ -304,7 +305,7 @@ for (const change of ['recovery', 'deletion', 'empty result']) { const { site, src, dest, read, logs } = await setup(t, { 'stream.pages.js': watchFactory(['partial', 'nested/partial'], 'initial stream failure'), }) - const result = await site.watch({ serve: false }) + const result = await startWatch(t, site, src) assert.ok(logs.some(line => JSON.parse(line).msg === 'Build Failed!')) assert.ok(logs.some(line => line.includes('initial stream failure'))) for (const [index, name] of ['partial', 'nested/partial'].entries()) { diff --git a/test-cases/incremental-global-data/helpers.js b/test-cases/incremental-global-data/helpers.js new file mode 100644 index 00000000..a8857118 --- /dev/null +++ b/test-cases/incremental-global-data/helpers.js @@ -0,0 +1,321 @@ +/** + * @import { TestContext } from 'node:test' + * @import { FSWatcher } from 'chokidar' + + * @import { Results, DomStackOpts } from '../../lib/builder.js' + * @typedef {{ type: 'added' | 'removed' | 'change', filepath: string }} InputEvent + * @typedef {object} ProducerCall + * @property {'reset' | 'delta'} kind + * @property {string} [reason] + * @property {InputEvent[]} events + * @property {string[]} pages + * @property {string[]} upserted + * @property {string[]} removed + * @property {string[]} rendered + * @property {string[] | null} previousKeys + * @property {boolean} previousIsMap + * @property {number} invocation + * @property {number} threadId + * @property {{ navigation: { url: string, title: string, tag: string | null, site: string, section: string }[], search: { url: string, html: string }[] }} publicData + */ +import assert from 'node:assert/strict' +import { EventEmitter } from 'node:events' + +import { mkdir, mkdtemp, readFile, readdir, rm, writeFile } from 'node:fs/promises' +import { dirname, join } from 'node:path' +import { setImmediate as nextTurn, setTimeout as delay } from 'node:timers/promises' +import chokidar from 'chokidar' +import pino from 'pino' +import { DomStack } from '../../index.js' + +/** @param {string} heading @param {string} [body] @param {string} [frontmatter] */ +export function article (heading, body = 'Original body.', frontmatter = '') { + return `---\narticle: true\nlayout: article\n${frontmatter}---\n# ${heading}\n\n${body}\n` +} + +/** @param {() => Promise} predicate @param {string} message */ +export async function waitFor (predicate, message) { + const deadline = Date.now() + 8000 + while (!(await predicate())) { + assert.ok(Date.now() < deadline, message) + await delay(20) + } +} + +/** + * Only source event delivery is mocked; discovery, dependency analysis, workers, + * rendering, writes, cleanup, esbuild and copy watchers use the real implementation. + * @param {TestContext} t + * @param {{ native?: boolean, files?: Record, opts?: DomStackOpts }} [options] + */ +export async function fixture (t, { native = false, files = {}, opts = {} } = {}) { + const root = await mkdtemp(join(import.meta.dirname, '.tmp-')) + const src = join(root, 'src') + const dest = join(root, 'public') + const log = join(root, 'producer.jsonl') + const gate = join(root, 'hold-producer') + const producerFailure = join(root, 'fail-producer') + const renderFailure = join(root, 'fail-render') + const logger = pino({ level: 'silent' }) + const buildOptions = { metafile: false, domstackManifest: false, logger, ...opts } + const dom = new DomStack(src, dest, buildOptions) + /** @type {FSWatcher | undefined} */ + let watcher + /** @type {Parameters[1]} */ + let watcherOptions + let nativeEventCount = 0 + const watch = chokidar.watch + /** @param {Parameters} args */ + const sourceWatch = (...args) => { + if (args[0] !== src) return watch(...args) + watcherOptions = args[1] + if (native) { + watcher = watch(...args) + watcher.on('all', () => { nativeEventCount++ }) + } else { + const events = new EventEmitter() + const fake = Object.assign(events, { + closed: false, + async close () { fake.closed = true }, + }) + watcher = /** @type {FSWatcher} */ (/** @type {unknown} */ (fake)) + setImmediate(() => { if (!fake.closed) fake.emit('ready') }) + } + return watcher + } + t.mock.method(chokidar, 'watch', sourceWatch) + t.after(async () => { + await rm(gate, { force: true }) + try { + if (dom.watching) await dom.stopWatching() + } finally { + await rm(root, { recursive: true, force: true }) + } + }) + + /** @param {string} name @param {string} content */ + const write = async (name, content) => { + const path = join(src, name) + await mkdir(dirname(path), { recursive: true }) + await writeFile(path, content) + } + const producer = ` +import assert from 'node:assert/strict' +import { appendFileSync, existsSync } from 'node:fs' +import { setTimeout as delay } from 'node:timers/promises' +import { threadId } from 'node:worker_threads' +import { prefix } from './producer-middle.js' +let invocation = 0 +export default async ({ pages, previousState, changes, setState }) => { + assert.ok(previousState === undefined || previousState instanceof Map) + const previousKeys = previousState === undefined ? null : [...previousState.keys()].sort() + const state = changes.kind === 'reset' ? new Map() : new Map(previousState) + const changed = changes.kind === 'reset' ? pages : changes.upserted + const rendered = [] + for (const path of changes.removed ?? []) state.delete(path) + for (const page of changed) { + assert.ok(pages.includes(page), 'upserts are current initialized PageData instances') + const path = page.pageInfo.pageFile.filepath + if (!page.vars.article) { state.delete(path); continue } + rendered.push(path) + state.set(path, { + url: page.pageInfo.url, + title: page.vars.title, + tag: page.vars.tag ?? null, + site: page.vars.site, + section: page.vars.section, + html: await page.renderFullPage(), + }) + } + const rows = [...state.values()].sort((a, b) => a.url.localeCompare(b.url)) + const publicData = { + navigation: rows.map(({ html, ...row }) => row), + search: rows.map(row => ({ url: row.url, html: prefix + row.html })), + } + // Capture this attempt's failure before the gate; a buffered retry can then recover. + const fail = existsSync(${JSON.stringify(producerFailure)}) + if (fail) state.set('failed-candidate', {}) + setState(state) + // Both mutations must be isolated from the explicit snapshot crossing workers. + state.set('post-setState-mutation', {}) + previousState?.clear() + appendFileSync(${JSON.stringify(log)}, JSON.stringify({ + kind: changes.kind, reason: changes.reason, + events: changes.events.map(({ type, filepath }) => ({ type, filepath })), + pages: pages.map(page => page.pageInfo.pageFile.filepath).sort(), + upserted: (changes.upserted ?? []).map(page => page.pageInfo.pageFile.filepath).sort(), + removed: [...(changes.removed ?? [])].sort(), rendered: rendered.sort(), + previousKeys, previousIsMap: previousState instanceof Map, + invocation: ++invocation, threadId, publicData, + }) + '\\n') + const deadline = Date.now() + 8000 + while (existsSync(${JSON.stringify(gate)})) { + if (Date.now() > deadline) throw new Error('Timed out waiting for test producer gate') + await delay(10) + } + if (fail) throw new Error('intentional producer failure after setState') + return publicData +} +` + const defaults = { + 'global.vars.js': "import { site } from './vars-middle.js'; export default { layout: 'root', site }\n", + 'vars-middle.js': "import { site } from './vars-leaf.js'; export { site }\n", + 'vars-leaf.js': "export const site = 'Site one'\n", + 'global.data.js': producer, + 'producer-middle.js': "import { prefix } from './producer-leaf.js'; export { prefix }\n", + 'producer-leaf.js': "export const prefix = 'Search one: '\n", + 'markdown-it.settings.js': "import { html } from './markdown-middle.js'; export default md => md.set({ html })\n", + 'markdown-middle.js': "import { html } from './markdown-leaf.js'; export { html }\n", + 'markdown-leaf.js': 'export const html = true\n', + 'root.layout.js': 'export default ({ children }) => children\n', + 'article.layout.js': "import { section } from './layout-middle.js'; export const vars = { section }; export default ({ children }) => '

' + section + children + '
'\n", + 'layout-middle.js': "import { section } from './layout-leaf.js'; export { section }\n", + 'layout-leaf.js': "export const section = 'Section one'\n", + 'a/page.md': article('Alpha'), + 'a/page.vars.js': "import { tag } from '../companion-middle.js'; export default { tag }\n", + 'companion-middle.js': "import { tag } from './companion-leaf.js'; export { tag }\n", + 'companion-leaf.js': "export const tag = 'Tag one'\n", + 'b/page.md': article('Beta'), + 'nav/page.js': "export const vars = { dataDeps: ['navigation'] }; export default ({ data }) => JSON.stringify(data.navigation)\n", + 'data.json.template.js': `import { existsSync } from 'node:fs' +export const dataDeps = ['navigation', 'search'] +export default ({ data }) => { + if (existsSync(${JSON.stringify(renderFailure)})) throw new Error('intentional later render failure') + return JSON.stringify({ navigation: data.navigation, search: data.search }) +}\n`, + 'summary.pages.js': "export const dataDeps = ['navigation']; export default ({ data }) => ({ outputName: 'summary.html', children: JSON.stringify(data.navigation) })\n", + ...files, + } + await Promise.all(Object.entries(defaults).map(([name, content]) => write(name, content))) + + /** @returns {Promise} */ + const calls = async () => { + const text = await readFile(log, 'utf8').catch(error => { + if (error.code === 'ENOENT') return '' + throw error + }) + return text.trim() ? text.trim().split('\n').map(line => JSON.parse(line)) : [] + } + /** @param {InputEvent[]} events */ + const emit = events => { + assert.ok(watcher, 'source watcher must exist before initial discovery/build finishes') + for (const { type, filepath } of events) { + watcher.emit(type === 'added' ? 'add' : type === 'removed' ? 'unlink' : 'change', filepath) + } + } + /** @param {string} name @param {InputEvent['type']} [type] @returns {InputEvent} */ + const event = (name, type = 'change') => ({ type, filepath: join(src, name) }) + /** @param {InputEvent[]} events */ + const rebuild = async events => { + const count = (await calls()).length + emit(events) + await nextTurn() + await dom.settled() + const added = (await calls()).slice(count) + assert.equal(added.length, 1, 'one producer invocation per event batch') + const call = added[0] + assert.ok(call) + return call + } + return { + root, + src, + dest, + dom, + write, + calls, + emit, + event, + rebuild, + gate, + producerFailure, + renderFailure, + async settleNative () { + assert.ok(native, 'quiescence waits are only for native source events') + await waitFor(async () => { + await dom.settled() + const count = nativeEventCount + // Allow delayed native notifications and the watcher's 300ms atomic window. + await delay(500) + await dom.settled() + return nativeEventCount === count + }, 'native source notifications become quiescent') + }, + watcherOptions: () => watcherOptions, + build: () => new DomStack(src, join(root, 'standalone'), buildOptions).build(), + async start () { + const report = await dom.watch({ serve: false }) + + assert.deepEqual(report.pageBuildResults?.errors, [], 'initial page build succeeded') + return report + }, + async matchesFresh () { + const watchedOutputs = await outputSnapshot(dest) + const watchedData = JSON.parse(await readFile(join(dest, 'data.json'), 'utf8')) + const freshDest = join(root, 'fresh') + await rm(freshDest, { recursive: true, force: true }) + const report = await new DomStack(src, freshDest, buildOptions).build() + assert.deepEqual(report.pageBuildResults?.errors, [], 'fresh page build succeeded') + const freshCall = (await calls()).at(-1) + assert.ok(freshCall) + assert.equal(freshCall.kind, 'reset') + assert.equal(freshCall.previousKeys, null) + assert.deepEqual(freshCall.events, []) + assert.deepEqual(watchedData, freshCall.publicData, 'public data matches a from-scratch producer') + assert.deepEqual(watchedOutputs, await outputSnapshot(freshDest), 'all public output names and bytes match a fresh build') + }, + } +} + +/** @param {Results} report */ +export function assertPrivateReport (report) { + assert.deepEqual(report.pageBuildResults?.errors, [], 'page build succeeded') + const pageReport = report.pageBuildResults?.report + assert.ok(pageReport) + for (const key of ['globalDataBaseline', 'previousGlobalDataBaseline', 'globalDataInputChanges', 'previousState']) { + assert.equal(Object.hasOwn(pageReport, key), false, `${key} stays private`) + assert.equal(Object.hasOwn(report, key), false, `${key} is not a public result`) + } +} + +/** @param {string} dir @returns {Promise>} */ +async function outputSnapshot (dir) { + /** @type {Record} */ + const files = {} + for (const entry of await readdir(dir, { withFileTypes: true })) { + const path = join(dir, entry.name) + if (entry.isDirectory()) { + for (const [name, content] of Object.entries(await outputSnapshot(path))) files[join(entry.name, name)] = content + } else { + files[entry.name] = await readFile(path, 'utf8') + } + } + return files +} + +/** + * @param {ProducerCall | undefined} call + * @param {string[]} upserted + * @param {string[]} [removed] + * @returns {asserts call is ProducerCall} + */ +export function assertDelta (call, upserted, removed = []) { + assert.ok(call) + assert.equal(call.kind, 'delta') + assert.equal(call.previousIsMap, true, 'Map state survives the worker boundary') + assert.deepEqual(call.upserted, [...upserted].sort()) + assert.deepEqual(call.removed, [...removed].sort()) + assert.equal(call.invocation, 1, 'each build loads the producer in a fresh worker') + assert.ok(call.previousKeys?.every(key => !key.includes('mutation') && key !== 'failed-candidate')) +} + +/** @param {ProducerCall | undefined} call @returns {asserts call is ProducerCall} */ +export function assertReset (call) { + assert.ok(call) + assert.equal(call.kind, 'reset') + assert.equal(typeof call.reason, 'string') + assert.ok(call.reason) + assert.equal(call.previousKeys, null) + assert.equal(call.previousIsMap, false) + assert.equal(call.invocation, 1) +} diff --git a/test-cases/incremental-global-data/index.test.js b/test-cases/incremental-global-data/index.test.js new file mode 100644 index 00000000..3b9a9f82 --- /dev/null +++ b/test-cases/incremental-global-data/index.test.js @@ -0,0 +1,535 @@ +/** + * @import { InputEvent, ProducerCall } from './helpers.js' + */ +import { test } from 'node:test' +import assert from 'node:assert/strict' +import fsPromises, { readFile, rename, rm, stat, utimes, writeFile } from 'node:fs/promises' +import { syncBuiltinESMExports } from 'node:module' +import { join } from 'node:path' +import { article, assertDelta, assertPrivateReport, assertReset, fixture, waitFor } from './helpers.js' + +const options = { timeout: 30_000 } + +test('standalone builds reset state and public build/watch reports do not expose private candidates', options, async t => { + const site = await fixture(t) + const first = await site.build() + const second = await site.build() + const calls = await site.calls() + assert.equal(calls.length, 2) + for (const call of calls) { + assertReset(call) + assert.deepEqual(call.events, []) + } + assert.deepEqual(calls[0]?.publicData, calls[1]?.publicData) + const watched = await site.start() + await site.matchesFresh() + await t.test('standalone reports', () => { + assertPrivateReport(first) + assertPrivateReport(second) + }) + await t.test('watch report', () => assertPrivateReport(watched)) +}) + +test('body-only edits render one producer input, preserve navigation sibling mtime, and match a fresh build', options, async t => { + const site = await fixture(t) + await site.start() + const initial = (await site.calls())[0] + assertReset(initial) + assert.deepEqual(initial.events, []) + assert.deepEqual(initial.pages, ['a/page.md', 'b/page.md', 'nav/page.js'].map(name => join(site.src, name)).sort(), 'templates and generated pages are not producer inputs') + assert.deepEqual(initial.rendered, ['a/page.md', 'b/page.md'].map(name => join(site.src, name))) + + const untouched = ['nav/index.html', 'b/index.html', 'summary.html'].map(name => join(site.dest, name)) + const sentinel = new Date('2000-01-01T00:00:00Z') + for (const path of untouched) await utimes(path, sentinel, sentinel) + const times = await Promise.all(untouched.map(async path => (await stat(path, { bigint: true })).mtimeNs)) + const event = site.event('a/page.md') + await site.write('a/page.md', article('Alpha', 'Only this body changed.')) + const call = await site.rebuild([event]) + assertDelta(call, [event.filepath]) + assert.notEqual(call.threadId, initial.threadId) + assert.deepEqual(call.previousKeys, initial.rendered) + assert.deepEqual(call.events, [event]) + assert.deepEqual(call.rendered, [event.filepath]) + assert.deepEqual(call.publicData.navigation, initial.publicData.navigation) + assert.notDeepEqual(call.publicData.search, initial.publicData.search) + assert.match(await readFile(join(site.dest, 'a/index.html'), 'utf8'), /Only this body changed/) + assert.deepEqual(await Promise.all(untouched.map(async path => (await stat(path, { bigint: true })).mtimeNs)), times) + await site.matchesFresh() + + // A standalone build on the same source must not replace the live session's state. + await site.write('b/page.md', article('Beta', 'A second edit.')) + const second = await site.rebuild([site.event('b/page.md')]) + assertDelta(second, [join(site.src, 'b/page.md')]) + assert.deepEqual(second.rendered, [join(site.src, 'b/page.md')]) + await site.matchesFresh() +}) + +test('H1, frontmatter title/metadata, companion vars, and transitive page/layout imports update input rows', options, async t => { + const site = await fixture(t, { + files: { + 'code/page.js': "import { content } from '../page-middle.js'; export const vars = { article: true, layout: 'article', title: 'Code' }; export default () => content\n", + 'page-middle.js': "import { content } from './page-leaf.js'; export { content }\n", + 'page-leaf.js': "export const content = 'Code one'\n", + }, + }) + await site.start() + const changes = [ + { name: 'a/page.md', content: article('New H1'), inputs: ['a/page.md'], expected: 'New H1' }, + { name: 'a/page.md', content: article('New H1', 'Body.', 'title: Explicit title\ntag: Frontmatter tag\n'), inputs: ['a/page.md'], expected: 'Explicit title' }, + { name: 'b/page.vars.js', content: "export default { tag: 'New companion' }\n", type: 'added', inputs: ['a/page.md', 'b/page.md', 'code/page.js', 'nav/page.js'], expected: 'New companion' }, + { name: 'b/page.vars.js', content: "export default { tag: 'Edited companion' }\n", inputs: ['b/page.md'], expected: 'Edited companion' }, + { name: 'a/page.md', content: article('New H1', 'Body.', 'title: Explicit title\n'), inputs: ['a/page.md'], expected: 'Tag one' }, + { name: 'companion-leaf.js', content: "export const tag = 'Imported companion'\n", inputs: ['a/page.md'], expected: 'Imported companion' }, + { name: 'page-leaf.js', content: "export const content = 'Code two'\n", inputs: ['code/page.js'], expected: 'Code two' }, + { name: 'layout-leaf.js', content: "export const section = 'Section two'\n", inputs: ['a/page.md', 'b/page.md', 'code/page.js'], expected: 'Section two' }, + ] + for (const change of changes) { + await t.test(change.name + ': ' + change.expected, async () => { + await site.write(change.name, change.content) + const event = site.event(change.name, /** @type {InputEvent['type']} */ (change.type ?? 'change')) + const call = await site.rebuild([event]) + assertDelta(call, change.inputs.map(name => join(site.src, name))) + assert.deepEqual(call.events, [event]) + assert.match(JSON.stringify(call.publicData), new RegExp(change.expected)) + await site.matchesFresh() + }) + } +}) + +test('barrel re-export leaves upsert source pages shared with templates and reset producer inputs', options, async t => { + const site = await fixture(t, { + files: { + 'code/page.js': "import { content } from '../page-barrel.js'; export const vars = { article: true, layout: 'article', title: 'Barrel page' }; export default () => content\n", + 'page-barrel.js': "export * from './page-named.js'\n", + 'page-named.js': "export { content } from './page-leaf.js'\n", + 'page-leaf.js': "export const content = 'Barrel one'\n", + 'shared.txt.template.js': "import { content } from './page-leaf.js'; export default () => content\n", + 'producer-middle.js': "export * from './producer-barrel.js'\n", + 'producer-barrel.js': "export { prefix } from './producer-leaf.js'\n", + }, + }) + await site.start() + assert.equal(await readFile(join(site.dest, 'shared.txt'), 'utf8'), 'Barrel one') + await site.write('page-leaf.js', "export const content = 'Barrel two'\n") + const event = site.event('page-leaf.js') + const call = await site.rebuild([event]) + const source = join(site.src, 'code/page.js') + assertDelta(call, [source]) + assert.deepEqual(call.events, [event]) + assert.deepEqual(call.rendered, [source], 'a direct template consumer must not hide the source page reached through barrels') + assert.equal(await readFile(join(site.dest, 'shared.txt'), 'utf8'), 'Barrel two') + assert.match(await readFile(join(site.dest, 'code/index.html'), 'utf8'), /Barrel two/) + assert.ok(call.publicData.search.some(row => row.html.includes('Barrel two'))) + await site.matchesFresh() + + await site.write('producer-leaf.js', "export const prefix = 'Barrel search: '\n") + const producerEvent = site.event('producer-leaf.js') + const reset = await site.rebuild([producerEvent]) + assertReset(reset) + assert.equal(reset.reason, 'global-data-changed', 'producer barrels are recognized dependencies, not unknown-event fallbacks') + assert.deepEqual(reset.events, [producerEvent]) + assert.deepEqual(reset.rendered, ['a/page.md', 'b/page.md', 'code/page.js'].map(name => join(site.src, name))) + assert.ok(reset.publicData.search.every(row => row.html.startsWith('Barrel search: '))) + await site.matchesFresh() +}) + +test('global vars, markdown settings, and producer roots and transitive imports reset the producer', options, async t => { + const site = await fixture(t, { files: { 'a/page.md': article('Alpha', 'Raw HTML') } }) + await site.start() + const changes = [ + ['vars-leaf.js', "export const site = 'Site two'\n", 'Site two'], + ['global.vars.js', "import { site } from './vars-middle.js'; export default { layout: 'root', site: site + ' direct' }\n", 'Site two direct'], + ['markdown-leaf.js', 'export const html = false\n', '<strong>'], + ['markdown-it.settings.js', "import { html } from './markdown-middle.js'; export default md => md.set({ html: !html })\n", ''], + ['producer-leaf.js', "export const prefix = 'Search two: '\n", 'Search two:'], + ['global.data.js', null, 'Search two:'], + ] + for (const [name, content, expected] of changes) { + assert.ok(name && expected) + await t.test(name, async () => { + await site.write(name, content ?? (await readFile(join(site.src, name), 'utf8')) + '\n// Edited producer entry.\n') + const event = site.event(name) + const call = await site.rebuild([event]) + assertReset(call) + assert.deepEqual(call.events, [event]) + assert.deepEqual(call.rendered, ['a/page.md', 'b/page.md'].map(name => join(site.src, name))) + assert.ok(JSON.stringify(call.publicData).includes(expected)) + await site.matchesFresh() + }) + } +}) + +test('adds, deletes, renames, and draft eligibility reconcile absolute source identities and stale outputs', options, async t => { + for (const buildDrafts of [false, true]) { + await t.test(`buildDrafts: ${buildDrafts}`, async t => { + const site = await fixture(t, { opts: { buildDrafts } }) + await site.start() + await site.write('c/page.md', article('Gamma')) + const added = site.event('c/page.md', 'added') + const call = await site.rebuild([added]) + assertDelta(call, [added.filepath]) + assert.deepEqual(call.events, [added]) + await site.matchesFresh() + + for (const [from, to] of /** @type {const} */ ([ + ['c/page.md', 'c/renamed.md'], + ['c/renamed.md', 'c/renamed.draft.md'], + ['c/renamed.draft.md', 'c/page.md'], + ])) { + await rename(join(site.src, from), join(site.src, to)) + const events = [site.event(from, 'removed'), site.event(to, 'added')] + const call = await site.rebuild(events) + const wasEligible = buildDrafts || !from.includes('.draft.') + const eligible = buildDrafts || !to.includes('.draft.') + assertDelta(call, eligible ? [join(site.src, to)] : [], wasEligible ? [join(site.src, from)] : []) + assert.deepEqual(call.events, events) + assert.equal(call.pages.includes(join(site.src, to)), eligible) + assert.equal(call.publicData.navigation.some(row => row.title === 'Gamma'), eligible) + await site.matchesFresh() + } + await rm(join(site.src, 'c/page.md')) + const removed = site.event('c/page.md', 'removed') + const deleted = await site.rebuild([removed]) + assertDelta(deleted, [], [removed.filepath]) + assert.deepEqual(deleted.rendered, []) + assert.deepEqual(deleted.events, [removed]) + await assert.rejects(stat(join(site.dest, 'c/index.html')), { code: 'ENOENT' }) + await site.matchesFresh() + }) + } +}) + +test('one batch unions direct and transitive inputs, deduplicates upserts, and retains event order', options, async t => { + const site = await fixture(t) + await site.start() + await site.write('a/page.md', article('Alpha', 'Batch body.')) + await site.write('companion-leaf.js', "export const tag = 'Batch tag'\n") + await site.write('b/page.md', article('Batch Beta')) + const events = [site.event('a/page.md'), site.event('companion-leaf.js'), site.event('a/page.md'), site.event('b/page.md')] + const call = await site.rebuild(events) + const inputs = ['a/page.md', 'b/page.md'].map(name => join(site.src, name)) + assertDelta(call, inputs) + assert.deepEqual(call.rendered, inputs) + assert.deepEqual(call.events, events) + await site.matchesFresh() + + await site.write('a/page.md', article('Reset batch')) + await site.write('producer-leaf.js', "export const prefix = 'Reset batch: '\n") + const resetEvents = [site.event('a/page.md'), site.event('producer-leaf.js')] + const reset = await site.rebuild(resetEvents) + assertReset(reset) + assert.deepEqual(reset.events, resetEvents) + await site.matchesFresh() +}) + +test('events arriving during an active producer are buffered into the next unioned batch', options, async t => { + const site = await fixture(t) + await site.start() + await writeFile(site.gate, '') + await site.write('a/page.md', article('Alpha', 'Active build body.')) + site.emit([site.event('a/page.md')]) + try { + await waitFor(async () => (await site.calls()).length === 2, 'active producer reaches its gate') + await site.write('b/page.md', article('Buffered Beta')) + await site.write('companion-leaf.js', "export const tag = 'Buffered tag'\n") + const events = [site.event('b/page.md'), site.event('companion-leaf.js'), site.event('b/page.md')] + site.emit(events) + await rm(site.gate) + await site.dom.settled() + const calls = await site.calls() + assert.equal(calls.length, 3, 'initial, active, and one buffered build') + assertDelta(calls[1], [join(site.src, 'a/page.md')]) + assertDelta(calls[2], ['a/page.md', 'b/page.md'].map(name => join(site.src, name))) + assert.deepEqual(calls[2].events, events) + await site.matchesFresh() + } finally { + await rm(site.gate, { force: true }) + } +}) + +test('watch registers before initial producer work and buffers startup changes and membership events', options, async t => { + const site = await fixture(t) + await writeFile(site.gate, '') + const starting = site.start() + try { + await waitFor(async () => (await site.calls()).length === 1, 'initial producer reaches its gate') + assert.equal(site.watcherOptions()?.ignoreInitial, true) + await site.write('a/page.md', article('Startup Alpha')) + await site.write('c/page.md', article('Startup Gamma')) + const events = [site.event('a/page.md'), site.event('c/page.md', 'added')] + site.emit(events) + await rm(site.gate) + await starting + await site.dom.settled() + const calls = await site.calls() + assert.equal(calls.length, 2, 'initial build followed by one buffered startup batch') + assertReset(calls[0]) + assertDelta(calls[1], ['a/page.md', 'c/page.md'].map(name => join(site.src, name))) + assert.deepEqual(calls[1].events, events) + await site.matchesFresh() + } finally { + await rm(site.gate, { force: true }) + await starting + } +}) + +test('setState from a failed producer or later render is not reused and recovery resumes deltas', options, async t => { + for (const phase of ['producer', 'render']) { + await t.test(phase, async t => { + const site = await fixture(t) + await site.start() + const before = await readFile(join(site.dest, 'data.json'), 'utf8') + const marker = phase === 'producer' ? site.producerFailure : site.renderFailure + await writeFile(marker, '') + await site.write('a/page.md', article('Alpha', 'Failed build body.')) + const failed = await site.rebuild([site.event('a/page.md')]) + assertDelta(failed, [join(site.src, 'a/page.md')]) + assert.equal(await readFile(join(site.dest, 'data.json'), 'utf8'), before, 'failed phase does not publish the new public-data template') + + await rm(marker) + await site.write('b/page.md', article('Recovered Beta')) + const event = site.event('b/page.md') + const recovered = await site.rebuild([event]) + assertReset(recovered) + assert.deepEqual(recovered.events, [event]) + assert.ok(JSON.stringify(recovered.publicData).includes('Failed build body.'), 'recovery recomputes inputs changed in the failed build too') + await site.matchesFresh() + + await site.write('a/page.md', article('Alpha', 'Successful delta after recovery.')) + const delta = await site.rebuild([site.event('a/page.md')]) + assertDelta(delta, [join(site.src, 'a/page.md')]) + assert.deepEqual(delta.rendered, [join(site.src, 'a/page.md')]) + await site.matchesFresh() + }) + } +}) + +test('cleanup failure discards producer state but retains new sidecar ownership for recovery', options, async t => { + const site = await fixture(t, { + files: { + 'a/page.md': article('Initial Alpha', 'Initial body.', 'sidecar: initial.txt\n'), + 'a/page.vars.js': `export default {} +export const pageOutputs = ({ vars }) => ({ outputName: vars.sidecar, content: vars.title })\n`, + }, + }) + await site.start() + const stale = join(site.dest, 'a/initial.txt') + const candidate = join(site.dest, 'a/candidate.txt') + assert.equal(await readFile(stale, 'utf8'), 'Initial Alpha') + const originalRm = fsPromises.rm + let cleanupAttempts = 0 + /** @param {Parameters} args */ + const failFirstCleanup = async (...args) => { + if (String(args[0]) === stale && cleanupAttempts++ === 0) { + throw Object.assign(new Error('intentional stale sidecar cleanup failure'), { code: 'EACCES' }) + } + return originalRm(...args) + } + const cleanup = t.mock.method(fsPromises, 'rm', failFirstCleanup) + syncBuiltinESMExports() + try { + await site.write('a/page.md', article('Candidate Alpha', 'Candidate body.', 'sidecar: candidate.txt\n')) + const failed = await site.rebuild([site.event('a/page.md')]) + assertDelta(failed, [join(site.src, 'a/page.md')]) + assert.equal(cleanupAttempts, 1, 'failure occurs in stale-output cleanup, after rendering') + assert.equal(await readFile(stale, 'utf8'), 'Initial Alpha') + assert.equal(await readFile(candidate, 'utf8'), 'Candidate Alpha', 'new page-owned output was already written') + assert.deepEqual(JSON.parse(await readFile(join(site.dest, 'data.json'), 'utf8')), failed.publicData) + + // Only B emits an event: failure recovery must rediscover A's final sidecar too. + await site.write('a/page.md', article('Recovered Alpha', 'Recovered body.', 'sidecar: recovered.txt\n')) + await site.write('b/page.md', article('Recovery Beta')) + const event = site.event('b/page.md') + const recovered = await site.rebuild([event]) + assertReset(recovered) + assert.deepEqual(recovered.events, [event]) + assert.equal(cleanupAttempts, 2) + assert.equal(await readFile(join(site.dest, 'a/recovered.txt'), 'utf8'), 'Recovered Alpha') + await assert.rejects(stat(stale), { code: 'ENOENT' }) + await assert.rejects(stat(candidate), { code: 'ENOENT' }, 'sidecar written before cleanup failed must not be orphaned') + await site.matchesFresh() + } finally { + cleanup.mock.restore() + syncBuiltinESMExports() + } +}) + +test('events buffered during a failing producer batch automatically recover without retaining its candidate', options, async t => { + const site = await fixture(t) + await site.start() + await writeFile(site.producerFailure, '') + await writeFile(site.gate, '') + await site.write('a/page.md', article('Alpha', 'Body from the failing batch.')) + const activeEvent = site.event('a/page.md') + site.emit([activeEvent]) + try { + await waitFor(async () => (await site.calls()).length === 2, 'failing producer snapshots its candidate and reaches the gate') + await site.write('b/page.md', article('Buffered recovery Beta')) + await site.write('companion-leaf.js', "export const tag = 'Buffered recovery tag'\n") + const events = [site.event('b/page.md'), site.event('companion-leaf.js')] + site.emit(events) + // The active worker captured failure already; the next worker must succeed. + await rm(site.producerFailure) + await rm(site.gate) + await site.dom.settled() + const calls = await site.calls() + assert.equal(calls.length, 3, 'initial, failed, and one buffered recovery build') + assertDelta(calls[1], [activeEvent.filepath]) + assert.deepEqual(calls[1].events, [activeEvent]) + assertReset(calls[2]) + assert.deepEqual(calls[2].events, events) + assert.deepEqual(calls[2].rendered, ['a/page.md', 'b/page.md'].map(name => join(site.src, name))) + assert.ok(JSON.stringify(calls[2].publicData).includes('Body from the failing batch.')) + assert.ok(JSON.stringify(calls[2].publicData).includes('Buffered recovery tag')) + await site.matchesFresh() + + await site.write('b/page.md', article('Beta after buffered recovery')) + const delta = await site.rebuild([site.event('b/page.md')]) + assertDelta(delta, [join(site.src, 'b/page.md')]) + assert.deepEqual(delta.rendered, [join(site.src, 'b/page.md')]) + await site.matchesFresh() + } finally { + await rm(site.producerFailure, { force: true }) + await rm(site.gate, { force: true }) + } +}) + +test('independent sessions and a stopped/restarted instance never inherit each other\'s Map state', options, async t => { + const first = await fixture(t) + const second = await fixture(t, { files: { 'a/page.md': article('Other session') } }) + await first.start() + await first.write('a/page.md', article('First session edited')) + await first.rebuild([first.event('a/page.md')]) + await second.start() + assertReset((await second.calls())[0]) + await second.write('b/page.md', article('Other Beta')) + const other = await second.rebuild([second.event('b/page.md')]) + assertDelta(other, [join(second.src, 'b/page.md')]) + assert.ok(other.previousKeys?.every(path => path.startsWith(second.src))) + assert.equal((await first.calls()).length, 2) + await second.matchesFresh() + + await first.dom.stopWatching() + await first.write('b/page.md', article('Edited while stopped')) + await first.start() + const restarted = (await first.calls()).at(-1) + assert.ok(restarted) + assertReset(restarted) + assert.deepEqual(restarted.events, []) + await first.write('a/page.md', article('Restart delta')) + const delta = await first.rebuild([first.event('a/page.md')]) + assertDelta(delta, [join(first.src, 'a/page.md')]) + await first.matchesFresh() +}) + +test('native static JSON leaf edits reset producer, global-vars, and markdown-settings inputs', options, async t => { + const site = await fixture(t, { + native: true, + files: { + 'a/page.md': article('Alpha', 'Raw JSON HTML'), + 'producer-leaf.js': "import settings from './producer.json' with { type: 'json' }; export const prefix = settings.prefix\n", + 'producer.json': JSON.stringify({ prefix: 'JSON search one: ' }), + 'vars-leaf.js': "import settings from './vars.json' with { type: 'json' }; export const site = settings.site\n", + 'vars.json': JSON.stringify({ site: 'JSON site one' }), + 'markdown-leaf.js': "import settings from './markdown.json' with { type: 'json' }; export const html = settings.html\n", + 'markdown.json': JSON.stringify({ html: true }), + }, + }) + await site.start() + const changes = [ + { name: 'producer.json', content: { prefix: 'JSON search two: ' }, reason: 'global-data-changed', expected: 'JSON search two: ' }, + { name: 'vars.json', content: { site: 'JSON site two' }, reason: 'global-vars-changed', expected: 'JSON site two' }, + { name: 'markdown.json', content: { html: false }, reason: 'markdown-settings-changed', expected: '<strong>' }, + ] + for (const change of changes) { + await t.test(change.name, async () => { + await site.settleNative() + const beforeEdit = (await site.calls()).length + await site.write(change.name, JSON.stringify(change.content)) + await waitFor(async () => (await site.calls()).length > beforeEdit, `native watcher dispatches ${change.name}`) + await site.settleNative() + const calls = (await site.calls()).slice(beforeEdit) + assertNativeEvents(calls, site.event(change.name)) + for (const call of calls) { + assertReset(call) + assert.equal(call.reason, change.reason) + assert.deepEqual(call.rendered, ['a/page.md', 'b/page.md'].map(name => join(site.src, name))) + assert.ok(JSON.stringify(call.publicData).includes(change.expected)) + } + await site.matchesFresh() + }) + } +}) + +test('native atomic replacement updates one producer input and keeps watching the replacement', options, async t => { + const site = await fixture(t, { native: true }) + await site.start() + await site.settleNative() + const source = join(site.src, 'a/page.md') + const oldInode = (await stat(source)).ino + const replacement = join(site.root, 'replacement.md') + await writeFile(replacement, article('Atomic Alpha', 'Atomically replaced body.')) + const beforeEdit = (await site.calls()).length + await rename(replacement, source) + assert.notEqual((await stat(source)).ino, oldInode, 'replacement uses a new inode, not an in-place write') + await waitFor(async () => (await site.calls()).length > beforeEdit, 'native watcher dispatches the atomic replacement') + await site.settleNative() + const calls = (await site.calls()).slice(beforeEdit) + assertNativeEvents(calls, site.event('a/page.md')) + for (const replaced of calls) { + assertDelta(replaced, [source]) + assert.deepEqual(replaced.rendered, [source]) + } + assert.match(await readFile(join(site.dest, 'a/index.html'), 'utf8'), /Atomically replaced body/) + await site.matchesFresh() + + await site.settleNative() + const beforeNextEdit = (await site.calls()).length + await site.write('a/page.md', article('Atomic Alpha', 'Edited replacement body.')) + await waitFor(async () => (await site.calls()).length > beforeNextEdit, 'native watcher follows the replacement inode') + await site.settleNative() + const nextCalls = (await site.calls()).slice(beforeNextEdit) + assertNativeEvents(nextCalls, site.event('a/page.md')) + for (const edited of nextCalls) { + assertDelta(edited, [source]) + assert.deepEqual(edited.rendered, [source]) + } + assert.match(await readFile(join(site.dest, 'a/index.html'), 'utf8'), /Edited replacement body/) + await site.matchesFresh() +}) + +test('native chokidar body edits reach the same incremental public API', options, async t => { + const site = await fixture(t, { native: true }) + await site.start() + // Native startup may conservatively report unchanged files; measure only the edit. + await site.settleNative() + const beforeEdit = (await site.calls()).length + await site.write('a/page.md', article('Alpha', 'Native watcher body edit.')) + await waitFor(async () => (await site.calls()).length > beforeEdit, 'native watcher dispatches the source edit') + await site.settleNative() + const calls = (await site.calls()).slice(beforeEdit) + assertNativeEvents(calls, site.event('a/page.md')) + for (const edited of calls) { + assertDelta(edited, [join(site.src, 'a/page.md')]) + assert.deepEqual(edited.rendered, [join(site.src, 'a/page.md')]) + } + await site.matchesFresh() +}) + +/** @param {ProducerCall[]} calls @param {InputEvent} expectedEvent */ +function assertNativeEvents (calls, expectedEvent) { + // Native delivery is not exactly-once: an edit can repeat within or across batches. + // Deterministic tests retain exact invocation and input-render counts per batch. + const details = JSON.stringify({ + expectedEvent, + calls: calls.map(({ kind, reason, events, upserted, removed, rendered }) => ({ kind, reason, events, upserted, removed, rendered })), + }, null, 2) + assert.ok(calls.length > 0, `Expected at least one native edit invocation: ${details}`) + for (const call of calls) { + assert.ok(call.events.length > 0, `Native edit invocation has no events: ${details}`) + for (const event of call.events) { + assert.deepEqual(event, expectedEvent, `Unexpected native event or source path: ${details}`) + } + } +} diff --git a/test-cases/nested-layouts/index.test.js b/test-cases/nested-layouts/index.test.js index 5a595457..3424acfc 100644 --- a/test-cases/nested-layouts/index.test.js +++ b/test-cases/nested-layouts/index.test.js @@ -5,6 +5,7 @@ import { mkdtemp, mkdir, writeFile, readFile, rm, stat, unlink } from 'node:fs/p import { dirname, join } from 'node:path' import pino from 'pino' import { DomStack } from '../../index.js' +import { startWatch } from '../watch/helpers.js' const rootLayout = ` import { label } from './label.js' @@ -97,7 +98,7 @@ test('nested layouts render source and generated pages, cascade vars and preserv test('watch follows ancestor edits, imports, reparenting, and asset membership', { timeout: 60_000 }, async t => { const { domstack, read, write, dest, src } = await setup(t) - await domstack.watch({ serve: false }) + await startWatch(t, domstack, src) const unrelatedTime = (await stat(join(dest, 'plain/index.html'))).mtimeMs const settle = async () => { await new Promise(resolve => setTimeout(resolve, 800)) @@ -169,7 +170,7 @@ test('manual composition preserves render values, forwarded assets, and imported outputName: 'manual-generated.html', vars: {layout: 'manual', title: 'Generated'}, children: {html: '

Generated content

'} }`) - await domstack.watch({ serve: false }) + await startWatch(t, domstack, src) const unrelatedTime = (await stat(join(dest, 'plain/index.html'))).mtimeMs assert.match(await read('manual/index.html'), /data-vars="manual:manual:Manual page"/) assert.match(await read('manual/index.html'), /
\s*

Manual content<\/p>/) @@ -191,12 +192,12 @@ test('manual composition preserves render values, forwarded assets, and imported }) test('a shared helper rebuilds layouts, pages, templates, and generated owners together', { timeout: 30_000 }, async t => { - const { domstack, write, read } = await setup(t) + const { domstack, src, write, read } = await setup(t) await write('plain/page.vars.js', "import {label} from '../label.js'; export default {label}") await write('other.layout.js', "export default ({children, vars}) => '

'") await write('label.template.js', "import {label} from './label.js'; export default () => label") await write('label.pages.js', "import {label} from './label.js'; export default {outputName:'label.html', children:label}") - await domstack.watch({ serve: false }) + await startWatch(t, domstack, src) await write('label.js', "export const label = 'shared-v2'") await new Promise(resolve => setTimeout(resolve, 800)) await domstack.settled() @@ -206,13 +207,13 @@ test('a shared helper rebuilds layouts, pages, templates, and generated owners t }) test('a browser entry point also rebuilds all of its server-side consumers', { timeout: 30_000 }, async t => { - const { domstack, write, read, dest } = await setup(t) + const { domstack, src, write, read, dest } = await setup(t) await write('global.client.js', "export const label = 'browser-v1'") await write('root.layout.js', rootLayout.replace('./label.js', './global.client.js')) await write('typed/page.ts', "import {label} from '../global.client.js'; export default () => label") await write('label.template.js', "import {label} from './global.client.js'; export default () => label") await write('label.pages.js', "import {label} from './global.client.js'; export default {outputName:'label.html', children:label}") - await domstack.watch({ serve: false }) + await startWatch(t, domstack, src) const unrelatedTime = (await stat(join(dest, 'plain/index.html'))).mtimeMs const affectedOutputs = ['source/index.html', 'typed/index.html', 'archive.html', 'label', 'label.html', 'global.client.js'] for (const output of affectedOutputs) assert.match(await read(output), /browser-v1/) @@ -233,9 +234,9 @@ test('watch recovers from initial layout failures before any routing state exist } for (const [name, layout] of Object.entries(failures)) { await t.test(name, async t => { - const { domstack, write, read, dest } = await setup(t) + const { domstack, src, write, read, dest } = await setup(t) await write('root.layout.js', layout) - const results = await domstack.watch({ serve: false }) + const results = await startWatch(t, domstack, src) assert.ok(results.pageBuildResults?.errors.length, 'startup reports the layout failure without stopping watch') await assert.rejects(read('source/index.html'), { code: 'ENOENT' }, 'the initial failure did not produce the page') if (name === 'nonError') { @@ -294,7 +295,7 @@ test('manual composition forwards declared data and rebuilds source and generate outputName: 'manual-generated.html', vars: { layout: 'manual', dataDeps: ['pageMessage'] }, children: ({ data }) => '

' + data.pageMessage + '

' }`) - await domstack.watch({ serve: false }) + await startWatch(t, domstack, src) const plainTime = (await stat(join(dest, 'plain/index.html'))).mtimeMs let currentData = globalData for (const [before, after, message] of /** @type {const} */ ([ @@ -374,8 +375,8 @@ test('each nested renderer gets only its own subscriptions; global data can rend }) test('watch subscribes outputs to the full layout chain and drops old ancestor subscriptions after reparenting', { timeout: 60_000 }, async t => { - const { domstack, read, write, dest } = await setupSubscriptions(t) - await domstack.watch({ serve: false }) + const { domstack, src, read, write, dest } = await setupSubscriptions(t) + await startWatch(t, domstack, src) const mtime = async (/** @type {string} */ name) => (await stat(join(dest, name))).mtimeMs const settle = async () => { await new Promise(resolve => setTimeout(resolve, 800)) diff --git a/test-cases/page-outputs/cache.test.js b/test-cases/page-outputs/cache.test.js index bd77de57..2adf8213 100644 --- a/test-cases/page-outputs/cache.test.js +++ b/test-cases/page-outputs/cache.test.js @@ -3,6 +3,7 @@ import assert from 'node:assert/strict' import { rm, stat, utimes, writeFile } from 'node:fs/promises' import { join } from 'node:path' import { hook, setup, settle } from './helpers.js' +import { startWatch } from '../watch/helpers.js' // Install the guard inside each build worker; parent-side reads remain available // for assertions, and syncBuiltinESMExports also guards already-imported bindings. @@ -23,7 +24,7 @@ test('watch caches identical hook bytes across workers without rereads and recre 'page.html': 'initial main', 'page.vars.js': companionSource, }) - await site.watch({ serve: false }) + await startWatch(t, site, src) assert.equal(await read('cached.txt'), 'cached bytes') const originalTime = await mtime('cached.txt') for (const content of ['first rebuild', 'second rebuild']) { @@ -75,7 +76,7 @@ test('watch repairs same-size external edits with exactly restored mtime using c 'page.html': 'initial main', 'page.vars.js': 'export default {}; ' + hook('metadata.txt', 'original'), }) - await site.watch({ serve: false }) + await startWatch(t, site, src) const output = join(dest, 'metadata.txt') const original = await stat(output, { bigint: true }) assert.equal(original.mtimeNs, BigInt(timestamp) * 1_000_000_000n) @@ -108,7 +109,7 @@ test('watch retains cached writes before iterator failure through another failed { outputName: 'stale.txt', content: 'keep until recovery' }, ]`, }) - await site.watch({ serve: false }) + await startWatch(t, site, src) const mainTime = await mtime('index.html') const existingTime = await mtime('existing.txt') await settle(site, logs, async () => { @@ -157,7 +158,7 @@ for (const writer of ['template', 'page']) { ? "export default () => ({ outputName: 'shared.html', content: 'initial other writer' })" : 'initial other writer', }) - await site.watch({ serve: false }) + await startWatch(t, site, src) // Add the hook only after the competing output exists, avoiding concurrent // writes to the same destination during the initial build. await settle(site, logs, async () => { diff --git a/test-cases/page-outputs/ownership.test.js b/test-cases/page-outputs/ownership.test.js index 02dcd7a3..7ecdcebf 100644 --- a/test-cases/page-outputs/ownership.test.js +++ b/test-cases/page-outputs/ownership.test.js @@ -3,13 +3,14 @@ import assert from 'node:assert/strict' import { mkdir, readFile, rm, stat, symlink, writeFile } from 'node:fs/promises' import { join } from 'node:path' import { hook, setup, settle } from './helpers.js' +import { startWatch } from '../watch/helpers.js' test('data-invalidated pages replace ownership using actual reports', { timeout: 15_000 }, async t => { const { site, src, dest, read, logs } = await setup(t, { 'global.data.js': "export default { name: 'old.txt' }", 'page.js': "export const vars = { dataDeps: ['name'] }; export default () => 'main'; export const pageOutputs = ({ data }) => ({ outputName: data.name, content: 'sidecar' })", }) - await site.watch({ serve: false }) + await startWatch(t, site, src) await settle(site, logs, async () => { await writeFile(join(src, 'global.data.js'), "export default { name: 'new.txt' }") }) @@ -25,7 +26,7 @@ for (const owner of ['page', 'template']) { ? { 'shared.md': 'other page' } : { 'shared.template.js': "export default () => ({ outputName: 'shared.html', content: 'template' })" }), }) - await site.watch({ serve: false }) + await startWatch(t, site, src) const shared = await read('shared.html') await settle(site, logs, async () => { await writeFile(join(src, 'page.js'), "export default () => 'updated main'") @@ -38,7 +39,7 @@ test('repeated failed watch builds union partial paths with successful ownership const { site, src, dest, read, logs } = await setup(t, { 'page.js': "export default () => 'old main'; " + hook('old.txt'), }) - await site.watch({ serve: false }) + await startWatch(t, site, src) await settle(site, logs, async () => { await writeFile(join(src, 'page.js'), `export default () => 'failed main'; export async function* pageOutputs () { yield { outputName: 'partial.txt', content: 'partial' } @@ -76,7 +77,7 @@ for (const change of ['recovery', 'source deletion', 'hook removal']) { throw Error('initial ownership failure') }`, }) - const result = await site.watch({ serve: false }) + const result = await startWatch(t, site, src) assert.ok(logs.some(line => JSON.parse(line).msg === 'Build Failed!')) assert.ok(logs.some(line => line.includes('initial ownership failure'))) assert.equal(await read('article/partial.txt'), 'partial') @@ -109,7 +110,7 @@ test('stale cleanup does not follow symlink ancestors outside dest', { timeout: const { site, src, dest, logs } = await setup(t, { 'page.js': "export default () => 'main'; " + hook('nested/owned.txt'), }) - await site.watch({ serve: false }) + await startWatch(t, site, src) const outside = join(dest, '..', 'outside') await mkdir(outside) await writeFile(join(outside, 'owned.txt'), 'keep') diff --git a/test-cases/page-outputs/watch.test.js b/test-cases/page-outputs/watch.test.js index aa68ca96..caccdc94 100644 --- a/test-cases/page-outputs/watch.test.js +++ b/test-cases/page-outputs/watch.test.js @@ -1,21 +1,39 @@ +/** + * @import { TestContext } from 'node:test' + */ import { test } from 'node:test' import assert from 'node:assert/strict' import { rename, rm, stat, writeFile } from 'node:fs/promises' import { join } from 'node:path' -import { hook, setup, settle, writeFiles } from './helpers.js' +import { hook, setup as setupSite, settle, writeFiles } from './helpers.js' +import { startWatch } from '../watch/helpers.js' + +/** @param {TestContext} t @param {Record} files */ +async function setup (t, files) { + const fixture = await setupSite(t, files) + + return { + ...fixture, + async startWatch () { + const report = await startWatch(t, fixture.site, fixture.src) + assert.deepEqual(fixture.logs.filter(line => JSON.parse(line).level >= 50), [], 'startup builds must not fail') + return report + }, + } +} const rawLayout = `export const vars = { dataDeps: ['navigation'] } export default ({ children, data }) => data.navigation + children export const pageOutputs = async ({ page }) => ({ outputName: 'source.txt', content: await page.readMarkdownContent() })` test('watch updates only changed raw content and retains unchanged ownership without a public manifest', { timeout: 30_000 }, async t => { - const { site, src, dest, read, mtime, logs } = await setup(t, { + const { site, startWatch, src, dest, read, mtime, logs } = await setup(t, { 'global.data.js': "export default { navigation: 'Navigation one' }", 'root.layout.js': rawLayout, 'a/page.md': '# Article A\n', 'b/page.md': '# Article B\n', }) - await site.watch({ serve: false }) + await startWatch() const siblingRaw = await mtime('b/source.txt') const siblingHtml = await mtime('b/index.html') const originalRaw = await mtime('a/source.txt') @@ -44,8 +62,8 @@ test('watch updates only changed raw content and retains unchanged ownership wit }) test('watch reconciles companion addition, output rename, hook removal, companion removal and re-addition', { timeout: 30_000 }, async t => { - const { site, src, dest, read, logs } = await setup(t, { 'article/page.html': '

Article

' }) - await site.watch({ serve: false }) + const { site, startWatch, src, dest, read, logs } = await setup(t, { 'article/page.html': '

Article

' }) + await startWatch() const companion = join(src, 'article/page.vars.js') await settle(site, logs, async () => { await writeFile(companion, 'export default {}; ' + hook('first.txt', 'first')) @@ -80,12 +98,12 @@ test('watch reconciles companion addition, output rename, hook removal, companio }) test('watch removes sidecars on source rename and draft exclusion', { timeout: 30_000 }, async t => { - const { site, src, dest, read, logs } = await setup(t, { + const { site, startWatch, src, dest, read, logs } = await setup(t, { 'root.layout.js': `export default ({ children }) => children export const pageOutputs = async ({ page }) => ({ outputName: page.outputName + '.txt', content: await page.readMarkdownContent() })`, 'article.md': '# Article\n', }) - await site.watch({ serve: false }) + await startWatch() assert.equal(await read('article.html.txt'), '# Article\n') await settle(site, logs, async () => { await rename(join(src, 'article.md'), join(src, 'renamed.md')) @@ -103,13 +121,13 @@ test('watch removes sidecars on source rename and draft exclusion', { timeout: 3 }) test('watch hook failure retains partial writes and recovery removes old and partial outputs', { timeout: 30_000 }, async t => { - const { site, src, dest, read, mtime, logs } = await setup(t, { + const { site, startWatch, src, dest, read, mtime, logs } = await setup(t, { 'page.js': `export default () => 'old main'; export const pageOutputs = () => [ { outputName: 'old.txt', content: 'old sidecar' }, { outputName: 'stale.txt', content: 'retain until recovery' }, ]`, }) - await site.watch({ serve: false }) + await startWatch() const oldHtmlTime = await mtime('index.html') await settle(site, logs, async () => { await writeFile(join(src, 'page.js'), `export default () => 'failed main'; export async function* pageOutputs () { @@ -136,11 +154,11 @@ test('watch hook failure retains partial writes and recovery removes old and par for (const change of ['source deletion', 'draft exclusion', 'hook removal', 'companion deletion', 'companion rename']) { test(`watch independently cleans up after ${change}`, { timeout: 15_000 }, async t => { - const { site, src, dest, read, logs } = await setup(t, { + const { site, startWatch, src, dest, read, logs } = await setup(t, { 'article/page.html': 'Article', 'article/page.vars.js': 'export default {}; ' + hook('owned.txt'), }) - await site.watch({ serve: false }) + await startWatch() assert.equal(await read('article/owned.txt'), 'sidecar') const page = join(src, 'article/page.html') const companion = join(src, 'article/page.vars.js') @@ -161,11 +179,11 @@ for (const change of ['source deletion', 'draft exclusion', 'hook removal', 'com } test('provider precedence warnings reach the configured logger on initial and incremental watch builds', { timeout: 15_000 }, async t => { - const { site, src, read, logs } = await setup(t, { + const { site, startWatch, src, read, logs } = await setup(t, { 'page.js': "export default () => 'initial'; " + hook('selected.txt', 'page module'), 'page.vars.js': "export default {}; export const pageOutputs = () => { throw Error('ignored companion ran') }", }) - const result = await site.watch({ serve: false }) + const result = await startWatch() assert.equal(result.pageBuildResults?.warnings.filter(warning => 'code' in warning && warning.code === 'DOM_STACK_WARNING_DUPLICATE_PAGE_OUTPUTS_PROVIDER').length, 1) const cursor = logs.length await settle(site, logs, async () => { @@ -184,13 +202,13 @@ test('provider precedence warnings reach the configured logger on initial and in }) test('hook-only data subscriptions invalidate their owner but not an unrelated sibling', { timeout: 30_000 }, async t => { - const { site, src, read, mtime, logs } = await setup(t, { + const { site, startWatch, src, read, mtime, logs } = await setup(t, { 'global.data.js': "export default { selected: 'first', unrelated: 'unchanged' }", 'a/page.html': 'A', 'a/page.vars.js': "export default { dataDeps: ['selected'] }; export const pageOutputs = ({ data }) => ({ outputName: 'data.txt', content: data.selected })", 'b/page.html': 'B', }) - await site.watch({ serve: false }) + await startWatch() const mainTime = await mtime('a/index.html') const siblingTime = await mtime('b/index.html') await settle(site, logs, async () => { diff --git a/test-cases/type-exports/index.test.ts b/test-cases/type-exports/index.test.ts index 60626a65..f096707c 100644 --- a/test-cases/type-exports/index.test.ts +++ b/test-cases/type-exports/index.test.ts @@ -117,6 +117,9 @@ const templateParams: TemplateFunctionParams> = { const globalDataParams: GlobalDataFunctionParams = { pages: [pageData], + previousState: undefined, + changes: { kind: 'reset', reason: 'initial', events: [] }, + setState () {}, } const layoutFunction: LayoutFunction<{ title: string }, string, string> = ({ vars, children }) => `${vars.title}: ${children}` diff --git a/test-cases/watch/helpers.js b/test-cases/watch/helpers.js new file mode 100644 index 00000000..a7984a31 --- /dev/null +++ b/test-cases/watch/helpers.js @@ -0,0 +1,54 @@ +/** + * @import { TestContext } from 'node:test' + * @import { FSWatcher } from 'chokidar' + * @import { DomStack } from '../../index.js' + */ +import assert from 'node:assert/strict' +import { setTimeout as delay } from 'node:timers/promises' +import chokidar from 'chokidar' + +/** + * Start real native watching and drain fixture-creation notifications before + * edits or baseline measurements. This does not suppress events or build errors. + * @param {TestContext} t + * @param {DomStack} site + * @param {string} src + * @param {Parameters[0]} [options] + */ +export async function startWatch (t, site, src, options = { serve: false }) { + let eventCount = 0 + /** @type {FSWatcher | undefined} */ + let sourceWatcher + const recordEvent = () => { eventCount++ } + const watch = chokidar.watch + /** @param {Parameters} args */ + const trackSourceEvents = (...args) => { + const watcher = watch(...args) + if (args[0] === src) { + sourceWatcher = watcher + watcher.on('all', recordEvent) + } + return watcher + } + const trackedWatch = t.mock.method(chokidar, 'watch', trackSourceEvents) + try { + const report = await site.watch(options) + assert.ok(sourceWatcher, 'native source events are observed from watcher creation') + // The source watcher now observes initial build work. Delayed native + // fixture-creation changes can be buffered even with ignoreInitial. + const deadline = performance.now() + 10_000 + while (true) { + await site.settled() + const before = eventCount + // Include Chokidar's 300ms atomic window, not just the current build lock. + await delay(500) + await site.settled() + if (eventCount === before) break + assert.ok(performance.now() < deadline, 'native startup events did not become quiescent') + } + return report + } finally { + sourceWatcher?.off('all', recordEvent) + trackedWatch.mock.restore() + } +} diff --git a/test-cases/watch/index.test.js b/test-cases/watch/index.test.js index 8bb9a897..efc2eb87 100644 --- a/test-cases/watch/index.test.js +++ b/test-cases/watch/index.test.js @@ -8,6 +8,7 @@ import assert from 'node:assert' import { DomStack } from '../../index.js' import { cp, rm, writeFile, readFile, unlink, mkdtemp, stat, readdir, mkdir } from 'fs/promises' import * as path from 'path' +import { startWatch } from './helpers.js' const fixtureDir = path.join(import.meta.dirname, '../general-features/src') @@ -49,7 +50,7 @@ async function setupTempWatch (t, { prefix, files, logger }) { if (domStack.watching) await domStack.stopWatching() await rm(tmp, { recursive: true, force: true }) }) - await domStack.watch({ serve: false }) + await startWatch(t, domStack, src) return { src, dest, domStack } } @@ -276,7 +277,7 @@ test.describe('watch', () => { if (domStack.watching) await domStack.stopWatching() await rm(tmp, { recursive: true, force: true }) }) - await domStack.watch({ serve: false }) + await startWatch(t, domStack, src) const output = path.join(dest, 'index.html') assert.ok(!(await readFile(output, 'utf8')).includes('src="./client.js"')) @@ -314,7 +315,7 @@ test.describe('watch', () => { await rm(tmp, { recursive: true, force: true }) }) - await domStack.watch({ serve: false }) + await startWatch(t, domStack, src) const alphaOutput = path.join(dest, 'generated/alpha.html') const alphaRenamedOutput = path.join(dest, 'generated/alpha-renamed.html') @@ -356,7 +357,7 @@ test.describe('watch', () => { }) // ── Initial build ──────────────────────────────────────────────── - const results = await domStack.watch({ serve: false }) + const results = await startWatch(t, domStack, src) assert.ok(results, 'watch() returned initial build results') assert.ok(results.siteData, 'results include siteData') assert.equal(results.domstackManifest, undefined, 'watch mode does not return a domstack manifest') diff --git a/types.ts b/types.ts index 0827934e..9132d0e8 100644 --- a/types.ts +++ b/types.ts @@ -13,6 +13,12 @@ import type { PageFunction as PageFunctionExport } from './lib/build-pages/page- import type { PageOutputsFunction as PageOutputsFunctionExport } from './lib/build-pages/page-outputs.js' export type { DataDeps } from './lib/build-pages/data-deps.js' +export type { + GlobalDataChanges, + GlobalDataDeltaChanges, + GlobalDataResetChanges, +} from './lib/build-pages/global-data-state.js' +export type { WatchEvent } from './lib/watch-plan.js' export type { PageOutput, PageOutputProvenance, From 3060afa8250a2ebc08e57b9588370285c5f20efe Mon Sep 17 00:00:00 2001 From: Bret Comnes Date: Tue, 15 Sep 2026 09:08:16 -0700 Subject: [PATCH 02/14] Expose relative source IDs for incremental global-data indexes --- docs/data/README.md | 22 +-- .../global-data-state-types.test.ts | 26 +++- lib/build-pages/global-data-state.js | 26 ++-- lib/build-pages/global-data-state.test.js | 133 +++++++++++++++--- lib/build-pages/page-data.js | 11 ++ lib/build-pages/page-data.test.js | 31 +++- site/globals/global.data.ts | 8 +- test-cases/incremental-global-data/helpers.js | 17 +-- .../incremental-global-data/index.test.js | 106 +++++++++----- 9 files changed, 283 insertions(+), 97 deletions(-) diff --git a/docs/data/README.md b/docs/data/README.md index a55ca476..91af30c3 100644 --- a/docs/data/README.md +++ b/docs/data/README.md @@ -87,15 +87,19 @@ The callback receives `{ pages, previousState, changes, setState }`: - `changes` is a discriminated union describing source inputs, independently of output-render filters: - `{ kind: 'reset', reason: string, events: WatchEvent[] }` means rebuild your state from all of `pages`. It has no `upserted` or `removed` fields. - - `{ kind: 'delta', upserted: PageData[], removed: string[], events: WatchEvent[] }` supplies current initialized pages whose inputs were invalidated or which became newly eligible, plus source paths no longer eligible. + - `{ kind: 'delta', upserted: PageData[], removed: string[], events: WatchEvent[] }` supplies current initialized pages whose inputs were invalidated or which became newly eligible, plus source IDs no longer eligible. Upserts are conservative input invalidations, not a guarantee that rendered content changed. -Use `page.pageInfo.pageFile.filepath`, a normalized absolute source path, as the identity for an upsert. -`changes.removed` uses the same normalized absolute paths, not URLs, output filenames, or source-relative directory paths. +Use the read-only `page.sourceId` getter as the identity for an upsert, for example `docs/data/README.md`. +It is `page.pageInfo.pageFile.relname` normalized relative to the source root, with `/` separators on every platform and no checkout-specific prefix. +`changes.removed` contains the same source IDs, not absolute paths, URLs, or output filenames. +IDs distinguish identical filenames in different directories and stay the same when the checkout moves; a source rename is a removal plus an upsert. +The absolute `page.pageInfo.pageFile.filepath` remains available for filesystem operations. Treat an upsert as replacement of that source's cached record and a removal as deletion of that key. If an upsert stops matching your own index filter, delete its cached record too. Both variants expose the existing raw `WatchEvent` objects through `changes.events`, with `type`, `filepath`, `name`, and `convention` fields. +Raw event `filepath` values and internal dependency paths remain absolute; do not use them as keys in a source-ID-keyed index. The event list preserves batch order, duplicates, and removals rather than becoming a deduplicated page-change list; the initial reset has an empty event list. Use `changes.kind`, `upserted`, and `removed` to update an index rather than reconstructing page membership from raw events. @@ -139,16 +143,16 @@ const globalData: AsyncGlobalDataFunction } export function checkGlobalDataStateTypes (pages: PageData[], events: WatchEvent[]) { const reset: GlobalDataResetChanges = { kind: 'reset', reason: 'initial', events } - const delta: GlobalDataDeltaChanges = { kind: 'delta', upserted: pages, removed: ['/removed.md'], events } + const delta: GlobalDataDeltaChanges = { kind: 'delta', upserted: pages, removed: ['posts/removed.md'], events } const changes: GlobalDataChanges = delta const params: GlobalDataFunctionParams = { pages, previousState: undefined, changes, - setState (next) { next.titles.set('/page.md', 'Title') }, + setState (next) { next.titles.set('page.md', 'Title') }, } const stateful: GlobalDataFunction = ({ pages, changes, previousState, setState }) => { const titles = previousState?.titles ?? new Map() @@ -33,8 +33,13 @@ export function checkGlobalDataStateTypes (pages: PageData titles.set(page.pageInfo.pageFile.filepath, page.vars.title)) - changes.removed.forEach(path => titles.delete(path)) + changes.upserted.forEach(page => { + const sourceId: string = page.sourceId + titles.set(sourceId, page.vars.title) + // @ts-expect-error Source IDs are read-only. + page.sourceId = 'other.md' + }) + changes.removed.forEach(sourceId => titles.delete(sourceId)) // @ts-expect-error Delta changes do not have a reset reason. String(changes.reason) } @@ -54,10 +59,17 @@ export function checkGlobalDataStateTypes (pages: PageData { + const filepath: string = event.filepath + upsertedPaths.push(filepath) + }) const options: BuildPagesOptions = { previousGlobalDataBaseline: baseline, globalDataInputChanges: input } const absent: BuildPagesOptions = { previousGlobalDataBaseline: null, globalDataInputChanges: undefined } const report: PageBuilderReport = { pages: [], templates: [], globalDataBaseline: baseline } - return { reset, delta, params, stateful, asyncStateful, legacy, asyncLegacy, defaultState, options, absent, report } + return { reset, delta, params, stateful, asyncStateful, legacy, asyncLegacy, defaultState, options, absent, report, sourceIds, removed, upsertedPaths } } diff --git a/lib/build-pages/global-data-state.js b/lib/build-pages/global-data-state.js index 4729f75f..5143ccfc 100644 --- a/lib/build-pages/global-data-state.js +++ b/lib/build-pages/global-data-state.js @@ -9,22 +9,22 @@ import { resolve } from 'node:path' * Internal candidate committed by the main build only after successful cleanup. * @typedef {object} GlobalDataBaseline * @property {unknown} state - * @property {string[]} sourcePaths + * @property {string[]} sourceIds - Source-root-relative normalized source IDs with POSIX separators. */ /** * Internal input changes supplied by the watch planner, not output filters. * @typedef {object} GlobalDataInputChanges * @property {string | undefined} [resetReason] - * @property {string[]} upsertedPaths - * @property {WatchEvent[]} events + * @property {string[]} upsertedPaths - Normalized absolute source-page candidate paths. + * @property {WatchEvent[]} events - Raw watch events with normalized absolute filepaths. */ /** * @typedef {object} GlobalDataResetChanges * @property {'reset'} kind * @property {string} reason - * @property {WatchEvent[]} events + * @property {WatchEvent[]} events - Raw watch events with normalized absolute filepaths. */ /** @@ -32,9 +32,9 @@ import { resolve } from 'node:path' * @template [U=any] - Source-page render values. * @typedef {object} GlobalDataDeltaChanges * @property {'delta'} kind - * @property {PageData[]} upserted - Current initialized pages, identified by their normalized absolute pageInfo.pageFile.filepath. - * @property {string[]} removed - Normalized absolute source-page paths no longer eligible. - * @property {WatchEvent[]} events + * @property {PageData[]} upserted - Current initialized pages, identified by their sourceId. + * @property {string[]} removed - Source IDs no longer eligible, matching PageData.sourceId: source-root-relative normalized relnames with POSIX separators. + * @property {WatchEvent[]} events - Raw watch events with normalized absolute filepaths, not source IDs. */ /** @@ -98,12 +98,12 @@ function rejectSharedMemory (value, operation, seen = new Set()) { * @param {GlobalDataInputChanges | undefined} [params.globalDataInputChanges] */ export function createGlobalDataState ({ pages, previousGlobalDataBaseline, globalDataInputChanges }) { - const sourcePaths = [...new Set(pages.map(page => resolve(page.pageInfo.pageFile.filepath)))] - const sourcePathSet = new Set(sourcePaths) + const sourceIds = [...new Set(pages.map(page => page.sourceId))] + const sourceIdSet = new Set(sourceIds) const reset = !previousGlobalDataBaseline || globalDataInputChanges?.resetReason !== undefined let state = reset ? undefined : cloneState(previousGlobalDataBaseline.state, 'previous state') const events = structuredClone(globalDataInputChanges?.events ?? []) - const previousPaths = new Set(previousGlobalDataBaseline?.sourcePaths.map(path => resolve(path))) + const previousIds = new Set(previousGlobalDataBaseline?.sourceIds) const upsertedPaths = new Set(globalDataInputChanges?.upsertedPaths.map(path => resolve(path))) /** @type {GlobalDataFunctionParams} */ @@ -116,9 +116,9 @@ export function createGlobalDataState ({ pages, previousGlobalDataBaseline, glob kind: 'delta', upserted: pages.filter(page => { const path = resolve(page.pageInfo.pageFile.filepath) - return upsertedPaths.has(path) || !previousPaths.has(path) + return upsertedPaths.has(path) || !previousIds.has(page.sourceId) }), - removed: [...previousPaths].filter(path => !sourcePathSet.has(path)), + removed: [...previousIds].filter(id => !sourceIdSet.has(id)), events, }, setState (next) { @@ -129,6 +129,6 @@ export function createGlobalDataState ({ pages, previousGlobalDataBaseline, glob return { context, /** @returns {GlobalDataBaseline} */ - getBaseline: () => ({ state, sourcePaths }), + getBaseline: () => ({ state, sourceIds }), } } diff --git a/lib/build-pages/global-data-state.test.js b/lib/build-pages/global-data-state.test.js index 3f2b525d..cd45a0b3 100644 --- a/lib/build-pages/global-data-state.test.js +++ b/lib/build-pages/global-data-state.test.js @@ -6,7 +6,7 @@ import assert from 'node:assert/strict' import { test } from 'node:test' import { mkdtemp, mkdir, writeFile, readFile, rm } from 'node:fs/promises' import { tmpdir } from 'node:os' -import { join, resolve } from 'node:path' +import { join } from 'node:path' import { createGlobalDataState } from './global-data-state.js' import { buildPages, buildPagesDirect } from './index.js' import { identifyPages } from '../identify-pages.js' @@ -29,7 +29,7 @@ async function fixture (t) { test('candidate and previousState are isolated, setState snapshots immediately and last call wins', () => { const state = { count: 1, map: new Map([['a', new Set([1])]]), date: new Date(0) } - const previousGlobalDataBaseline = { state, sourcePaths: [] } + const previousGlobalDataBaseline = { state, sourceIds: [] } const transaction = createGlobalDataState({ pages: [], previousGlobalDataBaseline }) const previous = /** @type {typeof state} */ (transaction.context.previousState) previous.count = 2 @@ -98,13 +98,13 @@ test('setState rejects nested shared memory without replacing or sharing the ret test('previous baselines containing shared memory are rejected instead of exposing aliased state', () => { const buffer = new SharedArrayBuffer(8) for (const state of sharedMemoryStates(buffer)) { - const previousGlobalDataBaseline = { state, sourcePaths: [] } + const previousGlobalDataBaseline = { state, sourceIds: [] } assert.throws(() => createGlobalDataState({ pages: [], previousGlobalDataBaseline }), /global\.data previous state.*SharedArrayBuffer.*non-shared ArrayBuffer/) assert.equal(previousGlobalDataBaseline.state, state) } const reset = createGlobalDataState({ pages: [], - previousGlobalDataBaseline: { state: buffer, sourcePaths: [] }, + previousGlobalDataBaseline: { state: buffer, sourceIds: [] }, globalDataInputChanges: { resetReason: 'recovery', upsertedPaths: [], events: [] }, }) assert.equal(reset.context.previousState, undefined) @@ -120,9 +120,9 @@ test('shared-memory validation inspects only cloned values and does not reread s const unsafe = { get value () { reads++; return new SharedArrayBuffer(8) } } assert.throws(() => transaction.context.setState(unsafe), /SharedArrayBuffer/) assert.equal(reads, 2) - assert.throws(() => createGlobalDataState({ pages: [], previousGlobalDataBaseline: { state: unsafe, sourcePaths: [] } }), /SharedArrayBuffer/) + assert.throws(() => createGlobalDataState({ pages: [], previousGlobalDataBaseline: { state: unsafe, sourceIds: [] } }), /SharedArrayBuffer/) assert.equal(reads, 3) - createGlobalDataState({ pages: [], previousGlobalDataBaseline: { state: safe, sourcePaths: [] } }) + createGlobalDataState({ pages: [], previousGlobalDataBaseline: { state: safe, sourceIds: [] } }) assert.equal(reads, 4, 'isolating both previousState and the candidate only reads the source getter once') class PrivateState { @@ -137,7 +137,7 @@ test('shared-memory validation inspects only cloned values and does not reread s test('non-shared buffers and views remain isolated across previousState and setState', () => { const bytes = new Uint8Array([1, 2]) const state = { bytes, view: new DataView(bytes.buffer) } - const transaction = createGlobalDataState({ pages: [], previousGlobalDataBaseline: { state, sourcePaths: [] } }) + const transaction = createGlobalDataState({ pages: [], previousGlobalDataBaseline: { state, sourceIds: [] } }) const previous = /** @type {typeof state} */ (transaction.context.previousState) previous.view.setUint8(0, 3) assert.equal(previous.bytes[0], 3) @@ -149,7 +149,7 @@ test('non-shared buffers and views remain isolated across previousState and setS }) test('resets discard state even without setState and do not try to clone invalid old state', () => { - for (const previousGlobalDataBaseline of [undefined, null, { state: () => {}, sourcePaths: ['/old.md'] }]) { + for (const previousGlobalDataBaseline of [undefined, null, { state: () => {}, sourceIds: ['old.md'] }]) { const transaction = createGlobalDataState({ pages: [], previousGlobalDataBaseline, @@ -157,7 +157,7 @@ test('resets discard state even without setState and do not try to clone invalid }) assert.equal(transaction.context.previousState, undefined) assert.deepEqual(transaction.context.changes, { kind: 'reset', reason: 'global-data-changed', events: [] }) - assert.deepEqual(transaction.getBaseline(), { state: undefined, sourcePaths: [] }) + assert.deepEqual(transaction.getBaseline(), { state: undefined, sourceIds: [] }) } const initial = createGlobalDataState({ pages: [] }) assert.deepEqual(initial.context.changes, { kind: 'reset', reason: 'initial', events: [] }) @@ -173,10 +173,10 @@ for (const [mode, build] of /** @type {const} */ ([['direct', buildPagesDirect], await writeFile(join(src, 'archive.pages.js'), 'export default () => ({ children: \'Generated\' })') await writeFile(join(src, 'global.data.js'), `export default ({ pages, previousState, changes, setState }) => { const summary = { - pages: pages.map(page => ({ path: page.pageInfo.pageFile.filepath, title: page.vars.title })), + pages: pages.map(page => ({ sourceId: page.sourceId, title: page.vars.title })), previousState, changes: changes.kind === 'reset' ? changes : { - ...changes, upserted: changes.upserted.map(page => page.pageInfo.pageFile.filepath), + ...changes, upserted: changes.upserted.map(page => page.sourceId), }, } const next = { count: (previousState?.count ?? 0) + 1 } @@ -195,9 +195,9 @@ for (const [mode, build] of /** @type {const} */ ([['direct', buildPagesDirect], assert.equal(first.changes.kind, 'reset') assert.equal(first.changes.reason, 'initial') assert.equal(first.previousState, undefined) - assert.deepEqual(first.pages.map((/** @type {{path: string}} */ p) => p.path).sort(), [join(src, 'other.md'), join(src, 'page.js')]) + assert.deepEqual(first.pages.map((/** @type {{sourceId: string}} */ p) => p.sourceId).sort(), ['other.md', 'page.js']) assert.equal(first.pages.find((/** @type {{title: string}} */ p) => p.title === 'Home').title, 'Home') - assert.deepEqual(initial.report.globalDataBaseline, { state: { count: 1 }, sourcePaths: site.pages.map(p => resolve(p.pageFile.filepath)) }) + assert.deepEqual(initial.report.globalDataBaseline, { state: { count: 1 }, sourceIds: site.pages.map(page => page.pageFile.relname) }) assert.ok(initial.report.pages.some(page => page.pagesFilePath === join(src, 'archive.pages.js'))) await rm(join(src, 'other.md')) @@ -224,8 +224,8 @@ for (const [mode, build] of /** @type {const} */ ([['direct', buildPagesDirect], const second = JSON.parse(await readFile(join(dest, 'index.html'), 'utf8')) assert.deepEqual(second.previousState, { count: 1 }) assert.equal(second.changes.kind, 'delta') - assert.deepEqual(second.changes.upserted.sort(), [join(src, 'hidden.draft.md'), join(src, 'new.md'), join(src, 'page.js')]) - assert.deepEqual(second.changes.removed, [join(src, 'other.md')]) + assert.deepEqual(second.changes.upserted.sort(), ['hidden.draft.md', 'new.md', 'page.js']) + assert.deepEqual(second.changes.removed, ['other.md']) assert.deepEqual(second.changes.events, [event]) assert.deepEqual(next.report.globalDataBaseline?.state, { count: 2 }) assert.deepEqual(next.report.pages.map(page => page.sourcePageFilePath), [join(src, 'page.js')], 'changed public data still invalidates its subscriber') @@ -242,6 +242,101 @@ for (const [mode, build] of /** @type {const} */ ([['direct', buildPagesDirect], assert.deepEqual(reset.report.globalDataBaseline?.state, { count: 1 }) }) + test(`${mode}: relocating the source root preserves membership while candidates still match absolute paths`, async t => { + const original = await fixture(t) + const relocated = await fixture(t) + for (const { src } of [original, relocated]) { + await mkdir(join(src, 'posts')) + await writeFile(join(src, 'posts', 'post.md'), 'Post') + await writeFile(join(src, 'global.data.js'), `export default ({ previousState, changes, setState }) => { + setState({ retained: true }) + return { summary: { + previousState, + changes: changes.kind === 'reset' ? changes : { + ...changes, upserted: changes.upserted.map(page => page.sourceId), + }, + } } + }`) + } + const initial = await build(original.src, original.dest, await identifyPages(original.src), { trackWatchDependencies: true }) + assert.deepEqual(initial.errors, []) + const previousGlobalDataBaseline = structuredClone(initial.report.globalDataBaseline) + const site = await identifyPages(relocated.src) + const oldPath = join(original.src, 'posts', 'post.md') + const oldEvent = classifyWatchEvent('change', oldPath) + const next = await build(relocated.src, relocated.dest, site, { + trackWatchDependencies: true, + previousGlobalDataBaseline, + globalDataInputChanges: { upsertedPaths: [oldPath], events: [oldEvent] }, + }) + assert.deepEqual(next.errors, []) + const summary = JSON.parse(await readFile(join(relocated.dest, 'index.html'), 'utf8')) + assert.deepEqual(summary.previousState, { retained: true }) + assert.deepEqual(summary.changes, { kind: 'delta', upserted: [], removed: [], events: JSON.parse(JSON.stringify([oldEvent])) }) + assert.equal(summary.changes.events[0].filepath, oldPath) + assert.deepEqual(next.report.globalDataBaseline, previousGlobalDataBaseline) + + const event = classifyWatchEvent('change', relocated.src + '/posts/./post.md') + const updated = await build(relocated.src, relocated.dest, site, { + trackWatchDependencies: true, + previousGlobalDataBaseline: next.report.globalDataBaseline, + globalDataInputChanges: { upsertedPaths: [relocated.src + '/posts/./post.md'], events: [event] }, + }) + assert.deepEqual(updated.errors, []) + const updateSummary = JSON.parse(await readFile(join(relocated.dest, 'index.html'), 'utf8')) + assert.deepEqual(updateSummary.changes, { kind: 'delta', upserted: ['posts/post.md'], removed: [], events: JSON.parse(JSON.stringify([event])) }) + assert.equal(updateSummary.changes.events[0].filepath, join(relocated.src, 'posts', 'post.md')) + }) + + test(`${mode}: relative source IDs distinguish duplicate basenames and removed IDs delete map entries`, async t => { + const { src, dest } = await fixture(t) + for (const directory of ['a', 'b', 'c']) await mkdir(join(src, directory)) + await writeFile(join(src, 'a', 'post.md'), '---\ntitle: Alpha\n---\nAlpha') + await writeFile(join(src, 'b', 'post.md'), '---\ntitle: Beta\n---\nBeta') + await writeFile(join(src, 'global.data.js'), `export default ({ pages, previousState, changes, setState }) => { + const titles = changes.kind === 'reset' ? new Map() : previousState.titles + const upserted = changes.kind === 'reset' ? pages : changes.upserted + const removed = changes.kind === 'reset' ? [] : changes.removed + for (const sourceId of removed) titles.delete(sourceId) + for (const page of upserted) titles.set(page.sourceId, page.vars.title) + setState({ titles }) + return { summary: { + titles: [...titles].sort(), + upserted: upserted.map(page => page.sourceId).sort(), + removed, + events: changes.events, + } } + }`) + const initial = await build(src, dest, await identifyPages(src), { trackWatchDependencies: true }) + assert.deepEqual(initial.errors, []) + const before = structuredClone(initial.report.globalDataBaseline) + const first = JSON.parse(await readFile(join(dest, 'index.html'), 'utf8')) + assert.deepEqual(first.titles, [['a/post.md', 'Alpha'], ['b/post.md', 'Beta'], ['page.js', 'Home']]) + assert.deepEqual(initial.report.globalDataBaseline?.sourceIds.slice().sort(), ['a/post.md', 'b/post.md', 'page.js']) + + await rm(join(src, 'a', 'post.md')) + await writeFile(join(src, 'c', 'post.md'), '---\ntitle: Gamma\n---\nGamma') + const events = [ + classifyWatchEvent('removed', src + '/a/./post.md'), + classifyWatchEvent('added', src + '/c/./post.md'), + ] + const next = await build(src, dest, await identifyPages(src), { + trackWatchDependencies: true, + previousGlobalDataBaseline: initial.report.globalDataBaseline, + globalDataInputChanges: { upsertedPaths: [], events }, + }) + assert.deepEqual(next.errors, []) + const second = JSON.parse(await readFile(join(dest, 'index.html'), 'utf8')) + assert.deepEqual(second.upserted, ['c/post.md'], 'new membership is upserted without an absolute candidate') + assert.deepEqual(second.removed, ['a/post.md']) + assert.deepEqual(second.titles, [['b/post.md', 'Beta'], ['c/post.md', 'Gamma'], ['page.js', 'Home']]) + assert.deepEqual(second.events, JSON.parse(JSON.stringify(events))) + assert.deepEqual(second.events.map((/** @type {{filepath: string}} */ event) => event.filepath), [join(src, 'a', 'post.md'), join(src, 'c', 'post.md')]) + assert.deepEqual(next.report.globalDataBaseline?.state, { titles: new Map(second.titles) }) + assert.deepEqual(next.report.globalDataBaseline?.sourceIds.slice().sort(), ['b/post.md', 'c/post.md', 'page.js']) + assert.deepEqual(initial.report.globalDataBaseline, before, 'deleting from previousState does not mutate the committed baseline') + }) + test(`${mode}: state does not leak into public data fingerprints or commit previousState mutations`, async t => { const { src, dest } = await fixture(t) await writeFile(join(src, 'global.data.js'), `export default ({ previousState, setState, changes }) => { @@ -293,7 +388,7 @@ for (const [mode, build] of /** @type {const} */ ([['direct', buildPagesDirect], assert.equal(Object.hasOwn(cold.report, 'globalDataBaseline'), false) const watch = await build(src, dest, site, { trackWatchDependencies: true }) assert.deepEqual(watch.errors, []) - assert.deepEqual(watch.report.globalDataBaseline, { state: { candidate: true }, sourcePaths: [join(src, 'page.js')] }) + assert.deepEqual(watch.report.globalDataBaseline, { state: { candidate: true }, sourceIds: ['page.js'] }) }) test(`${mode}: failed page, template, factory, and initialization phases never emit candidate baselines`, async t => { @@ -307,11 +402,11 @@ for (const [mode, build] of /** @type {const} */ ([['direct', buildPagesDirect], assert.ok(filename && content) await writeFile(join(src, filename), content) await writeFile(join(src, 'global.data.js'), 'export default ({ setState }) => { setState({ candidate: true }); return { summary: \'\' } }') - const previousGlobalDataBaseline = { state: { previous: true }, sourcePaths: [] } + const previousGlobalDataBaseline = { state: { previous: true }, sourceIds: [] } const result = await build(src, dest, await identifyPages(src), { previousGlobalDataBaseline, trackWatchDependencies: true }) assert.ok(result.errors.length > 0, filename) assert.equal(Object.hasOwn(result.report, 'globalDataBaseline'), false, filename) - assert.deepEqual(previousGlobalDataBaseline, { state: { previous: true }, sourcePaths: [] }) + assert.deepEqual(previousGlobalDataBaseline, { state: { previous: true }, sourceIds: [] }) } }) @@ -325,7 +420,7 @@ for (const [mode, build] of /** @type {const} */ ([['direct', buildPagesDirect], /** @type {BuildPagesFilterOptions} */ const opts = { trackWatchDependencies: true, - previousGlobalDataBaseline: previous ? { state: new Set([new Uint8Array(new SharedArrayBuffer(8))]), sourcePaths: [] } : undefined, + previousGlobalDataBaseline: previous ? { state: new Set([new Uint8Array(new SharedArrayBuffer(8))]), sourceIds: [] } : undefined, } const message = previous ? /global\.data previous state.*SharedArrayBuffer/ : /global\.data setState\(next\).*SharedArrayBuffer/ if (mode === 'direct') { diff --git a/lib/build-pages/page-data.js b/lib/build-pages/page-data.js index 7b78bf62..1aa32d09 100644 --- a/lib/build-pages/page-data.js +++ b/lib/build-pages/page-data.js @@ -7,6 +7,8 @@ */ import { readFile } from 'node:fs/promises' +import { normalize } from 'node:path' +import { toPosix } from '../helpers/path.js' import { resolveVars, resolvePostVars, resolveVarsExport } from './resolve-vars.js' import { pageBuilders } from './page-builders/index.js' import { parseMdFileContents } from './page-builders/md/parse-md.js' @@ -219,6 +221,15 @@ export class PageData { ] } + /** + * Source-root-relative identity with POSIX separators, independent of checkout + * location and output URL. Generated pages use their synthetic factory relname. + * @returns {string} + */ + get sourceId () { + return toPosix(normalize(this.pageInfo.pageFile.relname)) + } + /** * Returns the cached, shallow-frozen variable set for the page. Requires initialization. * @return {T} default/global, layout, page, and builder vars merged together diff --git a/lib/build-pages/page-data.test.js b/lib/build-pages/page-data.test.js index a60dd615..a769bc45 100644 --- a/lib/build-pages/page-data.test.js +++ b/lib/build-pages/page-data.test.js @@ -7,7 +7,7 @@ import { test } from 'node:test' import assert from 'node:assert' import { mkdtemp, writeFile, rm } from 'node:fs/promises' -import { join } from 'node:path' +import { join, sep } from 'node:path' import { tmpdir } from 'node:os' import { PageData, resolveLayout } from './page-data.js' import { computePageUrl } from './compute-page-url.js' @@ -51,6 +51,35 @@ function mdPageInfo (filepath) { } } +test('PageData.sourceId is a read-only normalized source-relative identity, not an output path', () => { + /** @param {string} root @param {string} relname */ + const createPage = (root, relname) => new PageData({ + pageInfo: { + ...mdPageInfo(join(root, relname)), + pageFile: { ...mdPageInfo(join(root, relname)).pageFile, root, relname }, + }, + globalVars: {}, + globalStyle: undefined, + globalClient: undefined, + defaultStyle: null, + defaultClient: null, + builderOptions, + }) + const relname = ['.', 'docs', 'topic', '..', 'README.md'].join(sep) + const original = createPage(join(tmpdir(), 'checkout-one'), relname) + const relocated = createPage(join(tmpdir(), 'checkout-two'), relname) + assert.equal(original.sourceId, 'docs/README.md') + assert.equal(relocated.sourceId, original.sourceId) + assert.notEqual(original.pageInfo.pageFile.filepath, relocated.pageInfo.pageFile.filepath) + assert.equal(original.pageInfo.pageFile.relname, relname, 'reading the ID does not mutate discovery metadata') + assert.equal(Reflect.set(original, 'sourceId', 'override'), false) + assert.equal(original.sourceId, 'docs/README.md') + assert.equal(createPage(tmpdir(), 'README.md').sourceId, 'README.md') + assert.equal(createPage(tmpdir(), join('first', 'page.md')).sourceId, 'first/page.md') + assert.equal(createPage(tmpdir(), join('second', 'page.md')).sourceId, 'second/page.md') + assert.equal(createPage(tmpdir(), join('docs', 'articles.pages.js#1')).sourceId, 'docs/articles.pages.js#1') +}) + test.describe('PageData.vars', () => { test('returns a cached, frozen vars object and invalidates when sources change', async () => { const dir = await mkdtemp(join(tmpdir(), 'domstack-pagedata-cache-test-')) diff --git a/site/globals/global.data.ts b/site/globals/global.data.ts index 54127367..d63592fa 100644 --- a/site/globals/global.data.ts +++ b/site/globals/global.data.ts @@ -7,15 +7,15 @@ export default async function ({ pages, previousState, changes, setState }: Glob const reset = changes.kind === 'reset' || previousState === undefined const index: DocsNavigationIndex = reset ? new Map() : new Map(previousState) if (changes.kind === 'delta') { - for (const filepath of changes.removed) index.delete(filepath) + for (const sourceId of changes.removed) index.delete(sourceId) } const inputs = changes.kind === 'delta' && !reset ? changes.upserted : pages await Promise.all(inputs.map(async page => { - const filepath = page.pageInfo.pageFile.filepath + const sourceId = page.sourceId // Excludes the data-dependent index and renders docs without their layouts. const record = await readDocsNavigationPage(page) - if (record) index.set(filepath, record) - else index.delete(filepath) + if (record) index.set(sourceId, record) + else index.delete(sourceId) })) const docsNavigation = projectDocsNavigation(index.values()) diff --git a/test-cases/incremental-global-data/helpers.js b/test-cases/incremental-global-data/helpers.js index a8857118..c1cb52b2 100644 --- a/test-cases/incremental-global-data/helpers.js +++ b/test-cases/incremental-global-data/helpers.js @@ -8,7 +8,7 @@ * @property {'reset' | 'delta'} kind * @property {string} [reason] * @property {InputEvent[]} events - * @property {string[]} pages + * @property {string[]} pages Source-root-relative POSIX IDs, as are all semantic observations below. * @property {string[]} upserted * @property {string[]} removed * @property {string[]} rendered @@ -112,13 +112,14 @@ export default async ({ pages, previousState, changes, setState }) => { const state = changes.kind === 'reset' ? new Map() : new Map(previousState) const changed = changes.kind === 'reset' ? pages : changes.upserted const rendered = [] - for (const path of changes.removed ?? []) state.delete(path) + for (const sourceId of changes.removed ?? []) state.delete(sourceId) for (const page of changed) { assert.ok(pages.includes(page), 'upserts are current initialized PageData instances') - const path = page.pageInfo.pageFile.filepath - if (!page.vars.article) { state.delete(path); continue } - rendered.push(path) - state.set(path, { + const sourceId = page.sourceId + assert.equal(typeof sourceId, 'string') + if (!page.vars.article) { state.delete(sourceId); continue } + rendered.push(sourceId) + state.set(sourceId, { url: page.pageInfo.url, title: page.vars.title, tag: page.vars.tag ?? null, @@ -142,8 +143,8 @@ export default async ({ pages, previousState, changes, setState }) => { appendFileSync(${JSON.stringify(log)}, JSON.stringify({ kind: changes.kind, reason: changes.reason, events: changes.events.map(({ type, filepath }) => ({ type, filepath })), - pages: pages.map(page => page.pageInfo.pageFile.filepath).sort(), - upserted: (changes.upserted ?? []).map(page => page.pageInfo.pageFile.filepath).sort(), + pages: pages.map(page => page.sourceId).sort(), + upserted: (changes.upserted ?? []).map(page => page.sourceId).sort(), removed: [...(changes.removed ?? [])].sort(), rendered: rendered.sort(), previousKeys, previousIsMap: previousState instanceof Map, invocation: ++invocation, threadId, publicData, diff --git a/test-cases/incremental-global-data/index.test.js b/test-cases/incremental-global-data/index.test.js index 3b9a9f82..99c43eb3 100644 --- a/test-cases/incremental-global-data/index.test.js +++ b/test-cases/incremental-global-data/index.test.js @@ -36,8 +36,8 @@ test('body-only edits render one producer input, preserve navigation sibling mti const initial = (await site.calls())[0] assertReset(initial) assert.deepEqual(initial.events, []) - assert.deepEqual(initial.pages, ['a/page.md', 'b/page.md', 'nav/page.js'].map(name => join(site.src, name)).sort(), 'templates and generated pages are not producer inputs') - assert.deepEqual(initial.rendered, ['a/page.md', 'b/page.md'].map(name => join(site.src, name))) + assert.deepEqual(initial.pages, ['a/page.md', 'b/page.md', 'nav/page.js'], 'templates and generated pages are not producer inputs') + assert.deepEqual(initial.rendered, ['a/page.md', 'b/page.md']) const untouched = ['nav/index.html', 'b/index.html', 'summary.html'].map(name => join(site.dest, name)) const sentinel = new Date('2000-01-01T00:00:00Z') @@ -46,11 +46,11 @@ test('body-only edits render one producer input, preserve navigation sibling mti const event = site.event('a/page.md') await site.write('a/page.md', article('Alpha', 'Only this body changed.')) const call = await site.rebuild([event]) - assertDelta(call, [event.filepath]) + assertDelta(call, ['a/page.md']) assert.notEqual(call.threadId, initial.threadId) assert.deepEqual(call.previousKeys, initial.rendered) assert.deepEqual(call.events, [event]) - assert.deepEqual(call.rendered, [event.filepath]) + assert.deepEqual(call.rendered, ['a/page.md']) assert.deepEqual(call.publicData.navigation, initial.publicData.navigation) assert.notDeepEqual(call.publicData.search, initial.publicData.search) assert.match(await readFile(join(site.dest, 'a/index.html'), 'utf8'), /Only this body changed/) @@ -60,8 +60,8 @@ test('body-only edits render one producer input, preserve navigation sibling mti // A standalone build on the same source must not replace the live session's state. await site.write('b/page.md', article('Beta', 'A second edit.')) const second = await site.rebuild([site.event('b/page.md')]) - assertDelta(second, [join(site.src, 'b/page.md')]) - assert.deepEqual(second.rendered, [join(site.src, 'b/page.md')]) + assertDelta(second, ['b/page.md']) + assert.deepEqual(second.rendered, ['b/page.md']) await site.matchesFresh() }) @@ -89,7 +89,7 @@ test('H1, frontmatter title/metadata, companion vars, and transitive page/layout await site.write(change.name, change.content) const event = site.event(change.name, /** @type {InputEvent['type']} */ (change.type ?? 'change')) const call = await site.rebuild([event]) - assertDelta(call, change.inputs.map(name => join(site.src, name))) + assertDelta(call, change.inputs) assert.deepEqual(call.events, [event]) assert.match(JSON.stringify(call.publicData), new RegExp(change.expected)) await site.matchesFresh() @@ -114,7 +114,7 @@ test('barrel re-export leaves upsert source pages shared with templates and rese await site.write('page-leaf.js', "export const content = 'Barrel two'\n") const event = site.event('page-leaf.js') const call = await site.rebuild([event]) - const source = join(site.src, 'code/page.js') + const source = 'code/page.js' assertDelta(call, [source]) assert.deepEqual(call.events, [event]) assert.deepEqual(call.rendered, [source], 'a direct template consumer must not hide the source page reached through barrels') @@ -129,7 +129,7 @@ test('barrel re-export leaves upsert source pages shared with templates and rese assertReset(reset) assert.equal(reset.reason, 'global-data-changed', 'producer barrels are recognized dependencies, not unknown-event fallbacks') assert.deepEqual(reset.events, [producerEvent]) - assert.deepEqual(reset.rendered, ['a/page.md', 'b/page.md', 'code/page.js'].map(name => join(site.src, name))) + assert.deepEqual(reset.rendered, ['a/page.md', 'b/page.md', 'code/page.js']) assert.ok(reset.publicData.search.every(row => row.html.startsWith('Barrel search: '))) await site.matchesFresh() }) @@ -153,14 +153,45 @@ test('global vars, markdown settings, and producer roots and transitive imports const call = await site.rebuild([event]) assertReset(call) assert.deepEqual(call.events, [event]) - assert.deepEqual(call.rendered, ['a/page.md', 'b/page.md'].map(name => join(site.src, name))) + assert.deepEqual(call.rendered, ['a/page.md', 'b/page.md']) assert.ok(JSON.stringify(call.publicData).includes(expected)) await site.matchesFresh() }) } }) -test('adds, deletes, renames, and draft eligibility reconcile absolute source identities and stale outputs', options, async t => { +test('same-basename source IDs keep separate cache entries and removals delete only the matching state', options, async t => { + const site = await fixture(t) + await site.start() + const initial = (await site.calls())[0] + assertReset(initial) + assert.deepEqual(initial.pages, ['a/page.md', 'b/page.md', 'nav/page.js']) + assert.deepEqual(initial.rendered, ['a/page.md', 'b/page.md']) + assert.deepEqual(initial.publicData.navigation.map(row => row.title), ['Alpha', 'Beta']) + + await rm(join(site.src, 'a/page.md')) + const event = site.event('a/page.md', 'removed') + const removed = await site.rebuild([event]) + assertDelta(removed, [], ['a/page.md']) + assert.deepEqual(removed.events, [event], 'watch events retain absolute filesystem paths') + assert.deepEqual(removed.previousKeys, ['a/page.md', 'b/page.md'], 'same basenames occupy distinct cache entries') + assert.deepEqual(removed.pages, ['b/page.md', 'nav/page.js']) + assert.deepEqual(removed.rendered, [], 'the surviving row comes from cached state') + assert.deepEqual(removed.publicData.navigation, initial.publicData.navigation.filter(row => row.title === 'Beta')) + assert.deepEqual(removed.publicData.search, initial.publicData.search.filter(row => row.url === initial.publicData.navigation[1]?.url)) + await assert.rejects(stat(join(site.dest, 'a/index.html')), { code: 'ENOENT' }) + await site.matchesFresh() + + await site.write('b/page.md', article('Surviving Beta')) + const edited = await site.rebuild([site.event('b/page.md')]) + assertDelta(edited, ['b/page.md']) + assert.deepEqual(edited.previousKeys, ['b/page.md'], 'the removed ID is absent from the next committed state') + assert.deepEqual(edited.rendered, ['b/page.md']) + assert.deepEqual(edited.publicData.navigation.map(row => row.title), ['Surviving Beta']) + await site.matchesFresh() +}) + +test('adds, deletes, renames, and draft eligibility reconcile relative source IDs and stale outputs', options, async t => { for (const buildDrafts of [false, true]) { await t.test(`buildDrafts: ${buildDrafts}`, async t => { const site = await fixture(t, { opts: { buildDrafts } }) @@ -168,7 +199,7 @@ test('adds, deletes, renames, and draft eligibility reconcile absolute source id await site.write('c/page.md', article('Gamma')) const added = site.event('c/page.md', 'added') const call = await site.rebuild([added]) - assertDelta(call, [added.filepath]) + assertDelta(call, ['c/page.md']) assert.deepEqual(call.events, [added]) await site.matchesFresh() @@ -182,16 +213,16 @@ test('adds, deletes, renames, and draft eligibility reconcile absolute source id const call = await site.rebuild(events) const wasEligible = buildDrafts || !from.includes('.draft.') const eligible = buildDrafts || !to.includes('.draft.') - assertDelta(call, eligible ? [join(site.src, to)] : [], wasEligible ? [join(site.src, from)] : []) + assertDelta(call, eligible ? [to] : [], wasEligible ? [from] : []) assert.deepEqual(call.events, events) - assert.equal(call.pages.includes(join(site.src, to)), eligible) + assert.equal(call.pages.includes(to), eligible) assert.equal(call.publicData.navigation.some(row => row.title === 'Gamma'), eligible) await site.matchesFresh() } await rm(join(site.src, 'c/page.md')) const removed = site.event('c/page.md', 'removed') const deleted = await site.rebuild([removed]) - assertDelta(deleted, [], [removed.filepath]) + assertDelta(deleted, [], ['c/page.md']) assert.deepEqual(deleted.rendered, []) assert.deepEqual(deleted.events, [removed]) await assert.rejects(stat(join(site.dest, 'c/index.html')), { code: 'ENOENT' }) @@ -208,7 +239,7 @@ test('one batch unions direct and transitive inputs, deduplicates upserts, and r await site.write('b/page.md', article('Batch Beta')) const events = [site.event('a/page.md'), site.event('companion-leaf.js'), site.event('a/page.md'), site.event('b/page.md')] const call = await site.rebuild(events) - const inputs = ['a/page.md', 'b/page.md'].map(name => join(site.src, name)) + const inputs = ['a/page.md', 'b/page.md'] assertDelta(call, inputs) assert.deepEqual(call.rendered, inputs) assert.deepEqual(call.events, events) @@ -239,8 +270,8 @@ test('events arriving during an active producer are buffered into the next union await site.dom.settled() const calls = await site.calls() assert.equal(calls.length, 3, 'initial, active, and one buffered build') - assertDelta(calls[1], [join(site.src, 'a/page.md')]) - assertDelta(calls[2], ['a/page.md', 'b/page.md'].map(name => join(site.src, name))) + assertDelta(calls[1], ['a/page.md']) + assertDelta(calls[2], ['a/page.md', 'b/page.md']) assert.deepEqual(calls[2].events, events) await site.matchesFresh() } finally { @@ -265,7 +296,7 @@ test('watch registers before initial producer work and buffers startup changes a const calls = await site.calls() assert.equal(calls.length, 2, 'initial build followed by one buffered startup batch') assertReset(calls[0]) - assertDelta(calls[1], ['a/page.md', 'c/page.md'].map(name => join(site.src, name))) + assertDelta(calls[1], ['a/page.md', 'c/page.md']) assert.deepEqual(calls[1].events, events) await site.matchesFresh() } finally { @@ -284,7 +315,7 @@ test('setState from a failed producer or later render is not reused and recovery await writeFile(marker, '') await site.write('a/page.md', article('Alpha', 'Failed build body.')) const failed = await site.rebuild([site.event('a/page.md')]) - assertDelta(failed, [join(site.src, 'a/page.md')]) + assertDelta(failed, ['a/page.md']) assert.equal(await readFile(join(site.dest, 'data.json'), 'utf8'), before, 'failed phase does not publish the new public-data template') await rm(marker) @@ -298,8 +329,8 @@ test('setState from a failed producer or later render is not reused and recovery await site.write('a/page.md', article('Alpha', 'Successful delta after recovery.')) const delta = await site.rebuild([site.event('a/page.md')]) - assertDelta(delta, [join(site.src, 'a/page.md')]) - assert.deepEqual(delta.rendered, [join(site.src, 'a/page.md')]) + assertDelta(delta, ['a/page.md']) + assert.deepEqual(delta.rendered, ['a/page.md']) await site.matchesFresh() }) } @@ -331,7 +362,7 @@ export const pageOutputs = ({ vars }) => ({ outputName: vars.sidecar, content: v try { await site.write('a/page.md', article('Candidate Alpha', 'Candidate body.', 'sidecar: candidate.txt\n')) const failed = await site.rebuild([site.event('a/page.md')]) - assertDelta(failed, [join(site.src, 'a/page.md')]) + assertDelta(failed, ['a/page.md']) assert.equal(cleanupAttempts, 1, 'failure occurs in stale-output cleanup, after rendering') assert.equal(await readFile(stale, 'utf8'), 'Initial Alpha') assert.equal(await readFile(candidate, 'utf8'), 'Candidate Alpha', 'new page-owned output was already written') @@ -375,19 +406,19 @@ test('events buffered during a failing producer batch automatically recover with await site.dom.settled() const calls = await site.calls() assert.equal(calls.length, 3, 'initial, failed, and one buffered recovery build') - assertDelta(calls[1], [activeEvent.filepath]) + assertDelta(calls[1], ['a/page.md']) assert.deepEqual(calls[1].events, [activeEvent]) assertReset(calls[2]) assert.deepEqual(calls[2].events, events) - assert.deepEqual(calls[2].rendered, ['a/page.md', 'b/page.md'].map(name => join(site.src, name))) + assert.deepEqual(calls[2].rendered, ['a/page.md', 'b/page.md']) assert.ok(JSON.stringify(calls[2].publicData).includes('Body from the failing batch.')) assert.ok(JSON.stringify(calls[2].publicData).includes('Buffered recovery tag')) await site.matchesFresh() await site.write('b/page.md', article('Beta after buffered recovery')) const delta = await site.rebuild([site.event('b/page.md')]) - assertDelta(delta, [join(site.src, 'b/page.md')]) - assert.deepEqual(delta.rendered, [join(site.src, 'b/page.md')]) + assertDelta(delta, ['b/page.md']) + assert.deepEqual(delta.rendered, ['b/page.md']) await site.matchesFresh() } finally { await rm(site.producerFailure, { force: true }) @@ -405,8 +436,9 @@ test('independent sessions and a stopped/restarted instance never inherit each o assertReset((await second.calls())[0]) await second.write('b/page.md', article('Other Beta')) const other = await second.rebuild([second.event('b/page.md')]) - assertDelta(other, [join(second.src, 'b/page.md')]) - assert.ok(other.previousKeys?.every(path => path.startsWith(second.src))) + assertDelta(other, ['b/page.md']) + assert.deepEqual(other.previousKeys, ['a/page.md', 'b/page.md']) + assert.deepEqual(other.publicData.navigation.map(row => row.title), ['Other session', 'Other Beta']) assert.equal((await first.calls()).length, 2) await second.matchesFresh() @@ -419,7 +451,7 @@ test('independent sessions and a stopped/restarted instance never inherit each o assert.deepEqual(restarted.events, []) await first.write('a/page.md', article('Restart delta')) const delta = await first.rebuild([first.event('a/page.md')]) - assertDelta(delta, [join(first.src, 'a/page.md')]) + assertDelta(delta, ['a/page.md']) await first.matchesFresh() }) @@ -454,7 +486,7 @@ test('native static JSON leaf edits reset producer, global-vars, and markdown-se for (const call of calls) { assertReset(call) assert.equal(call.reason, change.reason) - assert.deepEqual(call.rendered, ['a/page.md', 'b/page.md'].map(name => join(site.src, name))) + assert.deepEqual(call.rendered, ['a/page.md', 'b/page.md']) assert.ok(JSON.stringify(call.publicData).includes(change.expected)) } await site.matchesFresh() @@ -478,8 +510,8 @@ test('native atomic replacement updates one producer input and keeps watching th const calls = (await site.calls()).slice(beforeEdit) assertNativeEvents(calls, site.event('a/page.md')) for (const replaced of calls) { - assertDelta(replaced, [source]) - assert.deepEqual(replaced.rendered, [source]) + assertDelta(replaced, ['a/page.md']) + assert.deepEqual(replaced.rendered, ['a/page.md']) } assert.match(await readFile(join(site.dest, 'a/index.html'), 'utf8'), /Atomically replaced body/) await site.matchesFresh() @@ -492,8 +524,8 @@ test('native atomic replacement updates one producer input and keeps watching th const nextCalls = (await site.calls()).slice(beforeNextEdit) assertNativeEvents(nextCalls, site.event('a/page.md')) for (const edited of nextCalls) { - assertDelta(edited, [source]) - assert.deepEqual(edited.rendered, [source]) + assertDelta(edited, ['a/page.md']) + assert.deepEqual(edited.rendered, ['a/page.md']) } assert.match(await readFile(join(site.dest, 'a/index.html'), 'utf8'), /Edited replacement body/) await site.matchesFresh() @@ -511,8 +543,8 @@ test('native chokidar body edits reach the same incremental public API', options const calls = (await site.calls()).slice(beforeEdit) assertNativeEvents(calls, site.event('a/page.md')) for (const edited of calls) { - assertDelta(edited, [join(site.src, 'a/page.md')]) - assert.deepEqual(edited.rendered, [join(site.src, 'a/page.md')]) + assertDelta(edited, ['a/page.md']) + assert.deepEqual(edited.rendered, ['a/page.md']) } await site.matchesFresh() }) From 78c666dbf96a09e13c72379dab48dc3ef06cf59e Mon Sep 17 00:00:00 2001 From: Bret Comnes Date: Tue, 15 Sep 2026 09:40:52 -0700 Subject: [PATCH 03/14] Simplify incremental index flow and documentation example --- docs/data/README.md | 75 ++++++++++++++++++------------------- site/globals/global.data.ts | 19 +++++++--- 2 files changed, 51 insertions(+), 43 deletions(-) diff --git a/docs/data/README.md b/docs/data/README.md index 91af30c3..9db31a79 100644 --- a/docs/data/README.md +++ b/docs/data/README.md @@ -120,61 +120,60 @@ Returning from `global.data.ts` or calling `setState` does not commit it early. A failed build does not advance that baseline; the next page build resets and recomputes instead of trusting a partial candidate. This protects retained state, not filesystem outputs: output writes and deletions are not transactional and are not rolled back after failure. -### Source-keyed Markdown search example +### Simple documentation index -This example retains one rendered Markdown document per source file for a search-index consumer. -On reset it indexes all current Markdown pages; on a delta it renders only upserted Markdown pages and removes records that are no longer eligible. +This self-contained example caches each Markdown documentation page's title and URL, then returns a sorted navigation list. +It uses only initialized page metadata: no Markdown rendering, HTML parsing, or search service is needed. ```typescript // src/global.data.ts -import type { AsyncGlobalDataFunction } from '@domstack/static/types.js' +import type { GlobalDataFunctionParams } from '@domstack/static/types.js' type SourceVars = { title?: string } -type SearchDocument = { url: string, title: string, html: string } -type SearchData = { searchDocuments: SearchDocument[] } -type SearchState = Map +type Entry = { title: string, url: string } +type Index = Map -const globalData: AsyncGlobalDataFunction = async ({ +export default function ({ pages, previousState, changes, setState, -}) => { - const documents = changes.kind === 'reset' - ? new Map() - : previousState ?? new Map() - const upserted = changes.kind === 'reset' ? pages : changes.upserted - - if (changes.kind === 'delta') { - for (const sourceId of changes.removed) documents.delete(sourceId) +}: GlobalDataFunctionParams) { + let index: Index + let inputs: typeof pages + switch (changes.kind) { + case 'reset': + index = new Map() + inputs = pages + break + case 'delta': + index = previousState ?? new Map() + inputs = previousState === undefined ? pages : changes.upserted + for (const sourceId of changes.removed) index.delete(sourceId) + break + default: + throw new Error('Unhandled global-data changes', { cause: changes satisfies never }) } - for (const page of upserted) { - const sourceId = page.sourceId - if (page.pageInfo.type !== 'md') { - documents.delete(sourceId) + for (const page of inputs) { + const { url, type } = page.pageInfo + if (type !== 'md' || !url.startsWith('/docs/')) { + index.delete(page.sourceId) continue } - documents.set(sourceId, { - url: page.pageInfo.url, - title: page.vars.title ?? page.pageInfo.url, - html: await page.renderInnerPage(), - }) + index.set(page.sourceId, { title: page.vars.title ?? url, url }) } - setState(documents) + setState(index) return { - searchDocuments: [...documents.entries()] - .sort(([a], [b]) => a < b ? -1 : a > b ? 1 : 0) - .map(([, document]) => document), + docsNavigation: [...index.values()].sort((a, b) => a.url.localeCompare(b.url)), } } - -export default globalData ``` -The indexed Markdown pages must not themselves subscribe to global data when `renderInnerPage()` is called here; see [Rendering page content](#rendering-page-content). -A search template or generated-page factory subscribes with `export const dataDeps = ['searchDocuments']` and reads `data.searchDocuments` as usual. -The returned array of plain records is ordinary public data with the same top-level fingerprinting rules as a full recomputation. -Stable source-key ordering avoids changing its fingerprint merely because records were inserted in a different order. -`dataDeps` remains authoritative for consumer access and data-driven invalidation; retaining state neither adds subscriptions nor replaces output dependency tracking. +A reset fills the index from all pages; a delta replaces affected entries and removes deleted or newly ineligible pages. +The exhaustive switch makes TypeScript flag any unhandled change kind. +`previousState` is already isolated, so the delta can update that map directly; `setState(index)` stages a cloned snapshot. +A consuming layout declares `export const vars = { dataDeps: ['docsNavigation'] }` and reads `data.docsNavigation`. +Sorting by URL makes the public navigation independent of map insertion order, and unchanged navigation fingerprints leave its subscribers untouched. +Caching title and URL alone is inexpensive either way; the example demonstrates the state flow, while the real site below also caches extracted headings. ### How this documentation site uses the index @@ -195,8 +194,8 @@ A batch is not a snapshot of filesystem contents at event time, and an intermedi Write index updates as replacements and deletions that tolerate repeated invalidations, not as exactly-once event operations. Incremental indexing can reduce the producer's Markdown rendering work, but it does not make the entire build O(changed pages). -All source pages are still initialized, retained state is cloned, and returned data is still fingerprinted; this example also sorts the complete index on each callback. -A counter around `renderInnerPage()` in this index measures indexing render calls only, not total source-page initialization, downstream rendering, or filesystem output writes. +All source pages are still initialized, retained state is cloned, and returned data is still fingerprinted; the example also sorts the complete index on each callback. +For an index that renders Markdown, a counter around `renderInnerPage()` measures indexing render calls only, not total source-page initialization, downstream rendering, or filesystem output writes. Those are separate stages, and rendering an output does not necessarily write it when its bytes are unchanged. ## Data subscriptions diff --git a/site/globals/global.data.ts b/site/globals/global.data.ts index d63592fa..0992128c 100644 --- a/site/globals/global.data.ts +++ b/site/globals/global.data.ts @@ -4,12 +4,21 @@ import { render } from 'fragtml' import { readDocsNavigationPage, projectDocsNavigation, docsIndex } from '../layouts/docs/navigation.js' export default async function ({ pages, previousState, changes, setState }: GlobalDataFunctionParams) { - const reset = changes.kind === 'reset' || previousState === undefined - const index: DocsNavigationIndex = reset ? new Map() : new Map(previousState) - if (changes.kind === 'delta') { - for (const sourceId of changes.removed) index.delete(sourceId) + let index: DocsNavigationIndex + let inputs: typeof pages + switch (changes.kind) { + case 'reset': + index = new Map() + inputs = pages + break + case 'delta': + index = previousState ?? new Map() + inputs = previousState === undefined ? pages : changes.upserted + for (const sourceId of changes.removed) index.delete(sourceId) + break + default: + throw new Error('Unhandled global-data changes', { cause: changes satisfies never }) } - const inputs = changes.kind === 'delta' && !reset ? changes.upserted : pages await Promise.all(inputs.map(async page => { const sourceId = page.sourceId // Excludes the data-dependent index and renders docs without their layouts. From 08cdd899635e6e3fb73b06989d1d0fb9ce05ac6f Mon Sep 17 00:00:00 2001 From: Bret Comnes Date: Tue, 15 Sep 2026 09:50:37 -0700 Subject: [PATCH 04/14] Focus incremental global-data docs on usage --- docs/data/README.md | 98 ++++++++++----------------------------------- 1 file changed, 21 insertions(+), 77 deletions(-) diff --git a/docs/data/README.md b/docs/data/README.md index 9db31a79..dfd61494 100644 --- a/docs/data/README.md +++ b/docs/data/README.md @@ -75,55 +75,20 @@ export default buildGlobalData ## Incremental global data -In watch mode, a global-data callback can retain explicit state to avoid repeating expensive indexing work for unchanged source pages. -The callback receives `{ pages, previousState, changes, setState }`: - -- `pages` is the complete current, initialized source-backed `PageData[]` collection, not just the pages selected for output rendering. - Generated pages are not included, and source membership respects discovery and draft eligibility. -- `previousState` is an isolated clone of the state retained from the last successful build, or `undefined` on reset or when no state was retained. -- `setState(next)` immediately clones and stages an explicit state value for the next successful build. - The last call wins; later mutations to `next` do not alter that snapshot. - Mutating `previousState` alone does not stage an update: without `setState`, a delta retains the old state and a reset leaves state undefined. -- `changes` is a discriminated union describing source inputs, independently of output-render filters: - - `{ kind: 'reset', reason: string, events: WatchEvent[] }` means rebuild your state from all of `pages`. - It has no `upserted` or `removed` fields. - - `{ kind: 'delta', upserted: PageData[], removed: string[], events: WatchEvent[] }` supplies current initialized pages whose inputs were invalidated or which became newly eligible, plus source IDs no longer eligible. - Upserts are conservative input invalidations, not a guarantee that rendered content changed. - -Use the read-only `page.sourceId` getter as the identity for an upsert, for example `docs/data/README.md`. -It is `page.pageInfo.pageFile.relname` normalized relative to the source root, with `/` separators on every platform and no checkout-specific prefix. -`changes.removed` contains the same source IDs, not absolute paths, URLs, or output filenames. -IDs distinguish identical filenames in different directories and stay the same when the checkout moves; a source rename is a removal plus an upsert. -The absolute `page.pageInfo.pageFile.filepath` remains available for filesystem operations. -Treat an upsert as replacement of that source's cached record and a removal as deletion of that key. -If an upsert stops matching your own index filter, delete its cached record too. - -Both variants expose the existing raw `WatchEvent` objects through `changes.events`, with `type`, `filepath`, `name`, and `convention` fields. -Raw event `filepath` values and internal dependency paths remain absolute; do not use them as keys in a source-ID-keyed index. -The event list preserves batch order, duplicates, and removals rather than becoming a deduplicated page-change list; the initial reset has an empty event list. -Use `changes.kind`, `upserted`, and `removed` to update an index rather than reconstructing page membership from raw events. - -### Retained state and resets - -State must be structured-cloneable: plain data, arrays, `Map`, `Set`, `Date`, and other isolated values supported by `structuredClone` are suitable. -`SharedArrayBuffer` and views backed by shared memory are rejected because cloning them does not isolate their bytes; copy them into non-shared storage first. -Store extracted records, not `PageData` instances, functions, or renderers; uncloneable state passed to `setState` throws an actionable error. -State is private to the producer and separate from its ordinary returned data, so a retained `Map` does not itself participate in public-data fingerprints or subscriptions. -State is session-local, not a persistent disk cache or retained module globals. - -Rebuild from `pages` on every reset, regardless of the diagnostic `reason` string. -Resets cover the initial build and a new or restarted watch session, changes to the global-data producer or its tracked static imports, global vars or their tracked imports, Markdown settings or their tracked imports, broader global configuration changes, unknown or unreliable events, and recovery after build or dependency-analysis failure. -A reset always supplies `previousState: undefined`, even if an earlier build had retained state. - -Only a successful page/template phase, including generated pages, followed by successful cleanup commits the staged state and source-membership baseline. -Returning from `global.data.ts` or calling `setState` does not commit it early. -A failed build does not advance that baseline; the next page build resets and recomputes instead of trusting a partial candidate. -This protects retained state, not filesystem outputs: output writes and deletions are not transactional and are not rolled back after failure. +In watch mode, keep an index so you only recompute entries for affected source pages. +The callback receives four fields: + +- `pages`: All current source pages, excluding generated pages. +- `previousState`: Your saved state, or `undefined` when starting fresh. It is isolated and safe to modify. +- `changes`: On `kind: 'reset'`, rebuild from `pages`. On `kind: 'delta'`, replace entries for `changes.upserted` and delete the IDs in `changes.removed`. +- `setState(next)`: Save a snapshot for the next successful build. Always call it to retain your updates. + +Use `page.sourceId` as the index key: a read-only source-relative path such as `docs/data/README.md`, using `/` separators on every platform. +`changes.removed` contains these same IDs. ### Simple documentation index -This self-contained example caches each Markdown documentation page's title and URL, then returns a sorted navigation list. -It uses only initialized page metadata: no Markdown rendering, HTML parsing, or search service is needed. +This example caches titles and URLs for Markdown pages under `/docs/` and returns a sorted navigation list. ```typescript // src/global.data.ts @@ -168,35 +133,16 @@ export default function ({ } ``` -A reset fills the index from all pages; a delta replaces affected entries and removes deleted or newly ineligible pages. -The exhaustive switch makes TypeScript flag any unhandled change kind. -`previousState` is already isolated, so the delta can update that map directly; `setState(index)` stages a cloned snapshot. A consuming layout declares `export const vars = { dataDeps: ['docsNavigation'] }` and reads `data.docsNavigation`. -Sorting by URL makes the public navigation independent of map insertion order, and unchanged navigation fingerprints leave its subscribers untouched. -Caching title and URL alone is inexpensive either way; the example demonstrates the state flow, while the real site below also caches extracted headings. - -### How this documentation site uses the index - -DOMStack's own `site/globals/global.data.ts` retains an index keyed by `page.sourceId` of documentation titles, heading anchors, and ordering/group/parent metadata. -Only upserted documentation pages are rendered and parsed; removed or newly ineligible sources are dropped from the index. -The navigation helpers in `site/layouts/docs/navigation.js` then regenerate the ordered, nested `docsNavigation` and `docsIndexHtml` from those plain records without rendering Markdown again. -Parent/child nesting is assembled on fresh entries rather than mutating cached heading arrays, so repeated builds do not accumulate children and deletions or parent changes take effect immediately. -A body-only edit updates one record, but unchanged TOC fingerprints prevent other documentation pages from rebuilding. - -### Tracking, batching, and performance limits - -Dependency tracking covers relative static ESM imports and re-exports in the non-ignored watched source tree, including imported JSON files. -It does not automatically track bare-package internals, CommonJS `require()`, dynamic imports, arbitrary filesystem reads, network responses, environment variables, or other dynamic inputs. -When those inputs change, explicitly invalidate the relevant source input (or the producer for a full state reset), or restart the watch session; do not assume a raw event or a cached record discovers the change. +If the returned navigation is unchanged, its subscribers do not rebuild just because the index was updated. +For a larger example that also caches heading links, see `site/globals/global.data.ts` in this repository. -Watch events can be batched while a build is running, and subsequent builds converge on the current source tree. -A batch is not a snapshot of filesystem contents at event time, and an intermediate build may observe newer contents before queued events are processed. -Write index updates as replacements and deletions that tolerate repeated invalidations, not as exactly-once event operations. +### Usage notes -Incremental indexing can reduce the producer's Markdown rendering work, but it does not make the entire build O(changed pages). -All source pages are still initialized, retained state is cloned, and returned data is still fingerprinted; the example also sorts the complete index on each callback. -For an index that renders Markdown, a counter around `renderInnerPage()` measures indexing render calls only, not total source-page initialization, downstream rendering, or filesystem output writes. -Those are separate stages, and rendering an output does not necessarily write it when its bytes are unchanged. +- State lasts for the current watch session and is retained only after a successful build. Always handle resets, including after producer or settings changes and failed builds. +- Store cloneable records, arrays, or maps—not `PageData` instances, functions, or shared memory. +- An upsert may repeat even when content is unchanged. Replace its cached entry, or delete it if the page no longer belongs in your index. +- Relative static imports and re-exports within the watched source tree are tracked. Restart watching when other inputs change, such as arbitrary file reads, environment variables, or network data. ## Data subscriptions @@ -308,9 +254,8 @@ Use `AsyncGlobalDataFunction` instead when the implementation needs For typed source input, use `GlobalDataFunction` or its async counterpart. Helpers can accept `GlobalDataFunctionParams['pages']` without recovering types from the full global-data result. For retained state, add the final generic: `GlobalDataFunction`, `AsyncGlobalDataFunction`, or `GlobalDataFunctionParams`. -`State` defaults to `unknown`; existing result and source generic positions are unchanged. -`GlobalDataChanges`, `GlobalDataResetChanges`, `GlobalDataDeltaChanges`, and `WatchEvent` are also exported from `@domstack/static/types.js`. -Manually constructed `GlobalDataFunctionParams` objects must include `previousState`, `changes`, and `setState`, even though callbacks may ignore those fields. +`State` defaults to `unknown`. +Use `GlobalDataChanges` from `@domstack/static/types.js` when typing a helper that handles reset or delta changes. ## Global data caveats @@ -407,8 +352,7 @@ The current `page` is a `PageInfo` object with the following properties: - `generated`: Metadata about the `*.pages.ts` file that created a generated page, or `undefined` for a source-backed page. Each `PageData` entry supplied to `global.data.ts` exposes this object as `page.pageInfo`. -The read-only `page.sourceId` getter provides its normalized source-relative file identity directly, without traversing `pageInfo`; it is distinct from the output URL. -Generated `PageData` instances use their synthetic factory relname (such as `archive.pages.ts#0`), but are not part of the global-data source collection. +Use `page.sourceId` for the source-relative index key and `page.pageInfo.url` for the output URL. Combine `page.pageInfo.url` with a `siteUrl` from `global.vars.ts` to build an absolute URL: `` `${vars.siteUrl}${page.pageInfo.url}` ``. The [RSS and JSON feed recipe](../cookbook/feeds/) uses this pattern for feed item URLs. From 5cb9a0ee5c51337fe49cd97bb8d7e55e553c57ce Mon Sep 17 00:00:00 2001 From: Bret Comnes Date: Tue, 15 Sep 2026 09:52:38 -0700 Subject: [PATCH 05/14] Clarify when incremental global data is useful --- docs/data/README.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/docs/data/README.md b/docs/data/README.md index dfd61494..37213f84 100644 --- a/docs/data/README.md +++ b/docs/data/README.md @@ -75,6 +75,10 @@ export default buildGlobalData ## Incremental global data +> [!NOTE] +> Most sites do not need incremental global data. +> Start with a regular `global.data.ts` callback and consider incremental indexing when a large number of pages makes watch rebuilds slow. + In watch mode, keep an index so you only recompute entries for affected source pages. The callback receives four fields: From d96b74cae4198d28150b9dede2317d77c2a74447 Mon Sep 17 00:00:00 2001 From: Bret Comnes Date: Tue, 15 Sep 2026 09:56:13 -0700 Subject: [PATCH 06/14] Prioritize core data concepts before incremental indexing --- docs/data/README.md | 153 ++++++++++++++++++++++---------------------- 1 file changed, 76 insertions(+), 77 deletions(-) diff --git a/docs/data/README.md b/docs/data/README.md index 37213f84..73dc7a19 100644 --- a/docs/data/README.md +++ b/docs/data/README.md @@ -27,8 +27,7 @@ See [Supported file types](../typescript/#supported-file-types) for all availabl For data that aggregates across multiple pages — like blog indexes, sitemaps, recent-post lists, or RSS feed content — use `global.data.ts`. It is the only public build hook that receives the source-backed `PageData[]` collection. It returns an object of named, top-level values that downstream consumers can explicitly subscribe to. -Existing synchronous or asynchronous callbacks that accept only `({ pages })` remain supported, as do static default object exports such as `export default { siteName: 'My site' }`. -Incremental state is opt-in and does not change the meaning of the returned object. +The default export can be an object or a synchronous or asynchronous function that returns one. ```typescript // src/global.data.ts @@ -73,81 +72,6 @@ const buildGlobalData: AsyncGlobalDataFunction = async ({ pages }) = export default buildGlobalData ``` -## Incremental global data - -> [!NOTE] -> Most sites do not need incremental global data. -> Start with a regular `global.data.ts` callback and consider incremental indexing when a large number of pages makes watch rebuilds slow. - -In watch mode, keep an index so you only recompute entries for affected source pages. -The callback receives four fields: - -- `pages`: All current source pages, excluding generated pages. -- `previousState`: Your saved state, or `undefined` when starting fresh. It is isolated and safe to modify. -- `changes`: On `kind: 'reset'`, rebuild from `pages`. On `kind: 'delta'`, replace entries for `changes.upserted` and delete the IDs in `changes.removed`. -- `setState(next)`: Save a snapshot for the next successful build. Always call it to retain your updates. - -Use `page.sourceId` as the index key: a read-only source-relative path such as `docs/data/README.md`, using `/` separators on every platform. -`changes.removed` contains these same IDs. - -### Simple documentation index - -This example caches titles and URLs for Markdown pages under `/docs/` and returns a sorted navigation list. - -```typescript -// src/global.data.ts -import type { GlobalDataFunctionParams } from '@domstack/static/types.js' - -type SourceVars = { title?: string } -type Entry = { title: string, url: string } -type Index = Map - -export default function ({ - pages, previousState, changes, setState, -}: GlobalDataFunctionParams) { - let index: Index - let inputs: typeof pages - switch (changes.kind) { - case 'reset': - index = new Map() - inputs = pages - break - case 'delta': - index = previousState ?? new Map() - inputs = previousState === undefined ? pages : changes.upserted - for (const sourceId of changes.removed) index.delete(sourceId) - break - default: - throw new Error('Unhandled global-data changes', { cause: changes satisfies never }) - } - - for (const page of inputs) { - const { url, type } = page.pageInfo - if (type !== 'md' || !url.startsWith('/docs/')) { - index.delete(page.sourceId) - continue - } - index.set(page.sourceId, { title: page.vars.title ?? url, url }) - } - - setState(index) - return { - docsNavigation: [...index.values()].sort((a, b) => a.url.localeCompare(b.url)), - } -} -``` - -A consuming layout declares `export const vars = { dataDeps: ['docsNavigation'] }` and reads `data.docsNavigation`. -If the returned navigation is unchanged, its subscribers do not rebuild just because the index was updated. -For a larger example that also caches heading links, see `site/globals/global.data.ts` in this repository. - -### Usage notes - -- State lasts for the current watch session and is retained only after a successful build. Always handle resets, including after producer or settings changes and failed builds. -- Store cloneable records, arrays, or maps—not `PageData` instances, functions, or shared memory. -- An upsert may repeat even when content is unchanged. Replace its cached entry, or delete it if the page no longer belongs in your index. -- Relative static imports and re-exports within the watched source tree are tracked. Restart watching when other inputs change, such as arbitrary file reads, environment variables, or network data. - ## Data subscriptions The returned object is not merged into `vars`. @@ -428,3 +352,78 @@ export default globalData Rendering performed inside `global.data.ts` cannot use the derived values that the same file is still computing. After `global.data.ts` returns, consumers receive only the values named by their `dataDeps` declarations. + +## Incremental global data + +> [!NOTE] +> Most sites do not need incremental global data. +> Start with a regular `global.data.ts` callback and consider incremental indexing when a large number of pages makes watch rebuilds slow. + +In watch mode, keep an index so you only recompute entries for affected source pages. +The callback receives four fields: + +- `pages`: All current source pages, excluding generated pages. +- `previousState`: Your saved state, or `undefined` when starting fresh. It is isolated and safe to modify. +- `changes`: On `kind: 'reset'`, rebuild from `pages`. On `kind: 'delta'`, replace entries for `changes.upserted` and delete the IDs in `changes.removed`. +- `setState(next)`: Save a snapshot for the next successful build. Always call it to retain your updates. + +Use `page.sourceId` as the index key: a read-only source-relative path such as `docs/data/README.md`, using `/` separators on every platform. +`changes.removed` contains these same IDs. + +### Simple documentation index + +This example caches titles and URLs for Markdown pages under `/docs/` and returns a sorted navigation list. + +```typescript +// src/global.data.ts +import type { GlobalDataFunctionParams } from '@domstack/static/types.js' + +type SourceVars = { title?: string } +type Entry = { title: string, url: string } +type Index = Map + +export default function ({ + pages, previousState, changes, setState, +}: GlobalDataFunctionParams) { + let index: Index + let inputs: typeof pages + switch (changes.kind) { + case 'reset': + index = new Map() + inputs = pages + break + case 'delta': + index = previousState ?? new Map() + inputs = previousState === undefined ? pages : changes.upserted + for (const sourceId of changes.removed) index.delete(sourceId) + break + default: + throw new Error('Unhandled global-data changes', { cause: changes satisfies never }) + } + + for (const page of inputs) { + const { url, type } = page.pageInfo + if (type !== 'md' || !url.startsWith('/docs/')) { + index.delete(page.sourceId) + continue + } + index.set(page.sourceId, { title: page.vars.title ?? url, url }) + } + + setState(index) + return { + docsNavigation: [...index.values()].sort((a, b) => a.url.localeCompare(b.url)), + } +} +``` + +A consuming layout declares `export const vars = { dataDeps: ['docsNavigation'] }` and reads `data.docsNavigation`. +If the returned navigation is unchanged, its subscribers do not rebuild just because the index was updated. +For a larger example that also caches heading links, see `site/globals/global.data.ts` in this repository. + +### Usage notes + +- State lasts for the current watch session and is retained only after a successful build. Always handle resets, including after producer or settings changes and failed builds. +- Store cloneable records, arrays, or maps—not `PageData` instances, functions, or shared memory. +- An upsert may repeat even when content is unchanged. Replace its cached entry, or delete it if the page no longer belongs in your index. +- Relative static imports and re-exports within the watched source tree are tracked. Restart watching when other inputs change, such as arbitrary file reads, environment variables, or network data. From 694ff891c18c7d78ef660d6ec72c92d3919f08c2 Mon Sep 17 00:00:00 2001 From: Bret Comnes Date: Tue, 15 Sep 2026 10:15:33 -0700 Subject: [PATCH 07/14] Reduce incremental global data copies and iterations --- lib/build-pages/global-data-state.js | 51 +++++++----- lib/build-pages/global-data-state.test.js | 94 +++++++++++++++++++++++ lib/build-pages/index.js | 12 +-- 3 files changed, 133 insertions(+), 24 deletions(-) diff --git a/lib/build-pages/global-data-state.js b/lib/build-pages/global-data-state.js index 5143ccfc..55ceb4bc 100644 --- a/lib/build-pages/global-data-state.js +++ b/lib/build-pages/global-data-state.js @@ -79,12 +79,15 @@ function rejectSharedMemory (value, operation, seen = new Set()) { rejectSharedMemory(key, operation, seen) rejectSharedMemory(entry, operation, seen) } + return } else if (value instanceof Set) { for (const entry of value) rejectSharedMemory(entry, operation, seen) + return } // Include non-enumerable retained properties such as Error.cause. - for (const descriptor of Object.values(Object.getOwnPropertyDescriptors(value))) { - if ('value' in descriptor) rejectSharedMemory(descriptor.value, operation, seen) + for (const key of Object.getOwnPropertyNames(value)) { + const descriptor = Object.getOwnPropertyDescriptor(value, key) + if (descriptor && 'value' in descriptor) rejectSharedMemory(descriptor.value, operation, seen) } } @@ -98,29 +101,39 @@ function rejectSharedMemory (value, operation, seen = new Set()) { * @param {GlobalDataInputChanges | undefined} [params.globalDataInputChanges] */ export function createGlobalDataState ({ pages, previousGlobalDataBaseline, globalDataInputChanges }) { - const sourceIds = [...new Set(pages.map(page => page.sourceId))] - const sourceIdSet = new Set(sourceIds) const reset = !previousGlobalDataBaseline || globalDataInputChanges?.resetReason !== undefined let state = reset ? undefined : cloneState(previousGlobalDataBaseline.state, 'previous state') - const events = structuredClone(globalDataInputChanges?.events ?? []) - const previousIds = new Set(previousGlobalDataBaseline?.sourceIds) - const upsertedPaths = new Set(globalDataInputChanges?.upsertedPaths.map(path => resolve(path))) + const events = globalDataInputChanges ? structuredClone(globalDataInputChanges.events) : [] + const sourceIdSet = new Set() + /** @type {GlobalDataChanges} */ + let changes + if (reset) { + for (const page of pages) sourceIdSet.add(page.sourceId) + changes = { kind: 'reset', reason: globalDataInputChanges?.resetReason ?? 'initial', events } + } else { + const previousIds = new Set(previousGlobalDataBaseline.sourceIds) + const upsertedPaths = new Set() + for (const path of globalDataInputChanges?.upsertedPaths ?? []) upsertedPaths.add(resolve(path)) + changes = { kind: 'delta', upserted: [], removed: [], events } + for (const page of pages) { + const sourceId = page.sourceId + sourceIdSet.add(sourceId) + if (!previousIds.has(sourceId) || (upsertedPaths.size > 0 && upsertedPaths.has(resolve(page.pageInfo.pageFile.filepath)))) { + changes.upserted.push(page) + } + } + for (const sourceId of previousIds) { + if (!sourceIdSet.has(sourceId)) changes.removed.push(sourceId) + } + } + const sourceIds = [...sourceIdSet] /** @type {GlobalDataFunctionParams} */ const context = { pages, - previousState: reset ? undefined : cloneState(state, 'previous state'), - changes: reset - ? { kind: 'reset', reason: globalDataInputChanges?.resetReason ?? 'initial', events } - : { - kind: 'delta', - upserted: pages.filter(page => { - const path = resolve(page.pageInfo.pageFile.filepath) - return upsertedPaths.has(path) || !previousIds.has(page.sourceId) - }), - removed: [...previousIds].filter(id => !sourceIdSet.has(id)), - events, - }, + // The private snapshot is already validated; only callback isolation remains. + previousState: reset ? undefined : structuredClone(state), + changes, setState (next) { state = cloneState(next, 'setState(next)') }, diff --git a/lib/build-pages/global-data-state.test.js b/lib/build-pages/global-data-state.test.js index cd45a0b3..db392f57 100644 --- a/lib/build-pages/global-data-state.test.js +++ b/lib/build-pages/global-data-state.test.js @@ -9,6 +9,7 @@ import { tmpdir } from 'node:os' import { join } from 'node:path' import { createGlobalDataState } from './global-data-state.js' import { buildPages, buildPagesDirect } from './index.js' +import { PageData } from './page-data.js' import { identifyPages } from '../identify-pages.js' import { classifyWatchEvent } from '../watch-plan.js' import { resolveGlobalData } from './resolve-vars.js' @@ -49,6 +50,64 @@ test('candidate and previousState are isolated, setState snapshots immediately a assert.equal(state.date.getTime(), 0) }) +test('unchanged candidates retain the initial snapshot without rereading source getters', () => { + let reads = 0 + const nested = { count: 1 } + const state = { get nested () { reads++; return nested } } + const previousGlobalDataBaseline = { state, sourceIds: [] } + const transaction = createGlobalDataState({ pages: [], previousGlobalDataBaseline }) + nested.count = 99 + const previous = /** @type {{ nested: { count: number } }} */ (transaction.context.previousState) + assert.equal(previous.nested.count, 1) + previous.nested.count = 2 + const candidate = transaction.getBaseline().state + assert.deepEqual(candidate, { nested: { count: 1 } }) + assert.equal(transaction.getBaseline().state, candidate) + assert.equal(reads, 1) +}) + +test('membership reads each source ID once and skips paths without upsert candidates', async t => { + const { src } = await fixture(t) + await writeFile(join(src, 'other.md'), 'Other') + const site = await identifyPages(src) + let sourceIdReads = 0 + let filepathReads = 0 + const pages = site.pages.map(pageInfo => { + const page = new PageData({ + pageInfo, + globalVars: {}, + globalStyle: undefined, + globalClient: undefined, + defaultStyle: null, + defaultClient: null, + builderOptions: {}, + }) + const sourceId = page.sourceId + const filepath = pageInfo.pageFile.filepath + Object.defineProperty(page, 'sourceId', { get () { sourceIdReads++; return sourceId } }) + Object.defineProperty(pageInfo.pageFile, 'filepath', { get () { filepathReads++; return filepath } }) + return page + }) + const previousGlobalDataBaseline = { state: undefined, sourceIds: ['page.js', 'removed.md', 'removed.md'] } + const delta = createGlobalDataState({ pages, previousGlobalDataBaseline }) + assert.equal(sourceIdReads, pages.length) + assert.equal(filepathReads, 0) + assert.deepEqual(delta.context.changes, { + kind: 'delta', upserted: [pages.find(page => page.pageInfo.pageFile.relname === 'other.md')], removed: ['removed.md'], events: [], + }) + assert.deepEqual(delta.getBaseline().sourceIds, site.pages.map(page => page.pageFile.relname)) + + sourceIdReads = 0 + const reset = createGlobalDataState({ + pages, + previousGlobalDataBaseline, + globalDataInputChanges: { resetReason: 'recovery', upsertedPaths: [], events: [] }, + }) + assert.equal(sourceIdReads, pages.length) + assert.equal(filepathReads, 0) + assert.deepEqual(reset.context.changes, { kind: 'reset', reason: 'recovery', events: [] }) +}) + test('state supports cycles and uncloneable updates fail actionably without replacing the candidate', () => { const transaction = createGlobalDataState({ pages: [] }) const state = { self: /** @type {unknown} */ (null), value: 1n } @@ -165,6 +224,18 @@ test('resets discard state even without setState and do not try to clone invalid assert.deepEqual(delta.context.changes, { kind: 'delta', upserted: [], removed: [], events: [] }) }) +test('direct builds reuse page metadata when its filepath is already normalized', async t => { + const { src, dest } = await fixture(t) + const globalDataPath = join(src, 'global.data.js') + await writeFile(globalDataPath, `export let pageInfo + export default ({ pages }) => { pageInfo = pages[0].pageInfo; return { summary: '' } }`) + const site = await identifyPages(src) + const result = await buildPagesDirect(src, dest, site) + assert.deepEqual(result.errors, []) + const globalDataModule = await import(globalDataPath) + assert.equal(globalDataModule.pageInfo, site.pages[0]) +}) + for (const [mode, build] of /** @type {const} */ ([['direct', buildPagesDirect], ['worker', buildPages]])) { test(`${mode}: reset and delta use eligible source membership, not output filters or generated pages`, async t => { const { src, dest } = await fixture(t) @@ -379,6 +450,29 @@ for (const [mode, build] of /** @type {const} */ ([['direct', buildPagesDirect], assert.deepEqual(stateOnly.report.watchDependencies?.globalDataFingerprints, next.report.watchDependencies?.globalDataFingerprints) }) + test(`${mode}: resets never read or transfer the discarded baseline`, async t => { + const { src, dest } = await fixture(t) + await writeFile(join(src, 'global.data.js'), `export default ({ previousState, changes }) => { + if (previousState !== undefined) throw new Error('reset exposed old state') + return { summary: changes } + }`) + const site = await identifyPages(src) + const previousGlobalDataBaseline = { + /** @returns {never} */ + get state () { throw new Error('discarded state must not be read or cloned') }, + /** @returns {never} */ + get sourceIds () { throw new Error('discarded membership must not be read or cloned') }, + } + const result = await build(src, dest, site, { + trackWatchDependencies: true, + previousGlobalDataBaseline, + globalDataInputChanges: { resetReason: 'recovery', upsertedPaths: [], events: [] }, + }) + assert.deepEqual(result.errors, []) + assert.deepEqual(result.report.globalDataBaseline, { state: undefined, sourceIds: ['page.js'] }) + assert.deepEqual(JSON.parse(await readFile(join(dest, 'index.html'), 'utf8')), { kind: 'reset', reason: 'recovery', events: [] }) + }) + test(`${mode}: successful watch builds emit candidates while cold reports keep state private`, async t => { const { src, dest } = await fixture(t) await writeFile(join(src, 'global.data.js'), 'export default ({ setState }) => { setState({ candidate: true }); return { summary: \'\' } }') diff --git a/lib/build-pages/index.js b/lib/build-pages/index.js index 31fcd415..7ec2bdfe 100644 --- a/lib/build-pages/index.js +++ b/lib/build-pages/index.js @@ -476,7 +476,7 @@ export function buildPages (src, dest, siteData, opts) { previousWatchDependencies: opts?.previousWatchDependencies, trackWatchDependencies: opts?.trackWatchDependencies, previousPageOutputCache: opts?.previousPageOutputCache, - previousGlobalDataBaseline: opts?.previousGlobalDataBaseline, + previousGlobalDataBaseline: opts?.globalDataInputChanges?.resetReason === undefined ? opts?.previousGlobalDataBaseline : undefined, globalDataInputChanges: opts?.globalDataInputChanges, } @@ -622,10 +622,12 @@ export async function buildPagesDirect (_src, dest, siteData, opts) { } // Mix in resolveVars, renderInnerPage and renderFullPage methods for concrete pages. - const concretePages = await pMap(siteData.pages, pageInfo => initPageData({ - ...pageInfo, - pageFile: { ...pageInfo.pageFile, filepath: resolve(pageInfo.pageFile.filepath) }, - }), { concurrency: MAX_CONCURRENCY }) + const concretePages = await pMap(siteData.pages, pageInfo => { + const filepath = resolve(pageInfo.pageFile.filepath) + return initPageData(filepath === pageInfo.pageFile.filepath + ? pageInfo + : { ...pageInfo, pageFile: { ...pageInfo.pageFile, filepath } }) + }, { concurrency: MAX_CONCURRENCY }) if (result.errors.length > 0) return result From 1a37f6688d9da8d5138c6b40827463765deea937 Mon Sep 17 00:00:00 2001 From: Bret Comnes Date: Tue, 15 Sep 2026 11:03:26 -0700 Subject: [PATCH 08/14] Simplify global data plumbing and defer re-export tracking --- docs/data/README.md | 3 +- index.js | 2 +- lib/build-pages/global-data-state.test.js | 5 +- lib/build-pages/index.js | 1 - lib/build-pages/resolve-vars.js | 10 +- lib/helpers/find-dependencies.js | 71 ---- lib/helpers/find-dependencies.test.js | 343 ------------------ site/globals/global.data.ts | 8 +- .../incremental-global-data/index.test.js | 13 +- 9 files changed, 24 insertions(+), 432 deletions(-) delete mode 100644 lib/helpers/find-dependencies.js delete mode 100644 lib/helpers/find-dependencies.test.js diff --git a/docs/data/README.md b/docs/data/README.md index 73dc7a19..20c15639 100644 --- a/docs/data/README.md +++ b/docs/data/README.md @@ -426,4 +426,5 @@ For a larger example that also caches heading links, see `site/globals/global.da - State lasts for the current watch session and is retained only after a successful build. Always handle resets, including after producer or settings changes and failed builds. - Store cloneable records, arrays, or maps—not `PageData` instances, functions, or shared memory. - An upsert may repeat even when content is unchanged. Replace its cached entry, or delete it if the page no longer belongs in your index. -- Relative static imports and re-exports within the watched source tree are tracked. Restart watching when other inputs change, such as arbitrary file reads, environment variables, or network data. +- Relative static imports within the watched source tree are tracked. Re-exports (`export … from`) are not tracked yet ([#328](https://github.com/bcomnes/domstack/issues/328)); use an explicit import followed by a local export, or restart watching after those inputs change. +- Restart watching when other inputs change, such as arbitrary file reads, environment variables, or network data. diff --git a/index.js b/index.js index 2d036a43..d4e9c656 100644 --- a/index.js +++ b/index.js @@ -38,7 +38,7 @@ import ignore from 'ignore' import { watch as cpxWatch } from 'cpx2' import { inspect } from 'util' import { createServer } from '@domstack/sync' -import { findDependencies as find } from './lib/helpers/find-dependencies.js' +import { find } from '@11ty/dependency-tree-typescript' import { assertInsideDest } from './lib/helpers/path.js' import { getCopyGlob } from './lib/build-static/index.js' diff --git a/lib/build-pages/global-data-state.test.js b/lib/build-pages/global-data-state.test.js index db392f57..7eb2eb51 100644 --- a/lib/build-pages/global-data-state.test.js +++ b/lib/build-pages/global-data-state.test.js @@ -557,8 +557,9 @@ test('resolveGlobalData retains object and non-serializable public return values const globalDataPath = join(src, 'global.data.js') await writeFile(globalDataPath, 'export default { render: () => \'public function\', value: 1 }') const imported = await import(globalDataPath) - const result = await resolveGlobalData({ globalDataPath, pages: [] }) + const { context } = createGlobalDataState({ pages: [] }) + const result = await resolveGlobalData({ globalDataPath, context }) assert.equal(result, imported.default) assert.equal(Reflect.get(result, 'render')(), 'public function') - assert.deepEqual(await resolveGlobalData({ pages: [] }), {}) + assert.deepEqual(await resolveGlobalData({ context }), {}) }) diff --git a/lib/build-pages/index.js b/lib/build-pages/index.js index 7ec2bdfe..01988069 100644 --- a/lib/build-pages/index.js +++ b/lib/build-pages/index.js @@ -640,7 +640,6 @@ export async function buildPagesDirect (_src, dest, siteData, opts) { }) const globalData = /** @type {Record} */ (await resolveGlobalData({ globalDataPath: siteData.globalData?.filepath, - pages: concretePages, context: globalDataState.context, })) const changedGlobalDataKeys = watchDependencyTracker.updateGlobalDataFingerprints( diff --git a/lib/build-pages/resolve-vars.js b/lib/build-pages/resolve-vars.js index a5e05bef..91d46074 100644 --- a/lib/build-pages/resolve-vars.js +++ b/lib/build-pages/resolve-vars.js @@ -1,10 +1,9 @@ /** - * @import { PageData } from './page-data.js' + * @import { GlobalDataFunctionParams } from './index.js' */ import { isFunction, isObject, isPlainObject } from '../helpers/type-guards.js' -import { createGlobalDataState } from './global-data-state.js' /** * Resolve an object-or-function vars export. @@ -55,18 +54,17 @@ export async function resolveVars ({ * * @param {object} params * @param {string | undefined} [params.globalDataPath] - Path to the global.data file. - * @param {PageData[]} params.pages - Initialized source-backed PageData array. - * @param {GlobalDataFunctionParams} [params.context] - State context prepared by the page phase. + * @param {GlobalDataFunctionParams} params.context - Callback context prepared by the page phase. * @returns {Promise} */ -export async function resolveGlobalData ({ globalDataPath, pages, context }) { +export async function resolveGlobalData ({ globalDataPath, context }) { if (!globalDataPath) return {} const imported = await import(globalDataPath) const maybeGlobalData = imported.default if (isFunction(maybeGlobalData)) { - const result = await maybeGlobalData(context ?? createGlobalDataState({ pages }).context) + const result = await maybeGlobalData(context) if (isObject(result)) return result throw new Error('global.data default export function must return an object') } else if (isObject(maybeGlobalData)) { diff --git a/lib/helpers/find-dependencies.js b/lib/helpers/find-dependencies.js deleted file mode 100644 index 7397dd60..00000000 --- a/lib/helpers/find-dependencies.js +++ /dev/null @@ -1,71 +0,0 @@ -import { stat } from 'node:fs/promises' -import { createRequire } from 'node:module' -import { extname, resolve } from 'node:path' - -// Resolve the analyzer's own runtime dependencies, even when they are not hoisted. -// Its TS wrapper does not expose the ESM finder's parserOverride option. -const require = createRequire(import.meta.resolve('@11ty/dependency-tree-typescript')) -/** @type {{ find: (filepath: string, options: { parserOverride: typeof parserOverride }) => Promise }} */ -const { find } = require(require.resolve('@11ty/dependency-tree-esm')) -const { Parser } = require('acorn') -const { tsPlugin } = require('@sveltejs/acorn-typescript') -const parser = Parser.extend(tsPlugin()) - -const parserOverride = { - /** @param {string} contents */ - parse (contents) { - const ast = parser.parse(contents, { sourceType: 'module', ecmaVersion: 'latest' }) - return { - ...ast, - body: ast.body.filter(node => { - // Node erases declaration-level types only. Inline type specifiers leave - // an import/export (possibly empty) that still executes the source module. - if (node.type === 'ImportDeclaration') { - return !('importKind' in node && node.importKind === 'type') - } - if (node.type === 'ExportNamedDeclaration' || node.type === 'ExportAllDeclaration') { - return !('exportKind' in node && node.exportKind === 'type') - } - return true - }).map(node => { - if ((node.type === 'ExportNamedDeclaration' || node.type === 'ExportAllDeclaration') && node.source) { - // Only the dependency walk sees this projection. Keep source and import - // attributes intact so re-exported JSON/CSS remain attributed leaves. - return { ...node, type: 'ImportDeclaration' } - } - return node - }), - } - }, -} - -/** @param {string} filepath */ -async function assertFile (filepath) { - if (!(await stat(filepath)).isFile()) { - throw new Error(`Cannot analyze dependency "${filepath}": expected a file`) - } -} - -/** - * Find transitive static ESM imports and re-exports in JS/TS, using the existing - * analyzer's cwd-relative path strings and bare-specifier exclusion semantics. - * Declaration-level type-only imports/exports are excluded; inline type-only - * specifier lists retain runtime edges. JSON roots have no dependencies; - * attributed imports/re-exports remain leaves. - * Missing files and parse errors reject so callers can invalidate conservatively - * instead of installing an incomplete dependency index. No results are cached. - * - * @param {string} filepath - * @returns {Promise} - */ -export async function findDependencies (filepath) { - const root = resolve(filepath) - await assertFile(root) - if (extname(root) === '.json') return [] - - const dependencies = await find(root, { parserOverride }) - // The upstream finder silently accepts missing imports (including attributed - // leaves). Validate every edge before returning anything to the caller. - await Promise.all(dependencies.map(assertFile)) - return [...new Set(dependencies)].filter(dependency => resolve(dependency) !== root) -} diff --git a/lib/helpers/find-dependencies.test.js b/lib/helpers/find-dependencies.test.js deleted file mode 100644 index 83963394..00000000 --- a/lib/helpers/find-dependencies.test.js +++ /dev/null @@ -1,343 +0,0 @@ -/** - * @import { TestContext } from 'node:test' - * @import { WatchSnapshot } from '../watch-plan.js' - */ -import assert from 'node:assert/strict' -import { mkdir, mkdtemp, readFile, rm, writeFile } from 'node:fs/promises' -import { stripTypeScriptTypes } from 'node:module' -import { tmpdir } from 'node:os' -import { basename, dirname, join, relative, resolve } from 'node:path' -import { test } from 'node:test' -import { pathToFileURL } from 'node:url' -import { find } from '@11ty/dependency-tree-typescript' -import { classifyWatchEvent, planWatchBatch } from '../watch-plan.js' -import { findDependencies } from './find-dependencies.js' - -/** @param {TestContext} t @param {Record} files */ -async function fixture (t, files) { - const directory = await mkdtemp(join(tmpdir(), 'domstack-dependencies-')) - t.after(() => rm(directory, { recursive: true, force: true })) - for (const [name, source] of Object.entries(files)) { - const filepath = join(directory, name) - await mkdir(dirname(filepath), { recursive: true }) - await writeFile(filepath, source) - } - return directory -} - -/** @param {string[]} paths */ -function absolutePaths (paths) { - assert.ok(paths.every(path => typeof path === 'string')) - return paths.map(path => resolve(path)).sort() -} - -test('follows mixed JS/TS imports and named, star, and namespace re-exports transitively', async t => { - const directory = await fixture(t, { - 'root.js': 'import { value } from "./barrel.ts"; export default value;', - 'barrel.ts': 'export { value } from "./nested/named.mjs"; export * as namespace from "./namespace.mts";', - 'nested/named.mjs': 'export * from "../leaf.cts";', - 'namespace.mts': 'export { value } from "./leaf.cts";', - 'leaf.cts': 'import "./side-effect.js"; export const value: number = 1;', - 'side-effect.js': 'export const loaded = true;', - }) - const root = join(directory, 'root.js') - const expected = ['barrel.ts', 'nested/named.mjs', 'namespace.mts', 'leaf.cts', 'side-effect.js'].map(name => join(directory, name)).sort() - assert.deepEqual(absolutePaths(await findDependencies(root)), expected) - assert.deepEqual(absolutePaths(await findDependencies(relative(process.cwd(), root))), expected) - assert.equal((await findDependencies(root)).length, expected.length, 'shared leaves are deduplicated') -}) - -test('handles cycles and self-re-exports without returning the root', async t => { - const directory = await fixture(t, { - 'root.ts': 'export * from "./barrel.ts"; export * from "./root.ts";', - 'barrel.ts': 'export * from "./root.ts"; import "./leaf.js";', - 'leaf.js': 'import "./barrel.ts";', - }) - assert.deepEqual(absolutePaths(await findDependencies(join(directory, 'root.ts'))), [join(directory, 'barrel.ts'), join(directory, 'leaf.js')]) -}) - -test('uses parsed syntax, not import-like comments, strings, templates, or regular expressions', async t => { - const directory = await fixture(t, { - 'root.ts': [ - '// export * from "./missing-comment.js";', - '/* import "./missing-block.js"; */', - 'const text: string = \'export { x } from "./missing-string.js"\';', - 'const template = `export * from "./missing-template.js"`;', - 'const regex = /export .* from "missing-regex"/;', - 'export {', - ' value as renamed // a multiline re-export', - '} from "./le\\u0061f.js";', - 'export { text, template, regex };', - ].join('\n'), - 'leaf.js': 'export const value = 1;', - }) - assert.deepEqual(absolutePaths(await findDependencies(join(directory, 'root.ts'))), [join(directory, 'leaf.js')]) -}) - -test('ignores missing declaration-level type-only imports and re-exports erased by Node', async t => { - const directory = await fixture(t, { - 'root.ts': 'export * from "./barrel.ts";', - 'barrel.ts': '', - }) - for (const source of [ - 'import type { A } from "./missing.ts";', - 'import type Default from "./missing.ts";', - 'import type * as Types from "./missing.ts";', - - 'export type { A } from "./missing.ts";', - 'export type * from "./missing.ts";', - 'export type * as Types from "./missing.ts";', - - 'import type {} from "./missing.ts";', - 'export type {} from "./missing.ts";', - ]) { - assert.equal(stripTypeScriptTypes(source).trim(), '', source) - await writeFile(join(directory, 'barrel.ts'), source) - assert.deepEqual(await findDependencies(join(directory, 'barrel.ts')), [], source) - assert.deepEqual(absolutePaths(await findDependencies(join(directory, 'root.ts'))), [join(directory, 'barrel.ts')], source) - } -}) - -test('all-type specifier lists retain side effects and transitive runtime dependencies', async t => { - for (const source of [ - 'import { type A } from "./registration.ts";', - 'import { type A, type B as C } from "./registration.ts";', - 'export { type A } from "./registration.ts";', - 'export { type A, type B as C } from "./registration.ts";', - ]) { - await t.test(source, async t => { - const directory = await fixture(t, { - 'root.ts': source, - 'stripped.mjs': stripTypeScriptTypes(source), - 'registration.ts': 'import "./side-effect.ts"; export interface A { name: string } export type B = number;', - 'side-effect.ts': 'import { writeFileSync } from "node:fs"; writeFileSync(new URL("./executed.txt", import.meta.url), "registered");', - }) - const root = join(directory, 'root.ts') - const marker = join(directory, 'executed.txt') - const expected = ['registration.ts', 'side-effect.ts'].map(name => join(directory, name)) - assert.deepEqual(absolutePaths(await findDependencies(root)), expected) - assert.deepEqual(absolutePaths(await findDependencies(join(directory, 'stripped.mjs'))), expected, 'matches Node type stripping') - await assert.rejects(readFile(marker), { code: 'ENOENT' }, 'analysis must not execute the modules') - await import(pathToFileURL(root).href) - assert.equal(await readFile(marker, 'utf8'), 'registered', 'Node executes the transitive side effect') - await rm(join(directory, 'side-effect.ts')) - await assert.rejects(findDependencies(root), { code: 'ENOENT' }, 'missing transitive runtime edges still fail conservatively') - }) - } -}) - -test('does not traverse declaration-level type-only imports/re-exports into runtime dependencies', async t => { - const directory = await fixture(t, { - 'root.ts': 'import type { A } from "./types.ts"; export type { A } from "./types.ts";', - 'types.ts': 'import "./missing-library.js"; export interface A { name: string }', - }) - assert.deepEqual(await findDependencies(join(directory, 'root.ts')), []) -}) - -test('mixed value/type imports and re-exports retain the source and its transitive runtime edges', async t => { - const directory = await fixture(t, { - 'root.ts': '', - 'values.ts': 'import "./leaf.js"; export interface Shape { name: string } export const value = 1; export default value;', - 'leaf.js': 'export const loaded = true;', - }) - for (const source of [ - 'import { type Shape, value } from "./values.ts";', - 'import { value, type Shape } from "./values.ts";', - 'import value, { type Shape } from "./values.ts";', - 'export { type Shape, value } from "./values.ts";', - 'export { value, type Shape } from "./values.ts";', - ]) { - await writeFile(join(directory, 'root.ts'), source) - assert.deepEqual(absolutePaths(await findDependencies(join(directory, 'root.ts'))), ['leaf.js', 'values.ts'].map(name => join(directory, name)), source) - } - await rm(join(directory, 'values.ts')) - await assert.rejects(findDependencies(join(directory, 'root.ts')), { code: 'ENOENT' }) -}) - -test('preserves side-effect imports and empty named imports/re-exports as runtime edges', async t => { - const directory = await fixture(t, { - 'root.ts': '', - 'runtime.ts': 'import "./leaf.js"; export const value = 1;', - 'leaf.js': 'export const loaded = true;', - }) - const sources = [ - 'import "./runtime.ts";', - 'import {} from "./runtime.ts";', - 'export {} from "./runtime.ts";', - ] - for (const source of sources) { - await writeFile(join(directory, 'root.ts'), source) - assert.deepEqual(absolutePaths(await findDependencies(join(directory, 'root.ts'))), ['leaf.js', 'runtime.ts'].map(name => join(directory, name)), source) - } - await rm(join(directory, 'runtime.ts')) - for (const source of sources) { - await writeFile(join(directory, 'root.ts'), source) - await assert.rejects(findDependencies(join(directory, 'root.ts')), { code: 'ENOENT' }, source) - } -}) - -test('does not confuse value bindings named type with type-only specifiers', async t => { - const directory = await fixture(t, { - 'root.ts': 'import { type as value } from "./values.ts"; export { type } from "./values.ts";', - 'values.ts': 'export const type = 1;', - }) - assert.deepEqual(absolutePaths(await findDependencies(join(directory, 'root.ts'))), [join(directory, 'values.ts')]) -}) - -test('preserves existing import path strings and bare-import exclusion semantics', async t => { - const directory = await fixture(t, { - 'root.js': 'import "node:fs"; import "uninstalled-package"; import "#alias"; import "./leaf.js";', - 'barrel.js': 'export * from "uninstalled-package"; export { readFile } from "node:fs"; export * from "./leaf.js";', - 'leaf.js': 'export const value = 1;', - }) - const expected = await find(join(directory, 'root.js')) - assert.deepEqual(await findDependencies(join(directory, 'root.js')), expected) - assert.deepEqual(await findDependencies(join(directory, 'barrel.js')), expected) - assert.deepEqual(await find(join(directory, 'barrel.js')), [], 'the original analyzer is not patched globally') -}) - -test('returns JSON and other attributed leaves as path strings, without parsing them as modules', async t => { - const directory = await fixture(t, { - 'root.js': 'import data from "./data.json" with { type: "json" }; export * from "./barrel.ts";', - 'barrel.ts': 'export { default as more } from "./more.json" with { type: "json" }; export { default as css } from "./style.css" with { type: "css" };', - 'data.json': '{ "value": 1, "text": "export * from missing" }', - 'more.json': '{ "value": 2 }', - 'style.css': 'body { color: red; }', - }) - const dependencies = await findDependencies(join(directory, 'root.js')) - assert.deepEqual(absolutePaths(dependencies), ['barrel.ts', 'data.json', 'more.json', 'style.css'].map(name => join(directory, name))) - assert.deepEqual(await findDependencies(join(directory, 'data.json')), []) -}) - -test('rejects missing roots and unresolved direct, transitive, and attributed dependencies', async t => { - const directory = await fixture(t, { - 'root.js': 'import "./barrel.ts";', - 'barrel.ts': '', - }) - await assert.rejects(findDependencies(join(directory, 'missing.js')), { code: 'ENOENT' }) - await assert.rejects(findDependencies(join(directory, 'missing.json')), { code: 'ENOENT' }) - for (const source of [ - 'import "./missing.js";', - 'import { type A } from "./missing.ts";', - 'import { type A, type B as C } from "./missing.ts";', - 'export { type A } from "./missing.ts";', - 'export { type A, type B as C } from "./missing.ts";', - 'export { value } from "./missing.js";', - 'export * from "./missing.js";', - 'export * as values from "./missing.js";', - 'import data from "./missing.json" with { type: "json" };', - 'export { default as data } from "./missing.json" with { type: "json" };', - ]) { - await writeFile(join(directory, 'barrel.ts'), source) - await assert.rejects(findDependencies(join(directory, 'barrel.ts')), { code: 'ENOENT' }, source) - await assert.rejects(findDependencies(join(directory, 'root.js')), { code: 'ENOENT' }, source) - } -}) - -test('propagates parser errors and rejects directories rather than returning incomplete results', async t => { - const directory = await fixture(t, { - 'root.js': 'export * from "./broken.ts";', - 'broken.ts': 'export const value: = ;', - 'directory.js': 'export { default as data } from "./folder" with { type: "json" };', - 'folder/leaf.js': '', - }) - await assert.rejects(findDependencies(join(directory, 'root.js')), SyntaxError) - await assert.rejects(findDependencies(directory), /expected a file/) - await assert.rejects(findDependencies(join(directory, 'directory.js')), /expected a file/) -}) - -test('reanalyzes edited barrel edges and notices removed leaves on later calls', async t => { - const directory = await fixture(t, { - 'root.js': 'export * from "./barrel.js";', - 'barrel.js': 'export * from "./first.js";', - 'first.js': 'export const value = 1;', - 'second.js': 'export const value = 2;', - }) - const root = join(directory, 'root.js') - assert.deepEqual(absolutePaths(await findDependencies(root)), ['barrel.js', 'first.js'].map(name => join(directory, name))) - await writeFile(join(directory, 'barrel.js'), 'export * from "./second.js";') - assert.deepEqual(absolutePaths(await findDependencies(root)), ['barrel.js', 'second.js'].map(name => join(directory, name))) - await rm(join(directory, 'second.js')) - await assert.rejects(findDependencies(root), { code: 'ENOENT' }) -}) - -/** @param {string} directory */ -function watchFixture (directory) { - const filepath = join(directory, 'feed.template.js') - const template = { - templateFile: { root: directory, filepath, relname: basename(filepath), basename: basename(filepath), parentName: '.' }, - path: '', - outputName: 'feed.xml', - } - const leaf = join(directory, 'leaf.js') - /** @type {WatchSnapshot} */ - const state = { - siteData: { pages: [], templates: [template], pagesFiles: [], layouts: {} }, - layoutDepMap: new Map(), - layoutPageMap: new Map(), - pageFileMap: new Map(), - layoutFileMap: new Map(), - pageDepMap: new Map(), - templateDepMap: new Map([[leaf, new Set([template])]]), - pagesFileDepMap: new Map(), - pagesFileLayoutMap: new Map(), - globalDataDepPaths: new Set(), - globalVarsDepPaths: new Set(), - markdownDepPaths: new Set(), - dependencyAnalysisFailed: false, - pageBuildFailed: false, - esbuildEntryPoints: new Set([leaf]), - } - return { state, leaf, template } -} - -test('a barrel leaf shared with a template and browser role still invalidates each producer index', async t => { - const directory = await fixture(t, { - 'producer.js': 'import { value } from "./barrel.ts"; export default value;', - 'barrel.ts': 'export * from "./named.js";', - 'named.js': 'export { value } from "./leaf.js";', - 'leaf.js': 'export const value = 1;', - }) - const root = join(directory, 'producer.js') - const original = new Set(absolutePaths(await find(root))) - const complete = new Set(absolutePaths(await findDependencies(root))) - const cases = /** @type {const} */ ([ - ['globalDataDepPaths', 'global-data-changed'], - ['globalVarsDepPaths', 'global-vars-changed'], - ['markdownDepPaths', 'markdown-settings-changed'], - ]) - for (const [role, reason] of cases) { - const { state, leaf, template } = watchFixture(directory) - const event = classifyWatchEvent('change', leaf) - state[role] = original - const stale = planWatchBatch(state, [event]) - assert.equal(stale.inputChanges.resetReason, undefined, 'another known role masks the missing producer edge') - state[role] = complete - const batch = planWatchBatch(state, [event]) - assert.equal(batch.inputChanges.resetReason, reason) - if (role === 'globalVarsDepPaths') { - assert.equal(batch.plan.kind, 'full') - } else { - assert.equal(batch.plan.kind, 'pages') - if (batch.plan.kind !== 'pages') throw new Error('Expected a page plan') - assert.deepEqual(batch.plan.templateFilterPaths, [template.templateFile.filepath]) - } - } -}) - -test('analysis errors let the caller select conservative recovery even for a known shared role', async t => { - const directory = await fixture(t, { 'producer.js': 'export * from "./missing.js";' }) - const { state, leaf } = watchFixture(directory) - try { - state.globalDataDepPaths = new Set(absolutePaths(await findDependencies(join(directory, 'producer.js')))) - } catch { - state.dependencyAnalysisFailed = true - } - const batch = planWatchBatch(state, [classifyWatchEvent('change', leaf)]) - assert.equal(batch.inputChanges.resetReason, 'dependency-analysis-failed') - assert.equal(batch.plan.kind, 'pages') - if (batch.plan.kind !== 'pages') throw new Error('Expected a page plan') - assert.equal(batch.plan.pageFilterPaths, null) - assert.equal(batch.plan.templateFilterPaths, null) - assert.equal(batch.plan.pagesFileFilterPaths, null) -}) diff --git a/site/globals/global.data.ts b/site/globals/global.data.ts index 0992128c..2c2173a9 100644 --- a/site/globals/global.data.ts +++ b/site/globals/global.data.ts @@ -1,8 +1,12 @@ import type { GlobalDataFunctionParams } from '../../types.ts' import type { DocsNavigationIndex, DocsPageVars } from '../layouts/docs/navigation.js' +import { cpus } from 'node:os' +import pMap from 'p-map' import { render } from 'fragtml' import { readDocsNavigationPage, projectDocsNavigation, docsIndex } from '../layouts/docs/navigation.js' +const MAX_CONCURRENCY = Math.min(cpus().length, 24) + export default async function ({ pages, previousState, changes, setState }: GlobalDataFunctionParams) { let index: DocsNavigationIndex let inputs: typeof pages @@ -19,13 +23,13 @@ export default async function ({ pages, previousState, changes, setState }: Glob default: throw new Error('Unhandled global-data changes', { cause: changes satisfies never }) } - await Promise.all(inputs.map(async page => { + await pMap(inputs, async page => { const sourceId = page.sourceId // Excludes the data-dependent index and renders docs without their layouts. const record = await readDocsNavigationPage(page) if (record) index.set(sourceId, record) else index.delete(sourceId) - })) + }, { concurrency: MAX_CONCURRENCY }) const docsNavigation = projectDocsNavigation(index.values()) const docsIndexHtml = render(docsIndex(docsNavigation)) diff --git a/test-cases/incremental-global-data/index.test.js b/test-cases/incremental-global-data/index.test.js index 99c43eb3..43a717da 100644 --- a/test-cases/incremental-global-data/index.test.js +++ b/test-cases/incremental-global-data/index.test.js @@ -97,16 +97,19 @@ test('H1, frontmatter title/metadata, companion vars, and transitive page/layout } }) -test('barrel re-export leaves upsert source pages shared with templates and reset producer inputs', options, async t => { +test.todo('re-exported page helpers shared with templates upsert affected source pages (https://github.com/bcomnes/domstack/issues/328)') +test.todo('named, star, and namespace re-exports of global-data helpers trigger an index reset (https://github.com/bcomnes/domstack/issues/328)') + +test('explicit imports through barrels upsert source pages shared with templates and reset producer inputs', options, async t => { const site = await fixture(t, { files: { 'code/page.js': "import { content } from '../page-barrel.js'; export const vars = { article: true, layout: 'article', title: 'Barrel page' }; export default () => content\n", - 'page-barrel.js': "export * from './page-named.js'\n", - 'page-named.js': "export { content } from './page-leaf.js'\n", + 'page-barrel.js': "import { content } from './page-named.js'; export { content }\n", + 'page-named.js': "import { content } from './page-leaf.js'; export { content }\n", 'page-leaf.js': "export const content = 'Barrel one'\n", 'shared.txt.template.js': "import { content } from './page-leaf.js'; export default () => content\n", - 'producer-middle.js': "export * from './producer-barrel.js'\n", - 'producer-barrel.js': "export { prefix } from './producer-leaf.js'\n", + 'producer-middle.js': "import { prefix } from './producer-barrel.js'; export { prefix }\n", + 'producer-barrel.js': "import { prefix } from './producer-leaf.js'; export { prefix }\n", }, }) await site.start() From 7c99a4bddef11a4fffa20fe37cdf39220e92f3aa Mon Sep 17 00:00:00 2001 From: Bret Comnes Date: Tue, 15 Sep 2026 11:16:04 -0700 Subject: [PATCH 09/14] Reduce redundant watch planning and membership scans --- index.js | 4 ++- lib/watch-plan.js | 67 ++++++++++++++++++++++++++++-------------- lib/watch-plan.test.js | 48 ++++++++++++++++++++++++++++++ 3 files changed, 96 insertions(+), 23 deletions(-) diff --git a/index.js b/index.js index d4e9c656..5ebe56cf 100644 --- a/index.js +++ b/index.js @@ -487,7 +487,9 @@ export class DomStack { const { plan, inputChanges } = planWatchBatch(snapshot, events) // Keep the existing precise bundle-membership path for a single event. // Mixed batches use the conservative rediscovery plan instead. - const singlePlan = events.length === 1 ? planWatchEvent(snapshot, event) : null + const singlePlan = events.length === 1 && event.type !== 'change' && event.convention?.bundleScope + ? planWatchEvent(snapshot, event) + : null await this.#executeWatchPlan( snapshot.pageBuildFailed ? { kind: 'full', message: 'Rediscovering and retrying all pages after the previous build failure...' } diff --git a/lib/watch-plan.js b/lib/watch-plan.js index a48aaeb9..416a0c0e 100644 --- a/lib/watch-plan.js +++ b/lib/watch-plan.js @@ -100,13 +100,12 @@ export function planWatchEvent (state, event) { // The page worker selects subscribers after comparing the resulting values. const globalDataChanged = convention?.change === 'pages' || state.globalDataDepPaths.has(filepath) if (globalDataChanged || markdownChanged || pages.size || templates.size || owners.size) { - return { - ...selectedPages([...pages]), - templateFilterPaths: [...templates].map(template => template.templateFile.filepath), - pagesFileFilterPaths: [...owners], - templates: [...templates], - ...(globalDataChanged ? { message: `"${name}" changed, rebuilding data subscribers...` } : {}), - } + const plan = selectedPages([...pages]) + plan.templates = [...templates] + plan.templateFilterPaths = plan.templates.map(template => template.templateFile.filepath) + plan.pagesFileFilterPaths = [...owners] + if (globalDataChanged) plan.message = `"${name}" changed, rebuilding data subscribers...` + return plan } if (state.esbuildEntryPoints.has(filepath) || state.esbuildDepPaths?.has(filepath)) { return { kind: 'skip', message: `"${name}" changed, esbuild will handle rebundling.` } @@ -151,12 +150,21 @@ export function planWatchBatch (state, events) { let resetReason = state.pageBuildFailed ? 'page-build-failed' : state.dependencyAnalysisFailed ? 'dependency-analysis-failed' : undefined - const sourcePaths = new Set(pagePaths(state.siteData.pages.filter(page => !page.generated))) + const sourcePaths = new Set(/** @type {string[]} */ ([])) + for (const page of state.siteData.pages) { + if (!page.generated) sourcePaths.add(page.pageFile.filepath) + } const upsertedPaths = new Set(/** @type {string[]} */ ([])) + let allSourcesIncluded = false for (const event of events) { const eventPlan = planWatchEvent(state, event) const structural = event.type === 'added' || event.type === 'removed' - const changePlan = structural ? planWatchEvent(state, { ...event, type: 'change' }) : eventPlan + const sourceEvent = structural && (event.name.endsWith('.md') || + (event.convention?.change === 'page' && !fileConventions.pageVars.names.includes(event.name)) || + sourcePaths.has(event.filepath)) + // Only source membership changes need mapped consumers; other structural + // events already invalidate every current source input. + const changePlan = sourceEvent ? planWatchEvent(state, { ...event, type: 'change' }) : eventPlan const reason = inputResetReason(state, event, changePlan) resetReason ??= reason plan = unionWatchPlans(plan, eventPlan) @@ -164,14 +172,18 @@ export function planWatchBatch (state, events) { plan = { kind: 'full', message: `"${event.name}" cannot be routed reliably, triggering full rebuild...` } } - const sourceEvent = event.name.endsWith('.md') || - (event.convention?.change === 'page' && !fileConventions.pageVars.names.includes(event.name)) || - sourcePaths.has(event.filepath) - const allInputs = changePlan.kind === 'full' || - (structural && !sourceEvent) || reason === 'unreliable-event' - const selected = changePlan.kind === 'pages' ? changePlan.pageFilterPaths : [] - for (const path of allInputs || selected === null ? sourcePaths : selected) { - if (sourcePaths.has(path)) upsertedPaths.add(path) + if (!allSourcesIncluded) { + const allInputs = changePlan.kind === 'full' || + (structural && !sourceEvent) || reason === 'unreliable-event' + const selected = changePlan.kind === 'pages' ? changePlan.pageFilterPaths : [] + if (allInputs || selected === null) { + for (const path of sourcePaths) upsertedPaths.add(path) + allSourcesIncluded = true + } else { + for (const path of selected) { + if (sourcePaths.has(path)) upsertedPaths.add(path) + } + } } if (structural && sourceEvent) upsertedPaths.add(event.filepath) } @@ -241,7 +253,10 @@ function unionWatchPlans (left, right) { /** @param {string[] | null} left @param {string[] | null} right */ function unionFilterPaths (left, right) { - return left === null || right === null ? null : [...new Set([...left, ...right])] + if (left === null || right === null) return null + const paths = new Set(left) + for (const path of right) paths.add(path) + return [...paths] } /** @@ -298,7 +313,7 @@ function pagePaths (pages) { /** @param {PageInfo[]} pages @returns {PagePlan} */ function selectedPages (pages) { - return { ...allPages(), pageFilterPaths: pagePaths(pages), templateFilterPaths: [], pagesFileFilterPaths: [], pages } + return { kind: 'pages', pageFilterPaths: pagePaths(pages), templateFilterPaths: [], pagesFileFilterPaths: [], pages } } /** @@ -310,8 +325,16 @@ function layoutConsumers (state, layoutNames) { for (const layoutName of layoutNames) { for (const page of state.layoutPageMap.get(layoutName) ?? []) pages.add(page) } - const owners = new Set([...state.pagesFileLayoutMap] - .filter(([, usedLayouts]) => [...usedLayouts].some(name => layoutNames.has(name))) - .map(([owner]) => owner)) + const owners = new Set(/** @type {string[]} */ ([])) + if (layoutNames.size > 0) { + for (const [owner, usedLayouts] of state.pagesFileLayoutMap) { + for (const name of usedLayouts) { + if (layoutNames.has(name)) { + owners.add(owner) + break + } + } + } + } return { pages, owners } } diff --git a/lib/watch-plan.test.js b/lib/watch-plan.test.js index 3b2ccc22..a5b0b153 100644 --- a/lib/watch-plan.test.js +++ b/lib/watch-plan.test.js @@ -81,6 +81,17 @@ test('layout maps target source pages and generated-page owners', () => { assert.ok(unused.message?.includes('did not match any rebuild rule')) }) +test('events without affected layouts do not scan generated-page layout memberships', t => { + const { state } = fixture() + const scan = t.mock.method(state.pagesFileLayoutMap, Symbol.iterator) + for (const name of ['page.md', 'page-helper.js', 'template-helper.js', 'client.jsx']) { + planWatchEvent(state, classifyWatchEvent('change', `/site/${name}`)) + } + assert.equal(scan.mock.callCount(), 0) + planWatchEvent(state, classifyWatchEvent('change', '/site/layout-helper.js')) + assert.equal(scan.mock.callCount(), 1) +}) + test('page, template, and owner inputs retain their targeted scopes', () => { const { state, home, other, owner, template } = fixture() const cases = [ @@ -438,6 +449,43 @@ test('structural layout, vars, unknown, template, and owner events conservativel } }) +test('non-source structural events do not resolve individual page consumers', t => { + const { state, home, other } = fixture() + const lookup = t.mock.method(state.pageDepMap, 'get') + for (const type of /** @type {const} */ (['added', 'removed'])) { + for (const name of ['page.vars.js', 'root.layout.js', 'feed.template.js', 'archive.pages.js', 'helper.js', 'client.jsx']) { + const batch = planWatchBatch(state, [classifyWatchEvent(type, `/site/${name}`)]) + assert.deepEqual(batch.inputChanges.upsertedPaths, [home.pageFile.filepath, other.pageFile.filepath]) + } + } + assert.equal(lookup.mock.callCount(), 0) + const batch = planWatchBatch(state, [classifyWatchEvent('added', '/site/new/page.md')]) + assert.deepEqual(batch.inputChanges.upsertedPaths, ['/site/new/page.md']) + assert.equal(lookup.mock.callCount(), 1, 'structural source events still resolve mapped consumers') +}) + +test('repeated all-source events preserve candidate order and later membership changes', () => { + const { state, home, other } = fixture() + const events = [ + classifyWatchEvent('change', '/site/page-helper.js'), + classifyWatchEvent('added', '/site/first/post.md'), + classifyWatchEvent('added', '/site/feed.template.js'), + classifyWatchEvent('removed', '/site/root.layout.js'), + classifyWatchEvent('added', '/site/second/post.md'), + classifyWatchEvent('change', '/site/global.vars.js'), + classifyWatchEvent('removed', '/site/first/post.md'), + ] + const before = structuredClone(state) + const batch = planWatchBatch(state, events) + assert.equal(batch.plan.kind, 'full') + assert.deepEqual(batch.inputChanges, { + upsertedPaths: [other.pageFile.filepath, '/site/first/post.md', home.pageFile.filepath, '/site/second/post.md'], + resetReason: 'global-vars-changed', + events, + }) + assert.deepEqual(state, before) +}) + test('batch bundle restarts become full without resetting, including when mixed with page or full plans', () => { for (const name of ['client.jsx', 'root.layout.css', 'global.css', 'service-worker.js']) { const { state, home, other } = fixture() From 934b30c66e74ae7f7aedd6659412fb199670ec21 Mon Sep 17 00:00:00 2001 From: Bret Comnes Date: Tue, 15 Sep 2026 11:49:26 -0700 Subject: [PATCH 10/14] Simplify incremental state tests and watch plan construction --- index.js | 29 +- .../global-data-state-types.test.ts | 98 +-- lib/build-pages/global-data-state.test.js | 638 +++++------------- lib/watch-plan.js | 24 +- lib/watch-plan.test.js | 65 +- site/layouts/docs/navigation-watch.test.js | 403 +++-------- site/layouts/docs/navigation.test.js | 46 +- test-cases/incremental-global-data/helpers.js | 211 ++---- .../incremental-global-data/index.test.js | 567 ++++------------ test-cases/page-outputs/watch.test.js | 47 +- 10 files changed, 550 insertions(+), 1578 deletions(-) diff --git a/index.js b/index.js index 5ebe56cf..0c4ef9a5 100644 --- a/index.js +++ b/index.js @@ -474,16 +474,33 @@ export class DomStack { } } + /** @param {WatchEvent[]} events */ + #filterWatchEvents (events) { + // Unknown inputs may be needed to recover after a failed build or analysis. + if (this.#pageBuildFailed || this.#dependencyAnalysisFailed) return events + + const dependencies = [ + this.#globalDataDepPaths, + this.#globalVarsDepPaths, + this.#markdownDepPaths, + this.#layoutDepMap, + this.#pageDepMap, + this.#templateDepMap, + this.#pagesFileDepMap, + this.#esbuildDepPaths, + ] + return events.filter(({ filepath }) => + isProcessedFile(filepath) || dependencies.some(paths => paths.has(filepath)) + ) + } + /** @param {WatchEvent[]} events */ async #handleWatchBatch (events) { const snapshot = this.#watchSnapshot() - events = events.filter(event => isProcessedFile(event.filepath) || this.#pageBuildFailed || this.#dependencyAnalysisFailed || - this.#globalDataDepPaths.has(event.filepath) || this.#globalVarsDepPaths.has(event.filepath) || - this.#markdownDepPaths.has(event.filepath) || this.#layoutDepMap.has(event.filepath) || - this.#pageDepMap.has(event.filepath) || this.#templateDepMap.has(event.filepath) || - this.#pagesFileDepMap.has(event.filepath) || this.#esbuildDepPaths.has(event.filepath)) + if (!snapshot) return + events = this.#filterWatchEvents(events) const event = events[0] - if (!snapshot || !event) return + if (!event) return const { plan, inputChanges } = planWatchBatch(snapshot, events) // Keep the existing precise bundle-membership path for a single event. // Mixed batches use the conservative rediscovery plan instead. diff --git a/lib/build-pages/global-data-state-types.test.ts b/lib/build-pages/global-data-state-types.test.ts index 8aa1db19..1ce600f9 100644 --- a/lib/build-pages/global-data-state-types.test.ts +++ b/lib/build-pages/global-data-state-types.test.ts @@ -1,75 +1,39 @@ -import type { - AsyncGlobalDataFunction, - GlobalDataFunction, - GlobalDataFunctionParams, - GlobalDataChanges, - GlobalDataDeltaChanges, - GlobalDataResetChanges, - PageData, - WatchEvent, -} from '../../types.ts' -import type { BuildPagesOptions, PageBuilderReport } from './index.js' -import type { GlobalDataBaseline, GlobalDataInputChanges } from './global-data-state.js' +import type { AsyncGlobalDataFunction, GlobalDataFunction } from '../../types.ts' type Vars = { title: string } type Data = { titles: string[] } type State = { titles: Map } -export function checkGlobalDataStateTypes (pages: PageData[], events: WatchEvent[]) { - const reset: GlobalDataResetChanges = { kind: 'reset', reason: 'initial', events } - const delta: GlobalDataDeltaChanges = { kind: 'delta', upserted: pages, removed: ['posts/removed.md'], events } - const changes: GlobalDataChanges = delta - const params: GlobalDataFunctionParams = { - pages, - previousState: undefined, - changes, - setState (next) { next.titles.set('page.md', 'Title') }, - } - const stateful: GlobalDataFunction = ({ pages, changes, previousState, setState }) => { - const titles = previousState?.titles ?? new Map() - if (changes.kind === 'reset') { - const reason: string = changes.reason - titles.set(reason, pages[0]?.vars.title ?? '') - // @ts-expect-error Reset changes do not contain a partial upsert list. - String(changes.upserted) - } else { - changes.upserted.forEach(page => { - const sourceId: string = page.sourceId - titles.set(sourceId, page.vars.title) - // @ts-expect-error Source IDs are read-only. - page.sourceId = 'other.md' - }) - changes.removed.forEach(sourceId => titles.delete(sourceId)) - // @ts-expect-error Delta changes do not have a reset reason. - String(changes.reason) +export const stateful: GlobalDataFunction = ({ pages, changes, previousState, setState }) => { + const titles = previousState?.titles ?? new Map() + if (changes.kind === 'reset') { + const reason: string = changes.reason + titles.set(reason, pages[0]?.vars.title ?? '') + // @ts-expect-error Reset changes do not contain a partial upsert list. + String(changes.upserted) + } else { + for (const page of changes.upserted) { + titles.set(page.sourceId, page.vars.title) + // @ts-expect-error Source IDs are read-only. + page.sourceId = 'other.md' } - setState({ titles }) - // @ts-expect-error State generic constrains updates independently of returned data. - setState({ titles: ['wrong'] }) - return { titles: [...titles.values()] } - } - const asyncStateful: AsyncGlobalDataFunction = async context => stateful(context) - // Existing generic positions and pages-only callback implementations remain valid. - const legacy: GlobalDataFunction = ({ pages }) => ({ titles: pages.map(page => page.vars.title) }) - const asyncLegacy: AsyncGlobalDataFunction = async ({ pages }) => ({ titles: pages.map(page => page.vars.title) }) - const defaultState: GlobalDataFunction = ({ previousState, setState }) => { - const state: unknown = previousState - // @ts-expect-error Default state is unknown until narrowed or explicitly parameterized. - setState(previousState.titles) - setState(state) - return {} + changes.removed.forEach(sourceId => titles.delete(sourceId)) + // @ts-expect-error Delta changes do not have a reset reason. + String(changes.reason) } - const baseline: GlobalDataBaseline = { state: { titles: new Map() }, sourceIds: ['posts/page.md'] } - const sourceIds: string[] = baseline.sourceIds - const removed: string[] = delta.removed - const input: GlobalDataInputChanges = { resetReason: undefined, upsertedPaths: ['/src/posts/page.md'], events } - const upsertedPaths: string[] = input.upsertedPaths - events.forEach(event => { - const filepath: string = event.filepath - upsertedPaths.push(filepath) - }) - const options: BuildPagesOptions = { previousGlobalDataBaseline: baseline, globalDataInputChanges: input } - const absent: BuildPagesOptions = { previousGlobalDataBaseline: null, globalDataInputChanges: undefined } - const report: PageBuilderReport = { pages: [], templates: [], globalDataBaseline: baseline } - return { reset, delta, params, stateful, asyncStateful, legacy, asyncLegacy, defaultState, options, absent, report, sourceIds, removed, upsertedPaths } + setState({ titles }) + // @ts-expect-error State is constrained independently of returned data. + setState({ titles: ['wrong'] }) + return { titles: [...titles.values()] } +} + +export const asyncStateful: AsyncGlobalDataFunction = async context => stateful(context) + +export const pagesOnly: GlobalDataFunction = ({ pages }) => ({ titles: pages.map(page => page.vars.title) }) + +export const unknownState: GlobalDataFunction = ({ previousState, setState }) => { + // @ts-expect-error Default state is unknown until narrowed or parameterized. + setState(previousState.titles) + setState(previousState) + return {} } diff --git a/lib/build-pages/global-data-state.test.js b/lib/build-pages/global-data-state.test.js index 7eb2eb51..3cf621c4 100644 --- a/lib/build-pages/global-data-state.test.js +++ b/lib/build-pages/global-data-state.test.js @@ -1,6 +1,5 @@ /** * @import { TestContext } from 'node:test' - * @import { BuildPagesFilterOptions } from './index.js' */ import assert from 'node:assert/strict' import { test } from 'node:test' @@ -9,553 +8,238 @@ import { tmpdir } from 'node:os' import { join } from 'node:path' import { createGlobalDataState } from './global-data-state.js' import { buildPages, buildPagesDirect } from './index.js' -import { PageData } from './page-data.js' import { identifyPages } from '../identify-pages.js' import { classifyWatchEvent } from '../watch-plan.js' import { resolveGlobalData } from './resolve-vars.js' -/** @param {TestContext} t */ -async function fixture (t) { - const dir = await mkdtemp(join(tmpdir(), 'domstack-global-state-')) - t.after(() => rm(dir, { recursive: true, force: true })) - const src = join(dir, 'src') - const dest = join(dir, 'public') +/** @param {TestContext} t @param {string} producer */ +async function fixture (t, producer) { + const root = await mkdtemp(join(tmpdir(), 'domstack-global-state-')) + const src = join(root, 'src') + const dest = join(root, 'public') + t.after(() => rm(root, { recursive: true, force: true })) await mkdir(src) await writeFile(join(src, 'package.json'), '{"type":"module"}') await writeFile(join(src, 'root.layout.js'), 'export default ({ children }) => children') await writeFile(join(src, 'page.js'), `export const vars = { title: 'Home', dataDeps: ['summary'] } export default ({ data }) => JSON.stringify(data.summary)`) + await writeFile(join(src, 'global.data.js'), producer) return { src, dest } } -test('candidate and previousState are isolated, setState snapshots immediately and last call wins', () => { - const state = { count: 1, map: new Map([['a', new Set([1])]]), date: new Date(0) } - const previousGlobalDataBaseline = { state, sourceIds: [] } - const transaction = createGlobalDataState({ pages: [], previousGlobalDataBaseline }) +test('previousState mutations require setState, which snapshots immediately and uses the last update', () => { + const state = { titles: new Map([['page.md', 'Original']]) } + const transaction = createGlobalDataState({ pages: [], previousGlobalDataBaseline: { state, sourceIds: [] } }) const previous = /** @type {typeof state} */ (transaction.context.previousState) - previous.count = 2 - previous.map.get('a')?.add(2) - previous.date.setTime(100) + previous.titles.set('page.md', 'Changed') assert.deepEqual(transaction.getBaseline().state, state) - assert.notEqual(transaction.getBaseline().state, state) transaction.context.setState(previous) - previous.count = 3 - assert.equal(Reflect.get(/** @type {object} */ (transaction.getBaseline().state), 'count'), 2) + previous.titles.clear() + assert.deepEqual(transaction.getBaseline().state, { titles: new Map([['page.md', 'Changed']]) }) + assert.deepEqual(state, { titles: new Map([['page.md', 'Original']]) }) transaction.context.setState({ final: true }) assert.deepEqual(transaction.getBaseline().state, { final: true }) transaction.context.setState(undefined) assert.equal(transaction.getBaseline().state, undefined) - assert.equal(state.count, 1) - assert.deepEqual(state.map.get('a'), new Set([1])) - assert.equal(state.date.getTime(), 0) }) -test('unchanged candidates retain the initial snapshot without rereading source getters', () => { +test('both retained and callback state are isolated from the caller, and source getters run once', () => { let reads = 0 const nested = { count: 1 } const state = { get nested () { reads++; return nested } } - const previousGlobalDataBaseline = { state, sourceIds: [] } - const transaction = createGlobalDataState({ pages: [], previousGlobalDataBaseline }) + const transaction = createGlobalDataState({ pages: [], previousGlobalDataBaseline: { state, sourceIds: [] } }) nested.count = 99 const previous = /** @type {{ nested: { count: number } }} */ (transaction.context.previousState) - assert.equal(previous.nested.count, 1) + assert.deepEqual(previous, { nested: { count: 1 } }) previous.nested.count = 2 - const candidate = transaction.getBaseline().state - assert.deepEqual(candidate, { nested: { count: 1 } }) - assert.equal(transaction.getBaseline().state, candidate) + assert.deepEqual(transaction.getBaseline().state, { nested: { count: 1 } }) assert.equal(reads, 1) }) -test('membership reads each source ID once and skips paths without upsert candidates', async t => { - const { src } = await fixture(t) - await writeFile(join(src, 'other.md'), 'Other') - const site = await identifyPages(src) - let sourceIdReads = 0 - let filepathReads = 0 - const pages = site.pages.map(pageInfo => { - const page = new PageData({ - pageInfo, - globalVars: {}, - globalStyle: undefined, - globalClient: undefined, - defaultStyle: null, - defaultClient: null, - builderOptions: {}, - }) - const sourceId = page.sourceId - const filepath = pageInfo.pageFile.filepath - Object.defineProperty(page, 'sourceId', { get () { sourceIdReads++; return sourceId } }) - Object.defineProperty(pageInfo.pageFile, 'filepath', { get () { filepathReads++; return filepath } }) - return page - }) - const previousGlobalDataBaseline = { state: undefined, sourceIds: ['page.js', 'removed.md', 'removed.md'] } - const delta = createGlobalDataState({ pages, previousGlobalDataBaseline }) - assert.equal(sourceIdReads, pages.length) - assert.equal(filepathReads, 0) - assert.deepEqual(delta.context.changes, { - kind: 'delta', upserted: [pages.find(page => page.pageInfo.pageFile.relname === 'other.md')], removed: ['removed.md'], events: [], - }) - assert.deepEqual(delta.getBaseline().sourceIds, site.pages.map(page => page.pageFile.relname)) - - sourceIdReads = 0 - const reset = createGlobalDataState({ - pages, - previousGlobalDataBaseline, - globalDataInputChanges: { resetReason: 'recovery', upsertedPaths: [], events: [] }, - }) - assert.equal(sourceIdReads, pages.length) - assert.equal(filepathReads, 0) - assert.deepEqual(reset.context.changes, { kind: 'reset', reason: 'recovery', events: [] }) -}) - -test('state supports cycles and uncloneable updates fail actionably without replacing the candidate', () => { +test('cyclic state is supported and an uncloneable update leaves the last candidate intact', () => { const transaction = createGlobalDataState({ pages: [] }) - const state = { self: /** @type {unknown} */ (null), value: 1n } + const state = { self: /** @type {unknown} */ (null) } state.self = state transaction.context.setState(state) const candidate = /** @type {typeof state} */ (transaction.getBaseline().state) assert.equal(candidate.self, candidate) assert.notEqual(candidate, state) - for (const invalid of [() => {}, { renderer () {} }, new WeakMap(), Symbol('state')]) { - assert.throws(() => transaction.context.setState(invalid), /global\.data setState\(next\).*structured-cloneable.*PageData/) - assert.equal(transaction.getBaseline().state, candidate) - } + assert.throws(() => transaction.context.setState({ render () {} }), /global\.data setState\(next\).*structured-cloneable/) + assert.equal(transaction.getBaseline().state, candidate) }) -/** @param {SharedArrayBuffer} buffer */ -function sharedMemoryStates (buffer) { - const cycle = { self: /** @type {unknown} */ (null), nested: new Set([new Map([['view', new DataView(buffer)]])]) } +test('shared memory is rejected throughout nested state without replacing a valid candidate', () => { + const buffer = new SharedArrayBuffer(8) + const cycle = { self: /** @type {unknown} */ (null), buffer } cycle.self = cycle - return [ + const states = [ buffer, { nested: [buffer] }, new Uint8Array(buffer), - { nested: new DataView(buffer) }, + new DataView(buffer), new Map([[buffer, 'key']]), new Map([['value', buffer]]), new Set([buffer]), cycle, - new Error('nested cause', { cause: { buffer } }), + new Error('nested cause', { cause: buffer }), ] -} - -test('setState rejects nested shared memory without replacing or sharing the retained candidate', () => { - const buffer = new SharedArrayBuffer(8) - const bytes = new Uint8Array(buffer) - bytes[0] = 1 const transaction = createGlobalDataState({ pages: [] }) - transaction.context.setState({ bytes: Uint8Array.from(bytes) }) - const candidate = /** @type {{ bytes: Uint8Array }} */ (transaction.getBaseline().state) - for (const state of sharedMemoryStates(buffer)) { - assert.throws(() => transaction.context.setState(state), /global\.data setState\(next\).*SharedArrayBuffer.*non-shared ArrayBuffer/) + transaction.context.setState({ bytes: new Uint8Array([1]) }) + const candidate = transaction.getBaseline().state + for (const state of states) { + assert.throws(() => transaction.context.setState(state), /SharedArrayBuffer/) assert.equal(transaction.getBaseline().state, candidate) + assert.throws(() => createGlobalDataState({ + pages: [], previousGlobalDataBaseline: { state, sourceIds: [] }, + }), /global\.data previous state.*SharedArrayBuffer/) } - bytes[0] = 99 - assert.equal(candidate.bytes[0], 1, 'mutating rejected shared memory cannot mutate retained state') }) -test('previous baselines containing shared memory are rejected instead of exposing aliased state', () => { - const buffer = new SharedArrayBuffer(8) - for (const state of sharedMemoryStates(buffer)) { - const previousGlobalDataBaseline = { state, sourceIds: [] } - assert.throws(() => createGlobalDataState({ pages: [], previousGlobalDataBaseline }), /global\.data previous state.*SharedArrayBuffer.*non-shared ArrayBuffer/) - assert.equal(previousGlobalDataBaseline.state, state) - } +test('resets discard previous state and the first successful baseline permits a delta', () => { + const initial = createGlobalDataState({ pages: [] }) + assert.deepEqual(initial.context.changes, { kind: 'reset', reason: 'initial', events: [] }) + const delta = createGlobalDataState({ pages: [], previousGlobalDataBaseline: initial.getBaseline() }) + assert.deepEqual(delta.context.changes, { kind: 'delta', upserted: [], removed: [], events: [] }) const reset = createGlobalDataState({ pages: [], - previousGlobalDataBaseline: { state: buffer, sourceIds: [] }, + previousGlobalDataBaseline: { state: () => {}, sourceIds: ['old.md'] }, globalDataInputChanges: { resetReason: 'recovery', upsertedPaths: [], events: [] }, }) assert.equal(reset.context.previousState, undefined) - assert.equal(reset.getBaseline().state, undefined) -}) - -test('shared-memory validation inspects only cloned values and does not reread source getters', () => { - const transaction = createGlobalDataState({ pages: [] }) - let reads = 0 - const safe = { get value () { reads++; return new Uint8Array([1]) } } - transaction.context.setState({ first: safe, second: safe }) - assert.equal(reads, 1) - const unsafe = { get value () { reads++; return new SharedArrayBuffer(8) } } - assert.throws(() => transaction.context.setState(unsafe), /SharedArrayBuffer/) - assert.equal(reads, 2) - assert.throws(() => createGlobalDataState({ pages: [], previousGlobalDataBaseline: { state: unsafe, sourceIds: [] } }), /SharedArrayBuffer/) - assert.equal(reads, 3) - createGlobalDataState({ pages: [], previousGlobalDataBaseline: { state: safe, sourceIds: [] } }) - assert.equal(reads, 4, 'isolating both previousState and the candidate only reads the source getter once') - - class PrivateState { - #buffer = new SharedArrayBuffer(8) - value = 1 - read () { return this.#buffer } - } - transaction.context.setState(new PrivateState()) - assert.deepEqual(transaction.getBaseline().state, { value: 1 }, 'private fields discarded by structuredClone are not retained state') -}) - -test('non-shared buffers and views remain isolated across previousState and setState', () => { - const bytes = new Uint8Array([1, 2]) - const state = { bytes, view: new DataView(bytes.buffer) } - const transaction = createGlobalDataState({ pages: [], previousGlobalDataBaseline: { state, sourceIds: [] } }) - const previous = /** @type {typeof state} */ (transaction.context.previousState) - previous.view.setUint8(0, 3) - assert.equal(previous.bytes[0], 3) - assert.equal(bytes[0], 1) - assert.equal(/** @type {typeof state} */ (transaction.getBaseline().state).bytes[0], 1) - transaction.context.setState(previous) - previous.bytes[0] = 4 - assert.equal(/** @type {typeof state} */ (transaction.getBaseline().state).view.getUint8(0), 3) -}) - -test('resets discard state even without setState and do not try to clone invalid old state', () => { - for (const previousGlobalDataBaseline of [undefined, null, { state: () => {}, sourceIds: ['old.md'] }]) { - const transaction = createGlobalDataState({ - pages: [], - previousGlobalDataBaseline, - globalDataInputChanges: { resetReason: 'global-data-changed', upsertedPaths: [], events: [] }, - }) - assert.equal(transaction.context.previousState, undefined) - assert.deepEqual(transaction.context.changes, { kind: 'reset', reason: 'global-data-changed', events: [] }) - assert.deepEqual(transaction.getBaseline(), { state: undefined, sourceIds: [] }) - } - const initial = createGlobalDataState({ pages: [] }) - assert.deepEqual(initial.context.changes, { kind: 'reset', reason: 'initial', events: [] }) - const delta = createGlobalDataState({ pages: [], previousGlobalDataBaseline: initial.getBaseline() }) - assert.deepEqual(delta.context.changes, { kind: 'delta', upserted: [], removed: [], events: [] }) -}) - -test('direct builds reuse page metadata when its filepath is already normalized', async t => { - const { src, dest } = await fixture(t) - const globalDataPath = join(src, 'global.data.js') - await writeFile(globalDataPath, `export let pageInfo - export default ({ pages }) => { pageInfo = pages[0].pageInfo; return { summary: '' } }`) - const site = await identifyPages(src) - const result = await buildPagesDirect(src, dest, site) - assert.deepEqual(result.errors, []) - const globalDataModule = await import(globalDataPath) - assert.equal(globalDataModule.pageInfo, site.pages[0]) + assert.deepEqual(reset.context.changes, { kind: 'reset', reason: 'recovery', events: [] }) + assert.deepEqual(reset.getBaseline(), { state: undefined, sourceIds: [] }) }) -for (const [mode, build] of /** @type {const} */ ([['direct', buildPagesDirect], ['worker', buildPages]])) { - test(`${mode}: reset and delta use eligible source membership, not output filters or generated pages`, async t => { - const { src, dest } = await fixture(t) - await writeFile(join(src, 'other.md'), '---\ntitle: Other\n---\nOther') - await writeFile(join(src, 'hidden.draft.md'), 'Draft') - await writeFile(join(src, 'archive.pages.js'), 'export default () => ({ children: \'Generated\' })') - await writeFile(join(src, 'global.data.js'), `export default ({ pages, previousState, changes, setState }) => { - const summary = { - pages: pages.map(page => ({ sourceId: page.sourceId, title: page.vars.title })), - previousState, - changes: changes.kind === 'reset' ? changes : { - ...changes, upserted: changes.upserted.map(page => page.sourceId), - }, - } - const next = { count: (previousState?.count ?? 0) + 1 } - setState(next) - next.count = 999 - return { summary } - }`) - const site = await identifyPages(src) - const home = site.pages.find(page => page.pageFile.basename === 'page.js') - assert.ok(home) - home.pageFile.filepath = src + '/./page.js' - const initial = await build(src, dest, site, { trackWatchDependencies: true }) - assert.equal(home.pageFile.filepath, src + '/./page.js', 'normalizing public identities does not mutate discovery metadata') - assert.deepEqual(initial.errors, []) - const first = JSON.parse(await readFile(join(dest, 'index.html'), 'utf8')) - assert.equal(first.changes.kind, 'reset') - assert.equal(first.changes.reason, 'initial') - assert.equal(first.previousState, undefined) - assert.deepEqual(first.pages.map((/** @type {{sourceId: string}} */ p) => p.sourceId).sort(), ['other.md', 'page.js']) - assert.equal(first.pages.find((/** @type {{title: string}} */ p) => p.title === 'Home').title, 'Home') - assert.deepEqual(initial.report.globalDataBaseline, { state: { count: 1 }, sourceIds: site.pages.map(page => page.pageFile.relname) }) - assert.ok(initial.report.pages.some(page => page.pagesFilePath === join(src, 'archive.pages.js'))) - - await rm(join(src, 'other.md')) - await writeFile(join(src, 'new.md'), 'New') - const current = await identifyPages(src, { buildDrafts: true }) - const event = classifyWatchEvent('change', join(src, 'page.js')) - /** @type {BuildPagesFilterOptions} */ - const opts = { - trackWatchDependencies: true, - previousWatchDependencies: initial.report.watchDependencies, - previousGlobalDataBaseline: initial.report.globalDataBaseline, - pageFilterPaths: [], - templateFilterPaths: [], - pagesFileFilterPaths: [], - globalDataInputChanges: { - upsertedPaths: [src + '/./page.js', join(src, 'page.js'), join(src, 'other.md'), join(src, 'archive.pages.js')], - events: [event], +test('global data receives initialized source pages and membership changes independently of output filters', async t => { + const { src, dest } = await fixture(t, `export default ({ pages, previousState, changes, setState }) => { + setState({ count: (previousState?.count ?? 0) + 1 }) + return { summary: { + pages: pages.map(page => ({ sourceId: page.sourceId, title: page.vars.title })), + previousState, + changes: changes.kind === 'reset' ? changes : { + ...changes, upserted: changes.upserted.map(page => page.sourceId), }, - } - const before = structuredClone(opts) - const next = await build(src, dest, current, opts) - assert.deepEqual(next.errors, []) - assert.deepEqual(structuredClone(opts), before, 'worker and direct builds leave caller baselines and input changes untouched') - const second = JSON.parse(await readFile(join(dest, 'index.html'), 'utf8')) - assert.deepEqual(second.previousState, { count: 1 }) - assert.equal(second.changes.kind, 'delta') - assert.deepEqual(second.changes.upserted.sort(), ['hidden.draft.md', 'new.md', 'page.js']) - assert.deepEqual(second.changes.removed, ['other.md']) - assert.deepEqual(second.changes.events, [event]) - assert.deepEqual(next.report.globalDataBaseline?.state, { count: 2 }) - assert.deepEqual(next.report.pages.map(page => page.sourcePageFilePath), [join(src, 'page.js')], 'changed public data still invalidates its subscriber') - - const reset = await build(src, dest, current, { - ...opts, - previousGlobalDataBaseline: next.report.globalDataBaseline, - globalDataInputChanges: { resetReason: 'global-data-changed', upsertedPaths: [], events: [event] }, - }) - assert.deepEqual(reset.errors, []) - const third = JSON.parse(await readFile(join(dest, 'index.html'), 'utf8')) - assert.equal(third.previousState, undefined) - assert.deepEqual(third.changes, { kind: 'reset', reason: 'global-data-changed', events: [event] }) - assert.deepEqual(reset.report.globalDataBaseline?.state, { count: 1 }) - }) - - test(`${mode}: relocating the source root preserves membership while candidates still match absolute paths`, async t => { - const original = await fixture(t) - const relocated = await fixture(t) - for (const { src } of [original, relocated]) { - await mkdir(join(src, 'posts')) - await writeFile(join(src, 'posts', 'post.md'), 'Post') - await writeFile(join(src, 'global.data.js'), `export default ({ previousState, changes, setState }) => { - setState({ retained: true }) - return { summary: { - previousState, - changes: changes.kind === 'reset' ? changes : { - ...changes, upserted: changes.upserted.map(page => page.sourceId), - }, - } } - }`) - } - const initial = await build(original.src, original.dest, await identifyPages(original.src), { trackWatchDependencies: true }) - assert.deepEqual(initial.errors, []) - const previousGlobalDataBaseline = structuredClone(initial.report.globalDataBaseline) - const site = await identifyPages(relocated.src) - const oldPath = join(original.src, 'posts', 'post.md') - const oldEvent = classifyWatchEvent('change', oldPath) - const next = await build(relocated.src, relocated.dest, site, { - trackWatchDependencies: true, - previousGlobalDataBaseline, - globalDataInputChanges: { upsertedPaths: [oldPath], events: [oldEvent] }, - }) - assert.deepEqual(next.errors, []) - const summary = JSON.parse(await readFile(join(relocated.dest, 'index.html'), 'utf8')) - assert.deepEqual(summary.previousState, { retained: true }) - assert.deepEqual(summary.changes, { kind: 'delta', upserted: [], removed: [], events: JSON.parse(JSON.stringify([oldEvent])) }) - assert.equal(summary.changes.events[0].filepath, oldPath) - assert.deepEqual(next.report.globalDataBaseline, previousGlobalDataBaseline) - - const event = classifyWatchEvent('change', relocated.src + '/posts/./post.md') - const updated = await build(relocated.src, relocated.dest, site, { - trackWatchDependencies: true, - previousGlobalDataBaseline: next.report.globalDataBaseline, - globalDataInputChanges: { upsertedPaths: [relocated.src + '/posts/./post.md'], events: [event] }, - }) - assert.deepEqual(updated.errors, []) - const updateSummary = JSON.parse(await readFile(join(relocated.dest, 'index.html'), 'utf8')) - assert.deepEqual(updateSummary.changes, { kind: 'delta', upserted: ['posts/post.md'], removed: [], events: JSON.parse(JSON.stringify([event])) }) - assert.equal(updateSummary.changes.events[0].filepath, join(relocated.src, 'posts', 'post.md')) - }) - - test(`${mode}: relative source IDs distinguish duplicate basenames and removed IDs delete map entries`, async t => { - const { src, dest } = await fixture(t) - for (const directory of ['a', 'b', 'c']) await mkdir(join(src, directory)) - await writeFile(join(src, 'a', 'post.md'), '---\ntitle: Alpha\n---\nAlpha') - await writeFile(join(src, 'b', 'post.md'), '---\ntitle: Beta\n---\nBeta') - await writeFile(join(src, 'global.data.js'), `export default ({ pages, previousState, changes, setState }) => { - const titles = changes.kind === 'reset' ? new Map() : previousState.titles - const upserted = changes.kind === 'reset' ? pages : changes.upserted - const removed = changes.kind === 'reset' ? [] : changes.removed - for (const sourceId of removed) titles.delete(sourceId) - for (const page of upserted) titles.set(page.sourceId, page.vars.title) - setState({ titles }) - return { summary: { - titles: [...titles].sort(), - upserted: upserted.map(page => page.sourceId).sort(), - removed, - events: changes.events, - } } - }`) - const initial = await build(src, dest, await identifyPages(src), { trackWatchDependencies: true }) - assert.deepEqual(initial.errors, []) - const before = structuredClone(initial.report.globalDataBaseline) - const first = JSON.parse(await readFile(join(dest, 'index.html'), 'utf8')) - assert.deepEqual(first.titles, [['a/post.md', 'Alpha'], ['b/post.md', 'Beta'], ['page.js', 'Home']]) - assert.deepEqual(initial.report.globalDataBaseline?.sourceIds.slice().sort(), ['a/post.md', 'b/post.md', 'page.js']) - - await rm(join(src, 'a', 'post.md')) - await writeFile(join(src, 'c', 'post.md'), '---\ntitle: Gamma\n---\nGamma') - const events = [ - classifyWatchEvent('removed', src + '/a/./post.md'), - classifyWatchEvent('added', src + '/c/./post.md'), - ] - const next = await build(src, dest, await identifyPages(src), { - trackWatchDependencies: true, - previousGlobalDataBaseline: initial.report.globalDataBaseline, - globalDataInputChanges: { upsertedPaths: [], events }, - }) - assert.deepEqual(next.errors, []) - const second = JSON.parse(await readFile(join(dest, 'index.html'), 'utf8')) - assert.deepEqual(second.upserted, ['c/post.md'], 'new membership is upserted without an absolute candidate') - assert.deepEqual(second.removed, ['a/post.md']) - assert.deepEqual(second.titles, [['b/post.md', 'Beta'], ['c/post.md', 'Gamma'], ['page.js', 'Home']]) - assert.deepEqual(second.events, JSON.parse(JSON.stringify(events))) - assert.deepEqual(second.events.map((/** @type {{filepath: string}} */ event) => event.filepath), [join(src, 'a', 'post.md'), join(src, 'c', 'post.md')]) - assert.deepEqual(next.report.globalDataBaseline?.state, { titles: new Map(second.titles) }) - assert.deepEqual(next.report.globalDataBaseline?.sourceIds.slice().sort(), ['b/post.md', 'c/post.md', 'page.js']) - assert.deepEqual(initial.report.globalDataBaseline, before, 'deleting from previousState does not mutate the committed baseline') + } } + }`) + await writeFile(join(src, 'other.md'), '---\ntitle: Other\n---\nOther') + await writeFile(join(src, 'hidden.draft.md'), 'Draft') + await writeFile(join(src, 'archive.pages.js'), 'export default () => ({ children: "Generated" })') + const site = await identifyPages(src) + const home = site.pages.find(page => page.pageFile.basename === 'page.js') + assert.ok(home) + home.pageFile.filepath = src + '/./page.js' + const initial = await buildPagesDirect(src, dest, site, { trackWatchDependencies: true }) + assert.deepEqual(initial.errors, []) + assert.equal(home.pageFile.filepath, src + '/./page.js', 'path normalization must not mutate discovery metadata') + /** @type {{ pages: { sourceId: string, title: string }[] }} */ + const first = JSON.parse(await readFile(join(dest, 'index.html'), 'utf8')) + assert.deepEqual(first.pages.sort((a, b) => a.sourceId.localeCompare(b.sourceId)), [ + { sourceId: 'other.md', title: 'Other' }, { sourceId: 'page.js', title: 'Home' }, + ]) + + await rm(join(src, 'other.md')) + await writeFile(join(src, 'new.md'), 'New') + const event = classifyWatchEvent('change', join(src, 'page.js')) + const next = await buildPagesDirect(src, dest, await identifyPages(src, { buildDrafts: true }), { + trackWatchDependencies: true, + previousWatchDependencies: initial.report.watchDependencies, + previousGlobalDataBaseline: initial.report.globalDataBaseline, + pageFilterPaths: [], + templateFilterPaths: [], + pagesFileFilterPaths: [], + globalDataInputChanges: { + upsertedPaths: [src + '/./page.js', join(src, 'page.js'), join(src, 'other.md'), join(src, 'archive.pages.js')], + events: [event], + }, }) - - test(`${mode}: state does not leak into public data fingerprints or commit previousState mutations`, async t => { - const { src, dest } = await fixture(t) - await writeFile(join(src, 'global.data.js'), `export default ({ previousState, setState, changes }) => { - if (!previousState) setState({ count: 1 }) - else { - previousState.count++ - if (changes.events.length) setState(previousState) - } - return { summary: 'stable' } - }`) - const site = await identifyPages(src) - const initial = await build(src, dest, site, { trackWatchDependencies: true }) - assert.deepEqual(initial.errors, []) - const next = await build(src, dest, site, { - trackWatchDependencies: true, - previousGlobalDataBaseline: initial.report.globalDataBaseline, - previousWatchDependencies: initial.report.watchDependencies, - pageFilterPaths: [], - templateFilterPaths: [], - pagesFileFilterPaths: [], - }) - assert.deepEqual(next.errors, []) - assert.deepEqual(next.report.globalDataBaseline?.state, { count: 1 }) - assert.deepEqual(initial.report.globalDataBaseline?.state, { count: 1 }) - assert.deepEqual(next.report.pages, []) - assert.deepEqual(next.report.watchDependencies?.globalDataFingerprints, initial.report.watchDependencies?.globalDataFingerprints) - assert.deepEqual(Object.keys(next.report.watchDependencies?.globalDataFingerprints ?? {}), ['summary']) - const stateOnly = await build(src, dest, site, { - trackWatchDependencies: true, - previousGlobalDataBaseline: next.report.globalDataBaseline, - previousWatchDependencies: next.report.watchDependencies, - pageFilterPaths: [], - templateFilterPaths: [], - pagesFileFilterPaths: [], - globalDataInputChanges: { upsertedPaths: [], events: [classifyWatchEvent('change', join(src, 'global.data.js'))] }, - }) - assert.deepEqual(stateOnly.errors, []) - assert.deepEqual(stateOnly.report.globalDataBaseline?.state, { count: 2 }) - assert.deepEqual(stateOnly.report.pages, []) - assert.deepEqual(stateOnly.report.watchDependencies?.globalDataFingerprints, next.report.watchDependencies?.globalDataFingerprints) + assert.deepEqual(next.errors, []) + const second = JSON.parse(await readFile(join(dest, 'index.html'), 'utf8')) + assert.deepEqual(second.previousState, { count: 1 }) + second.changes.upserted.sort() + assert.deepEqual(second.changes, { + kind: 'delta', upserted: ['hidden.draft.md', 'new.md', 'page.js'], removed: ['other.md'], events: [event], }) + assert.deepEqual(initial.report.globalDataBaseline?.state, { count: 1 }) + assert.deepEqual(next.report.globalDataBaseline?.state, { count: 2 }) +}) - test(`${mode}: resets never read or transfer the discarded baseline`, async t => { - const { src, dest } = await fixture(t) - await writeFile(join(src, 'global.data.js'), `export default ({ previousState, changes }) => { - if (previousState !== undefined) throw new Error('reset exposed old state') - return { summary: changes } - }`) - const site = await identifyPages(src) - const previousGlobalDataBaseline = { - /** @returns {never} */ - get state () { throw new Error('discarded state must not be read or cloned') }, - /** @returns {never} */ - get sourceIds () { throw new Error('discarded membership must not be read or cloned') }, - } - const result = await build(src, dest, site, { - trackWatchDependencies: true, - previousGlobalDataBaseline, - globalDataInputChanges: { resetReason: 'recovery', upsertedPaths: [], events: [] }, - }) - assert.deepEqual(result.errors, []) - assert.deepEqual(result.report.globalDataBaseline, { state: undefined, sourceIds: ['page.js'] }) - assert.deepEqual(JSON.parse(await readFile(join(dest, 'index.html'), 'utf8')), { kind: 'reset', reason: 'recovery', events: [] }) +test('worker state updates do not enter public fingerprints or rebuild unchanged subscribers', async t => { + const { src, dest } = await fixture(t, `export default ({ previousState, setState }) => { + const next = { count: (previousState?.count ?? 0) + 1 } + setState(next) + next.count = 999 + return { summary: 'stable' } + }`) + const site = await identifyPages(src) + const cold = await buildPages(src, dest, site, {}) + assert.deepEqual(cold.errors, []) + assert.equal(Object.hasOwn(cold.report, 'globalDataBaseline'), false) + const initial = await buildPages(src, dest, site, { trackWatchDependencies: true }) + assert.deepEqual(initial.errors, []) + const next = await buildPages(src, dest, site, { + trackWatchDependencies: true, + previousGlobalDataBaseline: initial.report.globalDataBaseline, + previousWatchDependencies: initial.report.watchDependencies, + pageFilterPaths: [], + templateFilterPaths: [], + pagesFileFilterPaths: [], }) + assert.deepEqual(next.errors, []) + assert.deepEqual(initial.report.globalDataBaseline?.state, { count: 1 }) + assert.deepEqual(next.report.globalDataBaseline?.state, { count: 2 }) + assert.deepEqual(next.report.pages, []) + assert.deepEqual(next.report.watchDependencies?.globalDataFingerprints, initial.report.watchDependencies?.globalDataFingerprints) + assert.deepEqual(Object.keys(next.report.watchDependencies?.globalDataFingerprints ?? {}), ['summary']) +}) - test(`${mode}: successful watch builds emit candidates while cold reports keep state private`, async t => { - const { src, dest } = await fixture(t) - await writeFile(join(src, 'global.data.js'), 'export default ({ setState }) => { setState({ candidate: true }); return { summary: \'\' } }') - const site = await identifyPages(src) - const cold = await build(src, dest, site, {}) - assert.deepEqual(cold.errors, []) - assert.equal(Object.hasOwn(cold.report, 'globalDataBaseline'), false) - const watch = await build(src, dest, site, { trackWatchDependencies: true }) - assert.deepEqual(watch.errors, []) - assert.deepEqual(watch.report.globalDataBaseline, { state: { candidate: true }, sourceIds: ['page.js'] }) +test('worker resets discard even uncloneable previous state before transfer', async t => { + const { src, dest } = await fixture(t, 'export default ({ previousState, changes }) => ({ summary: { previousState, changes } })') + const result = await buildPages(src, dest, await identifyPages(src), { + trackWatchDependencies: true, + previousGlobalDataBaseline: { state: () => {}, sourceIds: ['old.md'] }, + globalDataInputChanges: { resetReason: 'recovery', upsertedPaths: [], events: [] }, }) - - test(`${mode}: failed page, template, factory, and initialization phases never emit candidate baselines`, async t => { - for (const [filename, content] of [ - ['page.js', 'export default () => { throw new Error(\'render failed\') }'], - ['feed.template.js', 'export default () => { throw new Error(\'template failed\') }'], - ['archive.pages.js', 'export default () => { throw new Error(\'factory failed\') }'], - ['page.vars.js', 'export default () => { throw new Error(\'vars failed\') }'], - ]) { - const { src, dest } = await fixture(t) - assert.ok(filename && content) - await writeFile(join(src, filename), content) - await writeFile(join(src, 'global.data.js'), 'export default ({ setState }) => { setState({ candidate: true }); return { summary: \'\' } }') - const previousGlobalDataBaseline = { state: { previous: true }, sourceIds: [] } - const result = await build(src, dest, await identifyPages(src), { previousGlobalDataBaseline, trackWatchDependencies: true }) - assert.ok(result.errors.length > 0, filename) - assert.equal(Object.hasOwn(result.report, 'globalDataBaseline'), false, filename) - assert.deepEqual(previousGlobalDataBaseline, { state: { previous: true }, sourceIds: [] }) - } + assert.deepEqual(result.errors, []) + assert.deepEqual(result.report.globalDataBaseline, { state: undefined, sourceIds: ['page.js'] }) + assert.deepEqual(JSON.parse(await readFile(join(dest, 'index.html'), 'utf8')), { + changes: { kind: 'reset', reason: 'recovery', events: [] }, }) +}) - test(`${mode}: shared-memory updates and previous baselines fail without emitting candidates`, async t => { - for (const previous of [false, true]) { - const { src, dest } = await fixture(t) - await writeFile(join(src, 'global.data.js'), previous - ? 'export default () => { throw new Error(\'shared baseline must be rejected before callback\') }' - : 'export default ({ setState }) => { setState({ nested: new Map([[\'bytes\', new DataView(new SharedArrayBuffer(8))]]) }); return { summary: \'\' } }') - const site = await identifyPages(src) - /** @type {BuildPagesFilterOptions} */ - const opts = { - trackWatchDependencies: true, - previousGlobalDataBaseline: previous ? { state: new Set([new Uint8Array(new SharedArrayBuffer(8))]), sourceIds: [] } : undefined, - } - const message = previous ? /global\.data previous state.*SharedArrayBuffer/ : /global\.data setState\(next\).*SharedArrayBuffer/ - if (mode === 'direct') { - await assert.rejects(build(src, dest, site, opts), message) - } else { - const result = await build(src, dest, site, opts) - assert.equal(result.errors.length, 1) - const failure = result.errors[0] - assert.ok(failure instanceof Error) - assert.match(failure.message, message) - assert.equal(Object.hasOwn(result.report, 'globalDataBaseline'), false) - } - } - }) +test('failed page, template, factory, and initialization phases do not emit a candidate', async t => { + for (const [filename, content] of [ + ['page.js', 'export default () => { throw new Error("render failed") }'], + ['feed.template.js', 'export default () => { throw new Error("template failed") }'], + ['archive.pages.js', 'export default () => { throw new Error("factory failed") }'], + ['page.vars.js', 'export default () => { throw new Error("vars failed") }'], + ]) { + assert.ok(filename && content) + const { src, dest } = await fixture(t, 'export default ({ setState }) => { setState({ candidate: true }); return { summary: "" } }') + await writeFile(join(src, filename), content) + const result = await buildPagesDirect(src, dest, await identifyPages(src), { trackWatchDependencies: true }) + assert.ok(result.errors.length > 0, filename) + assert.equal(Object.hasOwn(result.report, 'globalDataBaseline'), false, filename) + } +}) - test(`${mode}: uncloneable setState and global-data failures are actionable without candidates`, async t => { - for (const body of [ - 'setState({ renderer () {} })', - 'setState({ candidate: true }); throw new Error(\'global data failed\')', - ]) { - const { src, dest } = await fixture(t) - await writeFile(join(src, 'global.data.js'), `export default ({ setState }) => { ${body}; return { summary: '' } }`) - const site = await identifyPages(src) - if (mode === 'direct') { - await assert.rejects(build(src, dest, site, { trackWatchDependencies: true }), /global\.data setState\(next\).*structured-cloneable|global data failed/) - } else { - const result = await build(src, dest, site, { trackWatchDependencies: true }) - assert.equal(result.errors.length, 1) - const failure = result.errors[0] - assert.ok(failure instanceof Error) - assert.match(failure.message, /global\.data setState\(next\).*structured-cloneable|global data failed/) - assert.equal(Object.hasOwn(result.report, 'globalDataBaseline'), false) - } - } - }) -} +test('worker validation errors remain actionable and never emit a candidate', async t => { + for (const [expression, expected] of /** @type {const} */ ([ + ['{ render () {} }', /structured-cloneable/], + ['new SharedArrayBuffer(8)', /SharedArrayBuffer/], + ])) { + const { src, dest } = await fixture(t, `export default ({ setState }) => { setState(${expression}); return {} }`) + const result = await buildPages(src, dest, await identifyPages(src), { trackWatchDependencies: true }) + assert.equal(result.errors.length, 1) + const error = result.errors[0] + assert.ok(error instanceof Error) + assert.match(error.message, expected) + assert.equal(Object.hasOwn(result.report, 'globalDataBaseline'), false) + } +}) -test('resolveGlobalData retains object and non-serializable public return values unchanged', async t => { - const { src } = await fixture(t) +test('resolveGlobalData preserves static exports and accepts a missing global-data file', async t => { + const { src } = await fixture(t, 'export default { render: () => "public function", value: 1 }') const globalDataPath = join(src, 'global.data.js') - await writeFile(globalDataPath, 'export default { render: () => \'public function\', value: 1 }') const imported = await import(globalDataPath) const { context } = createGlobalDataState({ pages: [] }) const result = await resolveGlobalData({ globalDataPath, context }) diff --git a/lib/watch-plan.js b/lib/watch-plan.js index 416a0c0e..42bed124 100644 --- a/lib/watch-plan.js +++ b/lib/watch-plan.js @@ -24,9 +24,9 @@ * @property {string[] | null} pageFilterPaths * @property {string[] | null} templateFilterPaths * @property {string[] | null} pagesFileFilterPaths - * @property {string} [message] - * @property {PageInfo[]} [pages] - * @property {TemplateInfo[]} [templates] + * @property {string | undefined} [message] + * @property {PageInfo[] | undefined} [pages] + * @property {TemplateInfo[] | undefined} [templates] * @typedef {PagePlan | {kind: 'skip', message: string} | {kind: 'full', message: string} | {kind: 'restart', message: string}} WatchPlan * @typedef {object} WatchInputChanges * @property {string} [resetReason] Recompute all producer inputs instead of applying the delta. @@ -237,17 +237,13 @@ function unionWatchPlans (left, right) { pageFilterPaths: unionFilterPaths(left.pageFilterPaths, right.pageFilterPaths), templateFilterPaths: unionFilterPaths(left.templateFilterPaths, right.templateFilterPaths), pagesFileFilterPaths: unionFilterPaths(left.pagesFileFilterPaths, right.pagesFileFilterPaths), - ...(left.pages || right.pages - ? { - pages: [...new Map([...(left.pages ?? []), ...(right.pages ?? [])].map(page => [page.pageFile.filepath, page])).values()], - } - : {}), - ...(left.templates || right.templates - ? { - templates: [...new Map([...(left.templates ?? []), ...(right.templates ?? [])].map(template => [template.templateFile.filepath, template])).values()], - } - : {}), - ...(messages.size ? { message: [...messages].join('\n') } : {}), + pages: left.pages || right.pages + ? [...new Map([...(left.pages ?? []), ...(right.pages ?? [])].map(page => [page.pageFile.filepath, page])).values()] + : undefined, + templates: left.templates || right.templates + ? [...new Map([...(left.templates ?? []), ...(right.templates ?? [])].map(template => [template.templateFile.filepath, template])).values()] + : undefined, + message: messages.size ? [...messages].join('\n') : undefined, } } diff --git a/lib/watch-plan.test.js b/lib/watch-plan.test.js index a5b0b153..27129a99 100644 --- a/lib/watch-plan.test.js +++ b/lib/watch-plan.test.js @@ -81,17 +81,6 @@ test('layout maps target source pages and generated-page owners', () => { assert.ok(unused.message?.includes('did not match any rebuild rule')) }) -test('events without affected layouts do not scan generated-page layout memberships', t => { - const { state } = fixture() - const scan = t.mock.method(state.pagesFileLayoutMap, Symbol.iterator) - for (const name of ['page.md', 'page-helper.js', 'template-helper.js', 'client.jsx']) { - planWatchEvent(state, classifyWatchEvent('change', `/site/${name}`)) - } - assert.equal(scan.mock.callCount(), 0) - planWatchEvent(state, classifyWatchEvent('change', '/site/layout-helper.js')) - assert.equal(scan.mock.callCount(), 1) -}) - test('page, template, and owner inputs retain their targeted scopes', () => { const { state, home, other, owner, template } = fixture() const cases = [ @@ -302,7 +291,9 @@ test('single page plans preserve exact logging metadata and messages even when s assert.deepEqual(planWatchBatch(state, events).plan, expected, name) } const failed = { ...state, pageBuildFailed: true } - assert.deepEqual(planWatchBatch(failed, [event]).plan, planWatchEvent(failed, event), 'recovery preserves the single-event message') + const recovery = planWatchBatch(failed, [event]).plan + assert.deepEqual(scope(recovery), [null, null, null]) + assert.equal(recovery.message, planWatchEvent(failed, event).message, 'recovery preserves the single-event message') } }) @@ -342,34 +333,6 @@ test('known browser-only dependencies skip page rebuilding in both planners with assert.equal(unknown.inputChanges.resetReason, 'unknown-event') }) -test('browser dependency membership cannot hide mapped server consumers or global reset reasons', () => { - const { state } = fixture() - state.globalDataDepPaths.add('/site/data-helper.js') - state.markdownDepPaths.add('/site/markdown-helper.js') - state.globalVarsDepPaths.add('/site/vars-helper.js') - for (const name of ['page.md', 'page.vars.js', 'root.layout.js', 'layout-helper.js', 'page-helper.js', 'template-helper.js', 'archive-helper.js', 'data-helper.js', 'markdown-helper.js', 'vars-helper.js']) { - const filepath = `/site/${name}` - const event = classifyWatchEvent('change', filepath) - const single = planWatchEvent(state, event) - const batch = planWatchBatch(state, [event]) - assert.notEqual(single.kind, 'skip', name) - state.esbuildDepPaths.add(filepath) - assert.deepEqual(planWatchEvent(state, event), single, name) - assert.deepEqual(planWatchBatch(state, [event]), batch, name) - } -}) - -test('known browser dependencies do not suppress recovery after build or dependency-analysis failures', () => { - const { state } = fixture() - const event = classifyWatchEvent('change', '/site/browser-helper.js') - state.esbuildDepPaths.add(event.filepath) - for (const flag of /** @type {const} */ (['pageBuildFailed', 'dependencyAnalysisFailed'])) { - const batch = planWatchBatch({ ...state, [flag]: true }, [event]) - assert.deepEqual(scope(batch.plan), [null, null, null]) - assert.equal(batch.inputChanges.resetReason, flag === 'pageBuildFailed' ? 'page-build-failed' : 'dependency-analysis-failed') - } -}) - test('template and generated-page owner changes do not invent source inputs', () => { const { state, template, owner } = fixture() const batch = planWatchBatch(state, ['feed.template.js', 'archive.pages.js', 'archive-helper.js'] @@ -383,7 +346,7 @@ test('null filters dominate filtered scopes and failure resets override event re for (const flag of /** @type {const} */ (['pageBuildFailed', 'dependencyAnalysisFailed'])) { const { state: snapshot } = fixture() const state = { ...snapshot, pageBuildFailed: flag === 'pageBuildFailed', dependencyAnalysisFailed: flag === 'dependencyAnalysisFailed' } - const events = ['page.md', 'feed.template.js', 'global.data.js'].map(name => classifyWatchEvent('change', `/site/${name}`)) + const events = ['page.md', 'feed.template.js', 'global.data.js', 'client.jsx'].map(name => classifyWatchEvent('change', `/site/${name}`)) const batch = planWatchBatch(state, events) assert.deepEqual(scope(batch.plan), [null, null, null]) assert.equal(batch.inputChanges.resetReason, flag === 'pageBuildFailed' ? 'page-build-failed' : 'dependency-analysis-failed') @@ -408,7 +371,7 @@ test('empty and intentionally skipped batches have no input changes', () => { test('structural source pages carry membership candidates and change-plan consumers without resetting', () => { for (const type of /** @type {const} */ (['added', 'removed'])) { - for (const name of ['page.md', 'new/page.html', 'new/page.js', 'new/page.draft.mjs', 'new/README.md', 'new/README.draft.md', 'new/post.md', 'new/post.draft.md']) { + for (const name of ['page.md', 'new/page.html', 'new/post.md']) { const { state, home, other } = fixture() const filepath = `/site/${name}` state.pageDepMap.set(filepath, new Set([other])) @@ -449,21 +412,6 @@ test('structural layout, vars, unknown, template, and owner events conservativel } }) -test('non-source structural events do not resolve individual page consumers', t => { - const { state, home, other } = fixture() - const lookup = t.mock.method(state.pageDepMap, 'get') - for (const type of /** @type {const} */ (['added', 'removed'])) { - for (const name of ['page.vars.js', 'root.layout.js', 'feed.template.js', 'archive.pages.js', 'helper.js', 'client.jsx']) { - const batch = planWatchBatch(state, [classifyWatchEvent(type, `/site/${name}`)]) - assert.deepEqual(batch.inputChanges.upsertedPaths, [home.pageFile.filepath, other.pageFile.filepath]) - } - } - assert.equal(lookup.mock.callCount(), 0) - const batch = planWatchBatch(state, [classifyWatchEvent('added', '/site/new/page.md')]) - assert.deepEqual(batch.inputChanges.upsertedPaths, ['/site/new/page.md']) - assert.equal(lookup.mock.callCount(), 1, 'structural source events still resolve mapped consumers') -}) - test('repeated all-source events preserve candidate order and later membership changes', () => { const { state, home, other } = fixture() const events = [ @@ -515,9 +463,10 @@ test('global roots and imports reset on changes, additions, and removals regardl const root = classifyWatchEvent(type, `/site/${name}`) assert.equal(planWatchBatch(state, [root]).inputChanges.resetReason, reason, `${type} ${name}`) if (!depSet) continue - for (const sharedName of ['helper.js', 'client.jsx', 'page.md', 'domstack-manifest.settings.js']) { + for (const sharedName of ['helper.js', 'client.jsx', 'page.md']) { const filepath = `/site/${sharedName}` state[depSet].add(filepath) + state.esbuildDepPaths.add(filepath) state.pageDepMap.set(filepath, new Set([other])) const batch = planWatchBatch(state, [classifyWatchEvent(type, filepath)]) assert.equal(batch.inputChanges.resetReason, reason, `${type} ${sharedName}`) diff --git a/site/layouts/docs/navigation-watch.test.js b/site/layouts/docs/navigation-watch.test.js index 4c89a3e4..478b92b0 100644 --- a/site/layouts/docs/navigation-watch.test.js +++ b/site/layouts/docs/navigation-watch.test.js @@ -1,22 +1,17 @@ /** - * @import { TestContext } from 'node:test' - * @import { FSWatcher } from 'chokidar' - * @import { NavigationEntry } from './navigation.js' - * @typedef {{ docsNavigation: NavigationEntry[], docsIndexHtml: string }} DocsData - * @typedef {{ kind: 'reset' | 'delta', rendered: string[], threadId: number, invocation: number, publicData: DocsData }} ProducerCall - * @typedef {{ type: 'add' | 'unlink' | 'change', file: string }} SourceEvent + * @import { GlobalDataChanges, PageData } from '../../../types.ts' + * @import { DocsNavigationIndex, DocsPageVars } from './navigation.js' */ import assert from 'node:assert/strict' -import { EventEmitter } from 'node:events' import { test } from 'node:test' -import { cp, mkdir, mkdtemp, readFile, readdir, rename, rm, stat, utimes, writeFile } from 'node:fs/promises' +import { cp, mkdir, mkdtemp, readFile, rm, stat, writeFile } from 'node:fs/promises' import { dirname, join, resolve } from 'node:path' -import { setImmediate as nextTurn } from 'node:timers/promises' -import chokidar from 'chokidar' +import { load } from 'cheerio' import pino from 'pino' import { DomStack } from '../../../index.js' import { startWatch } from '../../../test-cases/watch/helpers.js' +import produceDocsData from '../../globals/global.data.ts' test('heading changes refresh shared navigation; body edits leave other pages alone', { timeout: 30_000 }, async t => { const root = resolve(import.meta.dirname, '../../..') @@ -70,319 +65,93 @@ test('heading changes refresh shared navigation; body edits leave other pages al assert.equal((await stat(output)).mtimeMs, mtime, 'unchanged navigation does not invalidate another page') }) -const eligibleDocs = [ - 'docs/first/README.md', - 'docs/first/child/README.md', - 'docs/first/child/leaf/README.md', - 'docs/second/README.md', -] -const firstDoc = doc('First', 'docsGroup: Guides\ndocsOrder: 10\n') -const secondDoc = doc('Second', 'docsGroup: Reference\ndocsOrder: 20\n') -const leafDoc = doc('Leaf', 'docsGroup: Guides\ndocsParent: /docs/first/child/\n') -const options = { timeout: 30_000 } - -/** @param {string} title @param {string} [vars] */ -function doc (title, vars = '') { - return `---\nlayout: docs\n${vars}---\n# ${title}\n\n## Original\n\nBody\n` -} - -/** @param {ProducerCall} call @param {'reset' | 'delta'} kind @param {string[]} rendered */ -function assertRenders (call, kind, rendered) { - assert.equal(call.kind, kind) - assert.deepEqual(call.rendered, [...rendered].sort(), 'actual producer renderInnerPage calls, including duplicates') +/** @param {string} html */ +function indexLinks (html) { + const $ = load(html) + return $('.docs-index a').toArray().map(link => [$(link).attr('href'), $(link).text()]) } -/** @param {string} directory @returns {Promise>} */ -async function docsSnapshot (directory) { - /** @type {Record} */ - const snapshot = {} - for (const entry of await readdir(directory, { withFileTypes: true })) { - const path = join(directory, entry.name) - if (entry.isDirectory()) { - for (const [name, content] of Object.entries(await docsSnapshot(path))) snapshot[join(entry.name, name)] = content - } else { - snapshot[entry.name] = await readFile(path, 'utf8') - } - } - return snapshot -} - -/** - * Mock only source notifications; discovery, dependency tracking, worker isolation, - * rendering and output writes still run through the real watch/build implementation. - * @param {TestContext} t - */ -async function instrumentedFixture (t) { - const root = resolve(import.meta.dirname, '../../..') - const temp = await mkdtemp(join(root, '.tmp-docs-navigation-')) - const src = join(temp, 'src') - const dest = join(temp, 'public') - const log = join(temp, 'producer.jsonl') - const buildOptions = { logger: pino({ level: 'silent' }), metafile: false, domstackManifest: false } - const domstack = new DomStack(src, dest, buildOptions) - t.after(async () => { - try { - if (domstack.watching) await domstack.stopWatching() - } finally { - await rm(temp, { recursive: true, force: true }) - } - }) - for (const file of [ - 'site/globals/global.data.ts', - 'site/layouts/docs/navigation.js', - 'site/layouts/docs/docs.layout.js', - 'site/layouts/root/root.layout.js', - ]) { - await mkdir(dirname(join(src, file)), { recursive: true }) - await cp(join(root, file), join(src, file)) - } - /** @param {string} file @param {string} text */ - async function write (file, text) { - await mkdir(dirname(join(src, file)), { recursive: true }) - await writeFile(join(src, file), text) - } - - // Keep the real producer's relative imports, but only the wrapper is a global root. - await rename(join(src, 'site/globals/global.data.ts'), join(src, 'site/globals/producer.ts')) - await write('site/globals/global.data.ts', ` -import { appendFileSync } from 'node:fs' -import { relative } from 'node:path' -import { threadId } from 'node:worker_threads' -import produce from './producer.ts' -let invocation = 0 -export default async function (params) { +test('docs producer retains navigation while rendering only the required sources', async t => { + /** @type {string[]} */ const rendered = [] - const originals = params.pages.map(page => [page, page.renderInnerPage]) - for (const [page, render] of originals) { - page.renderInnerPage = function (...args) { - rendered.push(relative(${JSON.stringify(src)}, page.pageInfo.pageFile.filepath)) - return Reflect.apply(render, this, args) - } - } - let publicData - try { - publicData = await Reflect.apply(produce, this, [params]) - } finally { - for (const [page, render] of originals) page.renderInnerPage = render - } - appendFileSync(${JSON.stringify(log)}, JSON.stringify({ - kind: params.changes.kind, rendered: rendered.sort(), - threadId, invocation: ++invocation, publicData, - }) + '\\n') - return publicData -} -`) - await write('docs/README.md', '---\nlayout: docs\ndataDeps: [docsIndexHtml]\n---\n# Docs\n\n{{{ data.docsIndexHtml }}}') - await write('docs/first/README.md', firstDoc) - await write('docs/second/README.md', secondDoc) - await write('docs/first/child/README.md', doc('Child', 'docsGroup: Guides\ndocsParent: /docs/first/\n')) - await write('docs/first/child/leaf/README.md', leafDoc) - await write('outside/README.md', '---\nlayout: root\n---\n# Not documentation') - await write('docs/script/page.js', "export const vars = { layout: 'docs' }; export default () => '

Not Markdown

'\n") - await write('docs/hidden/page.draft.md', doc('Unpublished')) - await write('docs/navigation.json.template.js', ` -export const dataDeps = ['docsNavigation', 'docsIndexHtml'] -export default ({ data }) => JSON.stringify({ docsNavigation: data.docsNavigation, docsIndexHtml: data.docsIndexHtml }) -`) - await write('site/globals/markdown-it.settings.js', 'export default md => md.set({ typographer: false })\n') - - /** @type {FSWatcher | undefined} */ - let watcher - const watch = chokidar.watch - /** @param {Parameters} args */ - const sourceWatch = (...args) => { - if (args[0] !== src) return watch(...args) - const fake = Object.assign(new EventEmitter(), { - closed: false, - async close () { fake.closed = true }, + /** @type {DocsNavigationIndex | undefined} */ + let state + /** @param {string} url @param {string} html @param {'md' | 'html'} [type] */ + function page (url, html, type = 'md') { + // Only the PageData fields consumed by the real docs producer are needed here. + return /** @type {PageData} */ ({ + sourceId: `${url.slice(1)}page.${type}`, + pageInfo: { url, type }, + vars: {}, + async renderInnerPage () { rendered.push(url); return html }, }) - watcher = /** @type {FSWatcher} */ (/** @type {unknown} */ (fake)) - setImmediate(() => { if (!fake.closed) fake.emit('ready') }) - return watcher } - t.mock.method(chokidar, 'watch', sourceWatch) - - /** @returns {Promise} */ - async function calls () { - const text = await readFile(log, 'utf8') - return text.trim().split('\n').map(line => JSON.parse(line)) - } - /** @type {Set} */ - const workers = new Set() - /** @param {number} before */ - async function newCall (before) { - const added = (await calls()).slice(before) - assert.equal(added.length, 1, 'one producer invocation per build, not one per output') - const call = added[0] - assert.ok(call) - assert.equal(call.invocation, 1, 'the producer module is loaded anew for every build') - assert.ok(call.threadId > 0, 'producer runs in a real worker') - assert.ok(!workers.has(call.threadId), 'each build uses a fresh worker, not module-local state') - workers.add(call.threadId) - assert.deepEqual(Object.keys(call.publicData).sort(), ['docsIndexHtml', 'docsNavigation'], 'public keys remain unchanged') - return call - } - /** @param {SourceEvent[]} events */ - async function rebuild (events) { - const before = (await calls()).length - assert.ok(watcher) - for (const { type, file } of events) watcher.emit(type, join(src, file)) - await nextTurn() - await domstack.settled() - return newCall(before) - } - /** @param {string} file @param {string} text */ - async function edit (file, text) { - await write(file, text) - return rebuild([{ type: 'change', file }]) - } - /** @param {string} file */ - const output = file => readFile(join(dest, 'docs', file), 'utf8') - /** @param {ProducerCall} watched @param {string[]} [eligible] */ - async function matchesFresh (watched, eligible = eligibleDocs) { - const snapshot = await docsSnapshot(join(dest, 'docs')) - assert.deepEqual(JSON.parse(await output('navigation.json')), watched.publicData, 'published data matches the producer return value') - const fresh = join(temp, 'fresh') - await rm(fresh, { recursive: true, force: true }) - const before = (await calls()).length - const report = await new DomStack(src, fresh, buildOptions).build() - assert.deepEqual(report.pageBuildResults?.errors, [], 'fresh build succeeded') - const call = await newCall(before) - assertRenders(call, 'reset', eligible) - assert.deepEqual(watched.publicData, call.publicData, 'incremental public data equals a full build') - assert.deepEqual(snapshot, await docsSnapshot(join(fresh, 'docs')), 'docs output paths and bytes equal a full build') - } - return { - src, - dest, - write, - rebuild, - edit, - output, - matchesFresh, - async start () { - const report = await domstack.watch({ serve: false }) - assert.deepEqual(report.pageBuildResults?.errors, [], 'initial watch build succeeded') - const call = await newCall(0) - assertRenders(call, 'reset', eligibleDocs) - return call - }, - } -} - -test('docs index resets render every eligible Markdown source in fresh workers', options, async t => { - const site = await instrumentedFixture(t) - const initial = await site.start() - assert.deepEqual(initial.publicData.docsNavigation.map(entry => entry.url), ['/docs/first/', '/docs/second/']) - assert.doesNotMatch(JSON.stringify(initial.publicData), /Not documentation|Not Markdown|Unpublished/) - await site.matchesFresh(initial) - await site.matchesFresh(initial) -}) - -test('body-only docs edits render one producer input and preserve public data and sibling mtimes', options, async t => { - const site = await instrumentedFixture(t) - const initial = await site.start() - const untouched = ['index.html', 'second/index.html', 'first/child/index.html', 'first/child/leaf/index.html', 'navigation.json'] - .map(file => join(site.dest, 'docs', file)) - const sentinel = new Date('2000-01-01T00:00:00Z') - for (const path of untouched) await utimes(path, sentinel, sentinel) - const mtimes = await Promise.all(untouched.map(async path => (await stat(path, { bigint: true })).mtimeNs)) - const call = await site.edit('docs/first/README.md', firstDoc.replace('Body', 'Only this body changed')) - assert.deepEqual(call.publicData, initial.publicData, 'both navigation and index HTML are unchanged') - assert.match(await site.output('first/index.html'), /Only this body changed/) - assert.deepEqual(await Promise.all(untouched.map(async path => (await stat(path, { bigint: true })).mtimeNs)), mtimes) - await site.matchesFresh(call) - assertRenders(call, 'delta', ['docs/first/README.md']) -}) - -test('heading edits render one producer input and refresh sibling navigation and the docs index', options, async t => { - const site = await instrumentedFixture(t) - const initial = await site.start() - const call = await site.edit('docs/first/README.md', firstDoc.replace('## Original', '## Renamed')) - assert.notDeepEqual(call.publicData.docsNavigation, initial.publicData.docsNavigation) - assert.notEqual(call.publicData.docsIndexHtml, initial.publicData.docsIndexHtml) - for (const file of ['index.html', 'second/index.html']) { - const html = await site.output(file) - assert.match(html, /first\/#renamed/) - assert.doesNotMatch(html, /first\/#original/) - } - await site.matchesFresh(call) - assertRenders(call, 'delta', ['docs/first/README.md']) -}) - -test('docs adds, renames and deletes reconcile navigation and output ownership like fresh builds', options, async t => { - const site = await instrumentedFixture(t) - await site.start() - const added = 'docs/third/README.md' - const renamed = 'docs/third/renamed.md' - await t.test('add', async () => { - await site.write(added, doc('Third', 'docsGroup: Guides\ndocsOrder: 5\n')) - const call = await site.rebuild([{ type: 'add', file: added }]) - assert.deepEqual(call.publicData.docsNavigation.map(entry => entry.url), ['/docs/third/', '/docs/first/', '/docs/second/']) - await site.matchesFresh(call, [...eligibleDocs, added]) - assertRenders(call, 'delta', [added]) + const original = '

First

Original

Body

' + let first = page('/docs/first/', original) + const second = page('/docs/second/', '

Second

') + let pages = [ + first, + second, + page('/docs/', '{{{ data.docsIndexHtml }}}'), + page('/elsewhere/', '

Not documentation

'), + page('/docs/html/', '

Not Markdown

', 'html'), + ] + /** @param {GlobalDataChanges} changes @param {string[]} expectedRenders */ + async function run (changes, expectedRenders) { + rendered.length = 0 + const data = await produceDocsData({ + pages, + previousState: state, + changes, + setState (next) { state = structuredClone(next) }, + }) + assert.deepEqual(rendered.sort(), expectedRenders, 'actual renderInnerPage calls, including duplicates') + assert.deepEqual(Object.keys(data).sort(), ['docsIndexHtml', 'docsNavigation']) + return data + } + + const initial = await run({ kind: 'reset', reason: 'initial', events: [] }, ['/docs/first/', '/docs/second/']) + await t.test('reset renders eligible docs and publishes navigation and index links', () => { + assert.deepEqual(initial.docsNavigation, [ + { + title: 'First', + url: '/docs/first/', + sections: [{ title: 'Original', url: '/docs/first/#original', sections: [] }], + }, + { title: 'Second', url: '/docs/second/', sections: [] }, + ]) + assert.deepEqual(indexLinks(initial.docsIndexHtml), [ + ['first/', 'First'], ['first/#original', 'Original'], ['second/', 'Second'], + ]) + assert.deepEqual([...state?.keys() ?? []].sort(), [first.sourceId, second.sourceId]) }) - await t.test('rename a directory index to a flat page', async () => { - await rename(join(site.src, added), join(site.src, renamed)) - const call = await site.rebuild([{ type: 'unlink', file: added }, { type: 'add', file: renamed }]) - assert.deepEqual(call.publicData.docsNavigation.map(entry => entry.url), ['/docs/third/renamed.html', '/docs/first/', '/docs/second/']) - await assert.rejects(site.output('third/index.html'), { code: 'ENOENT' }) - assert.match(await site.output('index.html'), /third\/renamed\.html#original/) - await site.matchesFresh(call, [...eligibleDocs, renamed]) - assertRenders(call, 'delta', [renamed]) + await t.test('body delta renders only the changed doc and leaves public data unchanged', async () => { + first = page('/docs/first/', original.replace('Body', 'Changed body')) + pages[0] = first + const data = await run({ kind: 'delta', upserted: [first], removed: [], events: [] }, ['/docs/first/']) + assert.deepEqual(data, initial) }) - await t.test('delete without rendering surviving docs', async () => { - await rm(join(site.src, renamed)) - const call = await site.rebuild([{ type: 'unlink', file: renamed }]) - assert.deepEqual(call.publicData.docsNavigation.map(entry => entry.url), ['/docs/first/', '/docs/second/']) - await assert.rejects(site.output('third/renamed.html'), { code: 'ENOENT' }) - assert.doesNotMatch(await site.output('index.html'), /Third/) - await site.matchesFresh(call) - assertRenders(call, 'delta', []) + await t.test('heading delta renders only the changed doc and updates both public values', async () => { + first = page('/docs/first/', '

First

Renamed

Changed body

') + pages[0] = first + const data = await run({ kind: 'delta', upserted: [first], removed: [], events: [] }, ['/docs/first/']) + assert.deepEqual(data.docsNavigation, [ + { + title: 'First', + url: '/docs/first/', + sections: [{ title: 'Renamed', url: '/docs/first/#renamed', sections: [] }], + }, + initial.docsNavigation[1], + ]) + assert.deepEqual(indexLinks(data.docsIndexHtml), [ + ['first/', 'First'], ['first/#renamed', 'Renamed'], ['second/', 'Second'], + ]) + }) + await t.test('delete removes cached navigation without rendering surviving docs', async () => { + pages = pages.filter(source => source.sourceId !== first.sourceId) + const data = await run({ kind: 'delta', upserted: [], removed: [first.sourceId], events: [] }, []) + assert.deepEqual(data.docsNavigation, [initial.docsNavigation[1]]) + assert.deepEqual(indexLinks(data.docsIndexHtml), [['second/', 'Second']]) + assert.deepEqual([...state?.keys() ?? []], [second.sourceId]) }) -}) - -test('group, order and parent edits reproject cached docs without accumulating nested entries', options, async t => { - const site = await instrumentedFixture(t) - await site.start() - for (const [name, text, titles] of /** @type {const} */ ([ - ['group', secondDoc.replace('Reference', 'Basics'), ['Second', 'First']], - ['order', secondDoc.replace('Reference', 'Guides').replace('20', '5'), ['Second', 'First']], - ['reorder', secondDoc.replace('Reference', 'Guides').replace('20', '30'), ['First', 'Second']], - ])) { - await t.test(name, async () => { - const call = await site.edit('docs/second/README.md', text) - assert.deepEqual(call.publicData.docsNavigation.map(entry => entry.title), titles) - assert.equal(call.publicData.docsNavigation.find(entry => entry.title === 'Second')?.group, name === 'group' ? 'Basics' : 'Guides') - await site.matchesFresh(call) - assertRenders(call, 'delta', ['docs/second/README.md']) - }) - } - for (const parent of ['/docs/first/', undefined, '/docs/first/child/']) { - await t.test(`parent: ${parent ?? 'none'}`, async () => { - const text = leafDoc.replace('docsParent: /docs/first/child/\n', parent ? `docsParent: ${parent}\n` : '') - const call = await site.edit('docs/first/child/leaf/README.md', text) - const first = call.publicData.docsNavigation.find(entry => entry.url === '/docs/first/') - const child = first?.sections.find(entry => entry.url === '/docs/first/child/') - assert.ok(first && child) - const container = parent === first.url ? first.sections : parent === child.url ? child.sections : call.publicData.docsNavigation - assert.equal(container.filter(entry => entry.url === '/docs/first/child/leaf/').length, 1, 'leaf appears once beneath its current parent') - await site.matchesFresh(call) - assertRenders(call, 'delta', ['docs/first/child/leaf/README.md']) - }) - } -}) - -test('producer, navigation helper and Markdown settings edits reset all eligible docs', options, async t => { - const site = await instrumentedFixture(t) - const initial = await site.start() - for (const file of ['site/globals/producer.ts', 'site/layouts/docs/navigation.js', 'site/globals/markdown-it.settings.js']) { - await t.test(file, async () => { - const text = await readFile(join(site.src, file), 'utf8') - const call = await site.edit(file, text + '\n// Trigger a dependency reset.\n') - assert.deepEqual(call.publicData, initial.publicData) - await site.matchesFresh(call) - assertRenders(call, 'reset', eligibleDocs) - }) - } }) diff --git a/site/layouts/docs/navigation.test.js b/site/layouts/docs/navigation.test.js index 60ec201c..a841bb51 100644 --- a/site/layouts/docs/navigation.test.js +++ b/site/layouts/docs/navigation.test.js @@ -112,35 +112,35 @@ test('navigation parents must be existing ancestor pages', async () => { } }) -test('navigation projections reuse cloneable records without mutating cached headings or nesting', async () => { - let renders = 0 +test('repeated navigation projections do not mutate cached headings or nesting', async () => { const pages = [ page('/docs/guide/', '# Guide\n\n## Overview\n\n### Details'), - page('/docs/guide/child/', '# Child\n\n## Child section', { docsParent: '/docs/guide/' }), - page('/docs/guide/child/grandchild/', '# Grandchild', { docsParent: '/docs/guide/child/' }), + page('/docs/guide/child/', '# Child', { docsParent: '/docs/guide/' }), ] - for (const source of pages) { - const render = source.renderInnerPage - source.renderInnerPage = (...args) => { renders++; return render(...args) } - } const records = (await Promise.all(pages.map(readDocsNavigationPage))).filter(record => record !== undefined) const retained = structuredClone(records) - const expected = projectDocsNavigation(retained) - for (let i = 0; i < 3; i++) { - assert.deepEqual(projectDocsNavigation(retained), expected) - assert.deepEqual(retained, records, 'projection must not attach child pages to retained heading arrays') - } - const changed = projectDocsNavigation(retained) - assert.ok(changed[0]?.sections[0]) - changed[0].sections[0].title = 'Consumer mutation' - assert.deepEqual(projectDocsNavigation(retained), expected, 'public entries do not alias retained records') - assert.equal(renders, 3, 'only initial record extraction renders source pages') + const expected = [{ + title: 'Guide', + url: '/docs/guide/', + sections: [ + { + title: 'Overview', + url: '/docs/guide/#overview', + sections: [{ title: 'Details', url: '/docs/guide/#details', sections: [] }], + }, + { title: 'Child', url: '/docs/guide/child/', sections: [] }, + ], + }] + const projected = projectDocsNavigation(retained) + assert.deepEqual(projected, expected) + assert.deepEqual(projectDocsNavigation(retained), expected, 'child pages must not accumulate on repeated projection') + assert.deepEqual(retained, records, 'projection must leave cached headings and nesting unchanged') - const survivors = retained.filter(record => record.entry.url !== '/docs/guide/child/grandchild/') - assert.deepEqual(projectDocsNavigation(survivors), await collectDocsNavigation(pages.slice(0, 2))) - const orphans = retained.filter(record => record.entry.url !== '/docs/guide/') - assert.throws(() => projectDocsNavigation(orphans), /Invalid documentation parent/) - assert.deepEqual(retained, records, 'even a failed projection leaves the retained records unchanged') + const heading = projected[0]?.sections[0]?.sections[0] + assert.ok(heading) + heading.title = 'Consumer mutation' + assert.deepEqual(retained, records, 'even nested public headings must not alias cached records') + assert.deepEqual(projectDocsNavigation(retained), expected) }) test('navigation links preserve deployment prefixes for directory and flat pages', () => { diff --git a/test-cases/incremental-global-data/helpers.js b/test-cases/incremental-global-data/helpers.js index c1cb52b2..b7c246b3 100644 --- a/test-cases/incremental-global-data/helpers.js +++ b/test-cases/incremental-global-data/helpers.js @@ -1,36 +1,30 @@ /** * @import { TestContext } from 'node:test' * @import { FSWatcher } from 'chokidar' - - * @import { Results, DomStackOpts } from '../../lib/builder.js' * @typedef {{ type: 'added' | 'removed' | 'change', filepath: string }} InputEvent * @typedef {object} ProducerCall * @property {'reset' | 'delta'} kind * @property {string} [reason] * @property {InputEvent[]} events - * @property {string[]} pages Source-root-relative POSIX IDs, as are all semantic observations below. - * @property {string[]} upserted + * @property {string[]} upserted Source-root-relative IDs. * @property {string[]} removed * @property {string[]} rendered * @property {string[] | null} previousKeys - * @property {boolean} previousIsMap - * @property {number} invocation - * @property {number} threadId - * @property {{ navigation: { url: string, title: string, tag: string | null, site: string, section: string }[], search: { url: string, html: string }[] }} publicData + * @typedef {{ url: string, title: string, html: string }} IndexRow */ import assert from 'node:assert/strict' import { EventEmitter } from 'node:events' - -import { mkdir, mkdtemp, readFile, readdir, rm, writeFile } from 'node:fs/promises' +import { mkdir, mkdtemp, readFile, rm, writeFile } from 'node:fs/promises' import { dirname, join } from 'node:path' import { setImmediate as nextTurn, setTimeout as delay } from 'node:timers/promises' import chokidar from 'chokidar' import pino from 'pino' import { DomStack } from '../../index.js' +import { startWatch } from '../watch/helpers.js' /** @param {string} heading @param {string} [body] @param {string} [frontmatter] */ export function article (heading, body = 'Original body.', frontmatter = '') { - return `---\narticle: true\nlayout: article\n${frontmatter}---\n# ${heading}\n\n${body}\n` + return `---\narticle: true\n${frontmatter}---\n# ${heading}\n\n${body}\n` } /** @param {() => Promise} predicate @param {string} message */ @@ -43,12 +37,12 @@ export async function waitFor (predicate, message) { } /** - * Only source event delivery is mocked; discovery, dependency analysis, workers, - * rendering, writes, cleanup, esbuild and copy watchers use the real implementation. + * Deterministic tests mock only source event delivery; native tests use real watchers. + * Builds, workers, rendering and cleanup are real in both modes. * @param {TestContext} t - * @param {{ native?: boolean, files?: Record, opts?: DomStackOpts }} [options] + * @param {{ native?: boolean, files?: Record }} [options] */ -export async function fixture (t, { native = false, files = {}, opts = {} } = {}) { +export async function fixture (t, { native = false, files = {} } = {}) { const root = await mkdtemp(join(import.meta.dirname, '.tmp-')) const src = join(root, 'src') const dest = join(root, 'public') @@ -56,34 +50,24 @@ export async function fixture (t, { native = false, files = {}, opts = {} } = {} const gate = join(root, 'hold-producer') const producerFailure = join(root, 'fail-producer') const renderFailure = join(root, 'fail-render') - const logger = pino({ level: 'silent' }) - const buildOptions = { metafile: false, domstackManifest: false, logger, ...opts } - const dom = new DomStack(src, dest, buildOptions) + const dom = new DomStack(src, dest, { metafile: false, domstackManifest: false, logger: pino({ level: 'silent' }) }) /** @type {FSWatcher | undefined} */ let watcher - /** @type {Parameters[1]} */ - let watcherOptions - let nativeEventCount = 0 - const watch = chokidar.watch - /** @param {Parameters} args */ - const sourceWatch = (...args) => { - if (args[0] !== src) return watch(...args) - watcherOptions = args[1] - if (native) { - watcher = watch(...args) - watcher.on('all', () => { nativeEventCount++ }) - } else { - const events = new EventEmitter() - const fake = Object.assign(events, { + if (!native) { + const watch = chokidar.watch + /** @param {Parameters} args */ + const sourceWatch = (...args) => { + if (args[0] !== src) return watch(...args) + const fake = Object.assign(new EventEmitter(), { closed: false, async close () { fake.closed = true }, }) watcher = /** @type {FSWatcher} */ (/** @type {unknown} */ (fake)) setImmediate(() => { if (!fake.closed) fake.emit('ready') }) + return watcher } - return watcher + t.mock.method(chokidar, 'watch', sourceWatch) } - t.mock.method(chokidar, 'watch', sourceWatch) t.after(async () => { await rm(gate, { force: true }) try { @@ -100,54 +84,31 @@ export async function fixture (t, { native = false, files = {}, opts = {} } = {} await writeFile(path, content) } const producer = ` -import assert from 'node:assert/strict' import { appendFileSync, existsSync } from 'node:fs' import { setTimeout as delay } from 'node:timers/promises' -import { threadId } from 'node:worker_threads' import { prefix } from './producer-middle.js' -let invocation = 0 export default async ({ pages, previousState, changes, setState }) => { - assert.ok(previousState === undefined || previousState instanceof Map) - const previousKeys = previousState === undefined ? null : [...previousState.keys()].sort() const state = changes.kind === 'reset' ? new Map() : new Map(previousState) - const changed = changes.kind === 'reset' ? pages : changes.upserted const rendered = [] for (const sourceId of changes.removed ?? []) state.delete(sourceId) - for (const page of changed) { - assert.ok(pages.includes(page), 'upserts are current initialized PageData instances') - const sourceId = page.sourceId - assert.equal(typeof sourceId, 'string') - if (!page.vars.article) { state.delete(sourceId); continue } - rendered.push(sourceId) - state.set(sourceId, { + for (const page of changes.kind === 'reset' ? pages : changes.upserted) { + if (!page.vars.article) { state.delete(page.sourceId); continue } + rendered.push(page.sourceId) + state.set(page.sourceId, { url: page.pageInfo.url, title: page.vars.title, - tag: page.vars.tag ?? null, - site: page.vars.site, - section: page.vars.section, - html: await page.renderFullPage(), + html: prefix + await page.renderFullPage(), }) } - const rows = [...state.values()].sort((a, b) => a.url.localeCompare(b.url)) - const publicData = { - navigation: rows.map(({ html, ...row }) => row), - search: rows.map(row => ({ url: row.url, html: prefix + row.html })), - } - // Capture this attempt's failure before the gate; a buffered retry can then recover. - const fail = existsSync(${JSON.stringify(producerFailure)}) - if (fail) state.set('failed-candidate', {}) setState(state) - // Both mutations must be isolated from the explicit snapshot crossing workers. - state.set('post-setState-mutation', {}) - previousState?.clear() + // Capture failure before the gate so a buffered retry can recover. + const fail = existsSync(${JSON.stringify(producerFailure)}) appendFileSync(${JSON.stringify(log)}, JSON.stringify({ kind: changes.kind, reason: changes.reason, events: changes.events.map(({ type, filepath }) => ({ type, filepath })), - pages: pages.map(page => page.sourceId).sort(), upserted: (changes.upserted ?? []).map(page => page.sourceId).sort(), removed: [...(changes.removed ?? [])].sort(), rendered: rendered.sort(), - previousKeys, previousIsMap: previousState instanceof Map, - invocation: ++invocation, threadId, publicData, + previousKeys: previousState === undefined ? null : [...previousState.keys()].sort(), }) + '\\n') const deadline = Date.now() + 8000 while (existsSync(${JSON.stringify(gate)})) { @@ -155,36 +116,21 @@ export default async ({ pages, previousState, changes, setState }) => { await delay(10) } if (fail) throw new Error('intentional producer failure after setState') - return publicData + return { index: [...state.values()].sort((a, b) => a.url.localeCompare(b.url)) } } ` const defaults = { - 'global.vars.js': "import { site } from './vars-middle.js'; export default { layout: 'root', site }\n", - 'vars-middle.js': "import { site } from './vars-leaf.js'; export { site }\n", - 'vars-leaf.js': "export const site = 'Site one'\n", 'global.data.js': producer, 'producer-middle.js': "import { prefix } from './producer-leaf.js'; export { prefix }\n", - 'producer-leaf.js': "export const prefix = 'Search one: '\n", - 'markdown-it.settings.js': "import { html } from './markdown-middle.js'; export default md => md.set({ html })\n", - 'markdown-middle.js': "import { html } from './markdown-leaf.js'; export { html }\n", - 'markdown-leaf.js': 'export const html = true\n', - 'root.layout.js': 'export default ({ children }) => children\n', - 'article.layout.js': "import { section } from './layout-middle.js'; export const vars = { section }; export default ({ children }) => '
' + section + children + '
'\n", - 'layout-middle.js': "import { section } from './layout-leaf.js'; export { section }\n", - 'layout-leaf.js': "export const section = 'Section one'\n", + 'producer-leaf.js': "export const prefix = 'Search: '\n", 'a/page.md': article('Alpha'), - 'a/page.vars.js': "import { tag } from '../companion-middle.js'; export default { tag }\n", - 'companion-middle.js': "import { tag } from './companion-leaf.js'; export { tag }\n", - 'companion-leaf.js': "export const tag = 'Tag one'\n", 'b/page.md': article('Beta'), - 'nav/page.js': "export const vars = { dataDeps: ['navigation'] }; export default ({ data }) => JSON.stringify(data.navigation)\n", 'data.json.template.js': `import { existsSync } from 'node:fs' -export const dataDeps = ['navigation', 'search'] +export const dataDeps = ['index'] export default ({ data }) => { if (existsSync(${JSON.stringify(renderFailure)})) throw new Error('intentional later render failure') - return JSON.stringify({ navigation: data.navigation, search: data.search }) + return JSON.stringify(data.index) }\n`, - 'summary.pages.js': "export const dataDeps = ['navigation']; export default ({ data }) => ({ outputName: 'summary.html', children: JSON.stringify(data.navigation) })\n", ...files, } await Promise.all(Object.entries(defaults).map(([name, content]) => write(name, content))) @@ -199,99 +145,43 @@ export default ({ data }) => { } /** @param {InputEvent[]} events */ const emit = events => { - assert.ok(watcher, 'source watcher must exist before initial discovery/build finishes') + assert.ok(watcher, 'source watcher exists before initial build finishes') for (const { type, filepath } of events) { watcher.emit(type === 'added' ? 'add' : type === 'removed' ? 'unlink' : 'change', filepath) } } - /** @param {string} name @param {InputEvent['type']} [type] @returns {InputEvent} */ - const event = (name, type = 'change') => ({ type, filepath: join(src, name) }) - /** @param {InputEvent[]} events */ - const rebuild = async events => { - const count = (await calls()).length - emit(events) - await nextTurn() - await dom.settled() - const added = (await calls()).slice(count) - assert.equal(added.length, 1, 'one producer invocation per event batch') - const call = added[0] - assert.ok(call) - return call - } return { - root, src, dest, dom, write, calls, emit, - event, - rebuild, gate, producerFailure, renderFailure, - async settleNative () { - assert.ok(native, 'quiescence waits are only for native source events') - await waitFor(async () => { - await dom.settled() - const count = nativeEventCount - // Allow delayed native notifications and the watcher's 300ms atomic window. - await delay(500) - await dom.settled() - return nativeEventCount === count - }, 'native source notifications become quiescent') + /** @returns {Promise} */ + async data () { return JSON.parse(await readFile(join(dest, 'data.json'), 'utf8')) }, + /** @param {string} name @param {InputEvent['type']} [type] @returns {InputEvent} */ + event: (name, type = 'change') => ({ type, filepath: join(src, name) }), + /** @param {InputEvent[]} events */ + async rebuild (events) { + const count = (await calls()).length + emit(events) + await nextTurn() + await dom.settled() + const added = (await calls()).slice(count) + assert.equal(added.length, 1, 'one producer invocation per event batch') + const call = added[0] + assert.ok(call) + return call }, - watcherOptions: () => watcherOptions, - build: () => new DomStack(src, join(root, 'standalone'), buildOptions).build(), async start () { - const report = await dom.watch({ serve: false }) - + const report = native ? await startWatch(t, dom, src) : await dom.watch({ serve: false }) assert.deepEqual(report.pageBuildResults?.errors, [], 'initial page build succeeded') return report }, - async matchesFresh () { - const watchedOutputs = await outputSnapshot(dest) - const watchedData = JSON.parse(await readFile(join(dest, 'data.json'), 'utf8')) - const freshDest = join(root, 'fresh') - await rm(freshDest, { recursive: true, force: true }) - const report = await new DomStack(src, freshDest, buildOptions).build() - assert.deepEqual(report.pageBuildResults?.errors, [], 'fresh page build succeeded') - const freshCall = (await calls()).at(-1) - assert.ok(freshCall) - assert.equal(freshCall.kind, 'reset') - assert.equal(freshCall.previousKeys, null) - assert.deepEqual(freshCall.events, []) - assert.deepEqual(watchedData, freshCall.publicData, 'public data matches a from-scratch producer') - assert.deepEqual(watchedOutputs, await outputSnapshot(freshDest), 'all public output names and bytes match a fresh build') - }, - } -} - -/** @param {Results} report */ -export function assertPrivateReport (report) { - assert.deepEqual(report.pageBuildResults?.errors, [], 'page build succeeded') - const pageReport = report.pageBuildResults?.report - assert.ok(pageReport) - for (const key of ['globalDataBaseline', 'previousGlobalDataBaseline', 'globalDataInputChanges', 'previousState']) { - assert.equal(Object.hasOwn(pageReport, key), false, `${key} stays private`) - assert.equal(Object.hasOwn(report, key), false, `${key} is not a public result`) - } -} - -/** @param {string} dir @returns {Promise>} */ -async function outputSnapshot (dir) { - /** @type {Record} */ - const files = {} - for (const entry of await readdir(dir, { withFileTypes: true })) { - const path = join(dir, entry.name) - if (entry.isDirectory()) { - for (const [name, content] of Object.entries(await outputSnapshot(path))) files[join(entry.name, name)] = content - } else { - files[entry.name] = await readFile(path, 'utf8') - } } - return files } /** @@ -303,20 +193,13 @@ async function outputSnapshot (dir) { export function assertDelta (call, upserted, removed = []) { assert.ok(call) assert.equal(call.kind, 'delta') - assert.equal(call.previousIsMap, true, 'Map state survives the worker boundary') assert.deepEqual(call.upserted, [...upserted].sort()) assert.deepEqual(call.removed, [...removed].sort()) - assert.equal(call.invocation, 1, 'each build loads the producer in a fresh worker') - assert.ok(call.previousKeys?.every(key => !key.includes('mutation') && key !== 'failed-candidate')) } /** @param {ProducerCall | undefined} call @returns {asserts call is ProducerCall} */ export function assertReset (call) { assert.ok(call) assert.equal(call.kind, 'reset') - assert.equal(typeof call.reason, 'string') - assert.ok(call.reason) assert.equal(call.previousKeys, null) - assert.equal(call.previousIsMap, false) - assert.equal(call.invocation, 1) } diff --git a/test-cases/incremental-global-data/index.test.js b/test-cases/incremental-global-data/index.test.js index 43a717da..e462e998 100644 --- a/test-cases/incremental-global-data/index.test.js +++ b/test-cases/incremental-global-data/index.test.js @@ -1,273 +1,124 @@ -/** - * @import { InputEvent, ProducerCall } from './helpers.js' - */ import { test } from 'node:test' import assert from 'node:assert/strict' -import fsPromises, { readFile, rename, rm, stat, utimes, writeFile } from 'node:fs/promises' +import fsPromises, { readFile, rm, stat, writeFile } from 'node:fs/promises' import { syncBuiltinESMExports } from 'node:module' import { join } from 'node:path' -import { article, assertDelta, assertPrivateReport, assertReset, fixture, waitFor } from './helpers.js' +import { article, assertDelta, assertReset, fixture, waitFor } from './helpers.js' const options = { timeout: 30_000 } -test('standalone builds reset state and public build/watch reports do not expose private candidates', options, async t => { +test('cached index adds, updates and removes distinct source IDs without rerendering unchanged rows', options, async t => { const site = await fixture(t) - const first = await site.build() - const second = await site.build() - const calls = await site.calls() - assert.equal(calls.length, 2) - for (const call of calls) { - assertReset(call) - assert.deepEqual(call.events, []) - } - assert.deepEqual(calls[0]?.publicData, calls[1]?.publicData) - const watched = await site.start() - await site.matchesFresh() - await t.test('standalone reports', () => { - assertPrivateReport(first) - assertPrivateReport(second) - }) - await t.test('watch report', () => assertPrivateReport(watched)) -}) - -test('body-only edits render one producer input, preserve navigation sibling mtime, and match a fresh build', options, async t => { - const site = await fixture(t) - await site.start() + const report = await site.start() + assert.equal(report.pageBuildResults?.report.globalDataBaseline, undefined, 'retained state is private to the watch session') const initial = (await site.calls())[0] assertReset(initial) assert.deepEqual(initial.events, []) - assert.deepEqual(initial.pages, ['a/page.md', 'b/page.md', 'nav/page.js'], 'templates and generated pages are not producer inputs') assert.deepEqual(initial.rendered, ['a/page.md', 'b/page.md']) + const original = await site.data() + assert.deepEqual(original.map(row => row.title), ['Alpha', 'Beta']) - const untouched = ['nav/index.html', 'b/index.html', 'summary.html'].map(name => join(site.dest, name)) - const sentinel = new Date('2000-01-01T00:00:00Z') - for (const path of untouched) await utimes(path, sentinel, sentinel) - const times = await Promise.all(untouched.map(async path => (await stat(path, { bigint: true })).mtimeNs)) - const event = site.event('a/page.md') - await site.write('a/page.md', article('Alpha', 'Only this body changed.')) - const call = await site.rebuild([event]) - assertDelta(call, ['a/page.md']) - assert.notEqual(call.threadId, initial.threadId) - assert.deepEqual(call.previousKeys, initial.rendered) - assert.deepEqual(call.events, [event]) - assert.deepEqual(call.rendered, ['a/page.md']) - assert.deepEqual(call.publicData.navigation, initial.publicData.navigation) - assert.notDeepEqual(call.publicData.search, initial.publicData.search) - assert.match(await readFile(join(site.dest, 'a/index.html'), 'utf8'), /Only this body changed/) - assert.deepEqual(await Promise.all(untouched.map(async path => (await stat(path, { bigint: true })).mtimeNs)), times) - await site.matchesFresh() + await site.write('c/page.md', article('Gamma')) + const added = await site.rebuild([site.event('c/page.md', 'added')]) + assertDelta(added, ['c/page.md']) + assert.deepEqual(added.rendered, ['c/page.md']) + assert.deepEqual((await site.data()).map(row => row.title), ['Alpha', 'Beta', 'Gamma']) - // A standalone build on the same source must not replace the live session's state. - await site.write('b/page.md', article('Beta', 'A second edit.')) - const second = await site.rebuild([site.event('b/page.md')]) - assertDelta(second, ['b/page.md']) - assert.deepEqual(second.rendered, ['b/page.md']) - await site.matchesFresh() -}) + await site.write('a/page.md', article('Updated Alpha', 'Updated body.')) + const event = site.event('a/page.md') + const updated = await site.rebuild([event]) + assertDelta(updated, ['a/page.md']) + assert.deepEqual(updated.events, [event]) + assert.deepEqual(updated.rendered, ['a/page.md']) + const data = await site.data() + assert.equal(data[0]?.title, 'Updated Alpha') + assert.match(data[0]?.html ?? '', /Updated body/) + assert.deepEqual(data[1], original[1], 'same-basename sibling retains its cached row') -test('H1, frontmatter title/metadata, companion vars, and transitive page/layout imports update input rows', options, async t => { - const site = await fixture(t, { - files: { - 'code/page.js': "import { content } from '../page-middle.js'; export const vars = { article: true, layout: 'article', title: 'Code' }; export default () => content\n", - 'page-middle.js': "import { content } from './page-leaf.js'; export { content }\n", - 'page-leaf.js': "export const content = 'Code one'\n", - }, - }) - await site.start() - const changes = [ - { name: 'a/page.md', content: article('New H1'), inputs: ['a/page.md'], expected: 'New H1' }, - { name: 'a/page.md', content: article('New H1', 'Body.', 'title: Explicit title\ntag: Frontmatter tag\n'), inputs: ['a/page.md'], expected: 'Explicit title' }, - { name: 'b/page.vars.js', content: "export default { tag: 'New companion' }\n", type: 'added', inputs: ['a/page.md', 'b/page.md', 'code/page.js', 'nav/page.js'], expected: 'New companion' }, - { name: 'b/page.vars.js', content: "export default { tag: 'Edited companion' }\n", inputs: ['b/page.md'], expected: 'Edited companion' }, - { name: 'a/page.md', content: article('New H1', 'Body.', 'title: Explicit title\n'), inputs: ['a/page.md'], expected: 'Tag one' }, - { name: 'companion-leaf.js', content: "export const tag = 'Imported companion'\n", inputs: ['a/page.md'], expected: 'Imported companion' }, - { name: 'page-leaf.js', content: "export const content = 'Code two'\n", inputs: ['code/page.js'], expected: 'Code two' }, - { name: 'layout-leaf.js', content: "export const section = 'Section two'\n", inputs: ['a/page.md', 'b/page.md', 'code/page.js'], expected: 'Section two' }, - ] - for (const change of changes) { - await t.test(change.name + ': ' + change.expected, async () => { - await site.write(change.name, change.content) - const event = site.event(change.name, /** @type {InputEvent['type']} */ (change.type ?? 'change')) - const call = await site.rebuild([event]) - assertDelta(call, change.inputs) - assert.deepEqual(call.events, [event]) - assert.match(JSON.stringify(call.publicData), new RegExp(change.expected)) - await site.matchesFresh() - }) - } + await rm(join(site.src, 'a/page.md')) + const removed = await site.rebuild([site.event('a/page.md', 'removed')]) + assertDelta(removed, [], ['a/page.md']) + assert.deepEqual(removed.rendered, []) + assert.deepEqual(await site.data(), data.slice(1)) + await assert.rejects(stat(join(site.dest, 'a/index.html')), { code: 'ENOENT' }) }) test.todo('re-exported page helpers shared with templates upsert affected source pages (https://github.com/bcomnes/domstack/issues/328)') test.todo('named, star, and namespace re-exports of global-data helpers trigger an index reset (https://github.com/bcomnes/domstack/issues/328)') -test('explicit imports through barrels upsert source pages shared with templates and reset producer inputs', options, async t => { +test('a shared dependency upserts a transitive source page even when a template imports it directly', options, async t => { const site = await fixture(t, { files: { - 'code/page.js': "import { content } from '../page-barrel.js'; export const vars = { article: true, layout: 'article', title: 'Barrel page' }; export default () => content\n", - 'page-barrel.js': "import { content } from './page-named.js'; export { content }\n", - 'page-named.js': "import { content } from './page-leaf.js'; export { content }\n", - 'page-leaf.js': "export const content = 'Barrel one'\n", + 'code/page.js': "import { content } from '../page-middle.js'; export const vars = { article: true, title: 'Code' }; export default () => content\n", + 'page-middle.js': "import { content } from './page-leaf.js'; export { content }\n", + 'page-leaf.js': "export const content = 'Code one'\n", 'shared.txt.template.js': "import { content } from './page-leaf.js'; export default () => content\n", - 'producer-middle.js': "import { prefix } from './producer-barrel.js'; export { prefix }\n", - 'producer-barrel.js': "import { prefix } from './producer-leaf.js'; export { prefix }\n", }, }) await site.start() - assert.equal(await readFile(join(site.dest, 'shared.txt'), 'utf8'), 'Barrel one') - await site.write('page-leaf.js', "export const content = 'Barrel two'\n") - const event = site.event('page-leaf.js') - const call = await site.rebuild([event]) - const source = 'code/page.js' - assertDelta(call, [source]) - assert.deepEqual(call.events, [event]) - assert.deepEqual(call.rendered, [source], 'a direct template consumer must not hide the source page reached through barrels') - assert.equal(await readFile(join(site.dest, 'shared.txt'), 'utf8'), 'Barrel two') - assert.match(await readFile(join(site.dest, 'code/index.html'), 'utf8'), /Barrel two/) - assert.ok(call.publicData.search.some(row => row.html.includes('Barrel two'))) - await site.matchesFresh() - - await site.write('producer-leaf.js', "export const prefix = 'Barrel search: '\n") - const producerEvent = site.event('producer-leaf.js') - const reset = await site.rebuild([producerEvent]) - assertReset(reset) - assert.equal(reset.reason, 'global-data-changed', 'producer barrels are recognized dependencies, not unknown-event fallbacks') - assert.deepEqual(reset.events, [producerEvent]) - assert.deepEqual(reset.rendered, ['a/page.md', 'b/page.md', 'code/page.js']) - assert.ok(reset.publicData.search.every(row => row.html.startsWith('Barrel search: '))) - await site.matchesFresh() -}) - -test('global vars, markdown settings, and producer roots and transitive imports reset the producer', options, async t => { - const site = await fixture(t, { files: { 'a/page.md': article('Alpha', 'Raw HTML') } }) - await site.start() - const changes = [ - ['vars-leaf.js', "export const site = 'Site two'\n", 'Site two'], - ['global.vars.js', "import { site } from './vars-middle.js'; export default { layout: 'root', site: site + ' direct' }\n", 'Site two direct'], - ['markdown-leaf.js', 'export const html = false\n', '<strong>'], - ['markdown-it.settings.js', "import { html } from './markdown-middle.js'; export default md => md.set({ html: !html })\n", ''], - ['producer-leaf.js', "export const prefix = 'Search two: '\n", 'Search two:'], - ['global.data.js', null, 'Search two:'], - ] - for (const [name, content, expected] of changes) { - assert.ok(name && expected) - await t.test(name, async () => { - await site.write(name, content ?? (await readFile(join(site.src, name), 'utf8')) + '\n// Edited producer entry.\n') - const event = site.event(name) - const call = await site.rebuild([event]) - assertReset(call) - assert.deepEqual(call.events, [event]) - assert.deepEqual(call.rendered, ['a/page.md', 'b/page.md']) - assert.ok(JSON.stringify(call.publicData).includes(expected)) - await site.matchesFresh() - }) - } -}) - -test('same-basename source IDs keep separate cache entries and removals delete only the matching state', options, async t => { - const site = await fixture(t) - await site.start() - const initial = (await site.calls())[0] - assertReset(initial) - assert.deepEqual(initial.pages, ['a/page.md', 'b/page.md', 'nav/page.js']) - assert.deepEqual(initial.rendered, ['a/page.md', 'b/page.md']) - assert.deepEqual(initial.publicData.navigation.map(row => row.title), ['Alpha', 'Beta']) - - await rm(join(site.src, 'a/page.md')) - const event = site.event('a/page.md', 'removed') - const removed = await site.rebuild([event]) - assertDelta(removed, [], ['a/page.md']) - assert.deepEqual(removed.events, [event], 'watch events retain absolute filesystem paths') - assert.deepEqual(removed.previousKeys, ['a/page.md', 'b/page.md'], 'same basenames occupy distinct cache entries') - assert.deepEqual(removed.pages, ['b/page.md', 'nav/page.js']) - assert.deepEqual(removed.rendered, [], 'the surviving row comes from cached state') - assert.deepEqual(removed.publicData.navigation, initial.publicData.navigation.filter(row => row.title === 'Beta')) - assert.deepEqual(removed.publicData.search, initial.publicData.search.filter(row => row.url === initial.publicData.navigation[1]?.url)) - await assert.rejects(stat(join(site.dest, 'a/index.html')), { code: 'ENOENT' }) - await site.matchesFresh() - - await site.write('b/page.md', article('Surviving Beta')) - const edited = await site.rebuild([site.event('b/page.md')]) - assertDelta(edited, ['b/page.md']) - assert.deepEqual(edited.previousKeys, ['b/page.md'], 'the removed ID is absent from the next committed state') - assert.deepEqual(edited.rendered, ['b/page.md']) - assert.deepEqual(edited.publicData.navigation.map(row => row.title), ['Surviving Beta']) - await site.matchesFresh() + await site.write('page-leaf.js', "export const content = 'Code two'\n") + const call = await site.rebuild([site.event('page-leaf.js')]) + assertDelta(call, ['code/page.js']) + assert.deepEqual(call.rendered, ['code/page.js']) + assert.equal(await readFile(join(site.dest, 'shared.txt'), 'utf8'), 'Code two') + assert.match(await readFile(join(site.dest, 'code/index.html'), 'utf8'), /Code two/) + assert.ok((await site.data()).some(row => row.html.includes('Code two'))) }) -test('adds, deletes, renames, and draft eligibility reconcile relative source IDs and stale outputs', options, async t => { - for (const buildDrafts of [false, true]) { - await t.test(`buildDrafts: ${buildDrafts}`, async t => { - const site = await fixture(t, { opts: { buildDrafts } }) - await site.start() - await site.write('c/page.md', article('Gamma')) - const added = site.event('c/page.md', 'added') - const call = await site.rebuild([added]) - assertDelta(call, ['c/page.md']) - assert.deepEqual(call.events, [added]) - await site.matchesFresh() - - for (const [from, to] of /** @type {const} */ ([ - ['c/page.md', 'c/renamed.md'], - ['c/renamed.md', 'c/renamed.draft.md'], - ['c/renamed.draft.md', 'c/page.md'], - ])) { - await rename(join(site.src, from), join(site.src, to)) - const events = [site.event(from, 'removed'), site.event(to, 'added')] - const call = await site.rebuild(events) - const wasEligible = buildDrafts || !from.includes('.draft.') - const eligible = buildDrafts || !to.includes('.draft.') - assertDelta(call, eligible ? [to] : [], wasEligible ? [from] : []) - assert.deepEqual(call.events, events) - assert.equal(call.pages.includes(to), eligible) - assert.equal(call.publicData.navigation.some(row => row.title === 'Gamma'), eligible) - await site.matchesFresh() - } - await rm(join(site.src, 'c/page.md')) - const removed = site.event('c/page.md', 'removed') - const deleted = await site.rebuild([removed]) - assertDelta(deleted, [], ['c/page.md']) - assert.deepEqual(deleted.rendered, []) - assert.deepEqual(deleted.events, [removed]) - await assert.rejects(stat(join(site.dest, 'c/index.html')), { code: 'ENOENT' }) - await site.matchesFresh() - }) - } -}) - -test('one batch unions direct and transitive inputs, deduplicates upserts, and retains event order', options, async t => { +test('a global producer dependency resets the whole index, including in a mixed page batch', options, async t => { const site = await fixture(t) await site.start() - await site.write('a/page.md', article('Alpha', 'Batch body.')) - await site.write('companion-leaf.js', "export const tag = 'Batch tag'\n") - await site.write('b/page.md', article('Batch Beta')) - const events = [site.event('a/page.md'), site.event('companion-leaf.js'), site.event('a/page.md'), site.event('b/page.md')] + await site.write('a/page.md', article('Reset Alpha')) + await site.write('producer-leaf.js', "export const prefix = 'New search: '\n") + const events = [site.event('a/page.md'), site.event('producer-leaf.js')] const call = await site.rebuild(events) - const inputs = ['a/page.md', 'b/page.md'] - assertDelta(call, inputs) - assert.deepEqual(call.rendered, inputs) + assertReset(call) + assert.equal(call.reason, 'global-data-changed') assert.deepEqual(call.events, events) - await site.matchesFresh() + assert.deepEqual(call.rendered, ['a/page.md', 'b/page.md']) + const data = await site.data() + assert.deepEqual(data.map(row => row.title), ['Reset Alpha', 'Beta']) + assert.ok(data.every(row => row.html.startsWith('New search: '))) +}) - await site.write('a/page.md', article('Reset batch')) - await site.write('producer-leaf.js', "export const prefix = 'Reset batch: '\n") - const resetEvents = [site.event('a/page.md'), site.event('producer-leaf.js')] - const reset = await site.rebuild(resetEvents) - assertReset(reset) - assert.deepEqual(reset.events, resetEvents) - await site.matchesFresh() +test('tracked global-vars and Markdown-settings helpers reset the index with updated output', options, async t => { + const site = await fixture(t, { + files: { + 'global.vars.js': "import { site } from './vars-helper.js'; export default { layout: 'root', site }\n", + 'vars-helper.js': "export const site = 'Site one'\n", + 'root.layout.js': "export default ({ vars, children }) => '
' + vars.site + '
' + children\n", + 'markdown-it.settings.js': "import { html } from './markdown-helper.js'; export default md => md.set({ html })\n", + 'markdown-helper.js': 'export const html = true\n', + 'a/page.md': article('Alpha', 'Raw HTML'), + }, + }) + await site.start() + assert.match((await site.data())[0]?.html ?? '', /
Site one<\/header>.*Raw HTML<\/strong>/s) + + await site.write('vars-helper.js', "export const site = 'Site two'\n") + const vars = await site.rebuild([site.event('vars-helper.js')]) + assertReset(vars) + assert.equal(vars.reason, 'global-vars-changed') + assert.ok((await site.data()).every(row => row.html.includes('
Site two
'))) + + await site.write('markdown-helper.js', 'export const html = false\n') + const markdown = await site.rebuild([site.event('markdown-helper.js')]) + assertReset(markdown) + assert.equal(markdown.reason, 'markdown-settings-changed') + assert.match((await site.data())[0]?.html ?? '', /<strong>Raw HTML<\/strong>/) }) -test('events arriving during an active producer are buffered into the next unioned batch', options, async t => { +test('in-flight events become one ordered batch with deduplicated inputs', options, async t => { const site = await fixture(t) await site.start() await writeFile(site.gate, '') - await site.write('a/page.md', article('Alpha', 'Active build body.')) + await site.write('a/page.md', article('Active Alpha')) site.emit([site.event('a/page.md')]) try { await waitFor(async () => (await site.calls()).length === 2, 'active producer reaches its gate') + await site.write('a/page.md', article('Buffered Alpha')) await site.write('b/page.md', article('Buffered Beta')) - await site.write('companion-leaf.js', "export const tag = 'Buffered tag'\n") - const events = [site.event('b/page.md'), site.event('companion-leaf.js'), site.event('b/page.md')] + const events = [site.event('b/page.md'), site.event('a/page.md'), site.event('b/page.md')] site.emit(events) await rm(site.gate) await site.dom.settled() @@ -276,19 +127,19 @@ test('events arriving during an active producer are buffered into the next union assertDelta(calls[1], ['a/page.md']) assertDelta(calls[2], ['a/page.md', 'b/page.md']) assert.deepEqual(calls[2].events, events) - await site.matchesFresh() + assert.deepEqual(calls[2].rendered, ['a/page.md', 'b/page.md']) + assert.deepEqual((await site.data()).map(row => row.title), ['Buffered Alpha', 'Buffered Beta']) } finally { await rm(site.gate, { force: true }) } }) -test('watch registers before initial producer work and buffers startup changes and membership events', options, async t => { +test('watch buffers edits and additions during the initial producer build', options, async t => { const site = await fixture(t) await writeFile(site.gate, '') const starting = site.start() try { await waitFor(async () => (await site.calls()).length === 1, 'initial producer reaches its gate') - assert.equal(site.watcherOptions()?.ignoreInitial, true) await site.write('a/page.md', article('Startup Alpha')) await site.write('c/page.md', article('Startup Gamma')) const events = [site.event('a/page.md'), site.event('c/page.md', 'added')] @@ -301,56 +152,42 @@ test('watch registers before initial producer work and buffers startup changes a assertReset(calls[0]) assertDelta(calls[1], ['a/page.md', 'c/page.md']) assert.deepEqual(calls[1].events, events) - await site.matchesFresh() + assert.deepEqual((await site.data()).map(row => row.title), ['Startup Alpha', 'Beta', 'Startup Gamma']) } finally { await rm(site.gate, { force: true }) await starting } }) -test('setState from a failed producer or later render is not reused and recovery resumes deltas', options, async t => { - for (const phase of ['producer', 'render']) { - await t.test(phase, async t => { - const site = await fixture(t) - await site.start() - const before = await readFile(join(site.dest, 'data.json'), 'utf8') - const marker = phase === 'producer' ? site.producerFailure : site.renderFailure - await writeFile(marker, '') - await site.write('a/page.md', article('Alpha', 'Failed build body.')) - const failed = await site.rebuild([site.event('a/page.md')]) - assertDelta(failed, ['a/page.md']) - assert.equal(await readFile(join(site.dest, 'data.json'), 'utf8'), before, 'failed phase does not publish the new public-data template') - - await rm(marker) - await site.write('b/page.md', article('Recovered Beta')) - const event = site.event('b/page.md') - const recovered = await site.rebuild([event]) - assertReset(recovered) - assert.deepEqual(recovered.events, [event]) - assert.ok(JSON.stringify(recovered.publicData).includes('Failed build body.'), 'recovery recomputes inputs changed in the failed build too') - await site.matchesFresh() - - await site.write('a/page.md', article('Alpha', 'Successful delta after recovery.')) - const delta = await site.rebuild([site.event('a/page.md')]) - assertDelta(delta, ['a/page.md']) - assert.deepEqual(delta.rendered, ['a/page.md']) - await site.matchesFresh() - }) - } +test('a later render failure discards the producer candidate and recovery resumes deltas', options, async t => { + const site = await fixture(t) + await site.start() + const before = await site.data() + await writeFile(site.renderFailure, '') + await site.write('a/page.md', article('Failed Alpha')) + assertDelta(await site.rebuild([site.event('a/page.md')]), ['a/page.md']) + assert.deepEqual(await site.data(), before, 'failed template does not publish candidate data') + + await rm(site.renderFailure) + await site.write('b/page.md', article('Recovered Beta')) + assertReset(await site.rebuild([site.event('b/page.md')])) + assert.deepEqual((await site.data()).map(row => row.title), ['Failed Alpha', 'Recovered Beta']) + + await site.write('b/page.md', article('Delta Beta')) + assertDelta(await site.rebuild([site.event('b/page.md')]), ['b/page.md']) + assert.deepEqual((await site.data()).map(row => row.title), ['Failed Alpha', 'Delta Beta']) }) -test('cleanup failure discards producer state but retains new sidecar ownership for recovery', options, async t => { +test('cleanup failure discards state but retains sidecar ownership for recovery', options, async t => { const site = await fixture(t, { files: { - 'a/page.md': article('Initial Alpha', 'Initial body.', 'sidecar: initial.txt\n'), - 'a/page.vars.js': `export default {} -export const pageOutputs = ({ vars }) => ({ outputName: vars.sidecar, content: vars.title })\n`, + 'a/page.md': article('Initial Alpha', 'Body.', 'sidecar: initial.txt\n'), + 'a/page.vars.js': 'export const pageOutputs = ({ vars }) => ({ outputName: vars.sidecar, content: vars.title })\n', }, }) await site.start() const stale = join(site.dest, 'a/initial.txt') const candidate = join(site.dest, 'a/candidate.txt') - assert.equal(await readFile(stale, 'utf8'), 'Initial Alpha') const originalRm = fsPromises.rm let cleanupAttempts = 0 /** @param {Parameters} args */ @@ -363,45 +200,37 @@ export const pageOutputs = ({ vars }) => ({ outputName: vars.sidecar, content: v const cleanup = t.mock.method(fsPromises, 'rm', failFirstCleanup) syncBuiltinESMExports() try { - await site.write('a/page.md', article('Candidate Alpha', 'Candidate body.', 'sidecar: candidate.txt\n')) - const failed = await site.rebuild([site.event('a/page.md')]) - assertDelta(failed, ['a/page.md']) - assert.equal(cleanupAttempts, 1, 'failure occurs in stale-output cleanup, after rendering') + await site.write('a/page.md', article('Candidate Alpha', 'Body.', 'sidecar: candidate.txt\n')) + assertDelta(await site.rebuild([site.event('a/page.md')]), ['a/page.md']) + assert.equal(cleanupAttempts, 1) assert.equal(await readFile(stale, 'utf8'), 'Initial Alpha') - assert.equal(await readFile(candidate, 'utf8'), 'Candidate Alpha', 'new page-owned output was already written') - assert.deepEqual(JSON.parse(await readFile(join(site.dest, 'data.json'), 'utf8')), failed.publicData) + assert.equal(await readFile(candidate, 'utf8'), 'Candidate Alpha', 'new output was written before cleanup failed') - // Only B emits an event: failure recovery must rediscover A's final sidecar too. - await site.write('a/page.md', article('Recovered Alpha', 'Recovered body.', 'sidecar: recovered.txt\n')) - await site.write('b/page.md', article('Recovery Beta')) - const event = site.event('b/page.md') - const recovered = await site.rebuild([event]) - assertReset(recovered) - assert.deepEqual(recovered.events, [event]) - assert.equal(cleanupAttempts, 2) + // Only B emits an event: recovery must rediscover A and clean up both old sidecars. + await site.write('a/page.md', article('Recovered Alpha', 'Body.', 'sidecar: recovered.txt\n')) + await site.write('b/page.md', article('Recovered Beta')) + assertReset(await site.rebuild([site.event('b/page.md')])) assert.equal(await readFile(join(site.dest, 'a/recovered.txt'), 'utf8'), 'Recovered Alpha') await assert.rejects(stat(stale), { code: 'ENOENT' }) - await assert.rejects(stat(candidate), { code: 'ENOENT' }, 'sidecar written before cleanup failed must not be orphaned') - await site.matchesFresh() + await assert.rejects(stat(candidate), { code: 'ENOENT' }, 'candidate sidecar must not be orphaned') + assert.deepEqual((await site.data()).map(row => row.title), ['Recovered Alpha', 'Recovered Beta']) } finally { cleanup.mock.restore() syncBuiltinESMExports() } }) -test('events buffered during a failing producer batch automatically recover without retaining its candidate', options, async t => { +test('a buffered batch recovers automatically after the producer fails after setState', options, async t => { const site = await fixture(t) await site.start() await writeFile(site.producerFailure, '') await writeFile(site.gate, '') - await site.write('a/page.md', article('Alpha', 'Body from the failing batch.')) - const activeEvent = site.event('a/page.md') - site.emit([activeEvent]) + await site.write('a/page.md', article('Failed Alpha')) + site.emit([site.event('a/page.md')]) try { - await waitFor(async () => (await site.calls()).length === 2, 'failing producer snapshots its candidate and reaches the gate') - await site.write('b/page.md', article('Buffered recovery Beta')) - await site.write('companion-leaf.js', "export const tag = 'Buffered recovery tag'\n") - const events = [site.event('b/page.md'), site.event('companion-leaf.js')] + await waitFor(async () => (await site.calls()).length === 2, 'failing producer reaches its gate') + await site.write('b/page.md', article('Buffered Beta')) + const events = [site.event('b/page.md')] site.emit(events) // The active worker captured failure already; the next worker must succeed. await rm(site.producerFailure) @@ -410,161 +239,59 @@ test('events buffered during a failing producer batch automatically recover with const calls = await site.calls() assert.equal(calls.length, 3, 'initial, failed, and one buffered recovery build') assertDelta(calls[1], ['a/page.md']) - assert.deepEqual(calls[1].events, [activeEvent]) assertReset(calls[2]) assert.deepEqual(calls[2].events, events) - assert.deepEqual(calls[2].rendered, ['a/page.md', 'b/page.md']) - assert.ok(JSON.stringify(calls[2].publicData).includes('Body from the failing batch.')) - assert.ok(JSON.stringify(calls[2].publicData).includes('Buffered recovery tag')) - await site.matchesFresh() + assert.deepEqual((await site.data()).map(row => row.title), ['Failed Alpha', 'Buffered Beta']) - await site.write('b/page.md', article('Beta after buffered recovery')) - const delta = await site.rebuild([site.event('b/page.md')]) - assertDelta(delta, ['b/page.md']) - assert.deepEqual(delta.rendered, ['b/page.md']) - await site.matchesFresh() + await site.write('b/page.md', article('Delta Beta')) + assertDelta(await site.rebuild([site.event('b/page.md')]), ['b/page.md']) + assert.deepEqual((await site.data()).map(row => row.title), ['Failed Alpha', 'Delta Beta']) } finally { await rm(site.producerFailure, { force: true }) await rm(site.gate, { force: true }) } }) -test('independent sessions and a stopped/restarted instance never inherit each other\'s Map state', options, async t => { +test('independent sessions and a stopped/restarted instance do not inherit cached state', options, async t => { const first = await fixture(t) - const second = await fixture(t, { files: { 'a/page.md': article('Other session') } }) + const second = await fixture(t, { files: { 'a/page.md': article('Other Alpha') } }) await first.start() - await first.write('a/page.md', article('First session edited')) + await first.write('a/page.md', article('First Alpha')) await first.rebuild([first.event('a/page.md')]) await second.start() assertReset((await second.calls())[0]) await second.write('b/page.md', article('Other Beta')) - const other = await second.rebuild([second.event('b/page.md')]) - assertDelta(other, ['b/page.md']) - assert.deepEqual(other.previousKeys, ['a/page.md', 'b/page.md']) - assert.deepEqual(other.publicData.navigation.map(row => row.title), ['Other session', 'Other Beta']) - assert.equal((await first.calls()).length, 2) - await second.matchesFresh() + assertDelta(await second.rebuild([second.event('b/page.md')]), ['b/page.md']) + assert.deepEqual((await second.data()).map(row => row.title), ['Other Alpha', 'Other Beta']) + assert.deepEqual((await first.data()).map(row => row.title), ['First Alpha', 'Beta']) await first.dom.stopWatching() await first.write('b/page.md', article('Edited while stopped')) await first.start() - const restarted = (await first.calls()).at(-1) - assert.ok(restarted) - assertReset(restarted) - assert.deepEqual(restarted.events, []) - await first.write('a/page.md', article('Restart delta')) - const delta = await first.rebuild([first.event('a/page.md')]) - assertDelta(delta, ['a/page.md']) - await first.matchesFresh() + assertReset((await first.calls()).at(-1)) + await first.write('a/page.md', article('Restart Alpha')) + assertDelta(await first.rebuild([first.event('a/page.md')]), ['a/page.md']) + assert.deepEqual((await first.data()).map(row => row.title), ['Restart Alpha', 'Edited while stopped']) }) -test('native static JSON leaf edits reset producer, global-vars, and markdown-settings inputs', options, async t => { +test('a native imported JSON edit resets the producer and publishes every updated row', options, async t => { const site = await fixture(t, { native: true, files: { - 'a/page.md': article('Alpha', 'Raw JSON HTML'), - 'producer-leaf.js': "import settings from './producer.json' with { type: 'json' }; export const prefix = settings.prefix\n", - 'producer.json': JSON.stringify({ prefix: 'JSON search one: ' }), - 'vars-leaf.js': "import settings from './vars.json' with { type: 'json' }; export const site = settings.site\n", - 'vars.json': JSON.stringify({ site: 'JSON site one' }), - 'markdown-leaf.js': "import settings from './markdown.json' with { type: 'json' }; export const html = settings.html\n", - 'markdown.json': JSON.stringify({ html: true }), + 'producer-leaf.js': "import settings from './prefix.json' with { type: 'json' }; export const prefix = settings.prefix\n", + 'prefix.json': JSON.stringify({ prefix: 'Original prefix: ' }), }, }) await site.start() - const changes = [ - { name: 'producer.json', content: { prefix: 'JSON search two: ' }, reason: 'global-data-changed', expected: 'JSON search two: ' }, - { name: 'vars.json', content: { site: 'JSON site two' }, reason: 'global-vars-changed', expected: 'JSON site two' }, - { name: 'markdown.json', content: { html: false }, reason: 'markdown-settings-changed', expected: '<strong>' }, - ] - for (const change of changes) { - await t.test(change.name, async () => { - await site.settleNative() - const beforeEdit = (await site.calls()).length - await site.write(change.name, JSON.stringify(change.content)) - await waitFor(async () => (await site.calls()).length > beforeEdit, `native watcher dispatches ${change.name}`) - await site.settleNative() - const calls = (await site.calls()).slice(beforeEdit) - assertNativeEvents(calls, site.event(change.name)) - for (const call of calls) { - assertReset(call) - assert.equal(call.reason, change.reason) - assert.deepEqual(call.rendered, ['a/page.md', 'b/page.md']) - assert.ok(JSON.stringify(call.publicData).includes(change.expected)) - } - await site.matchesFresh() - }) - } -}) - -test('native atomic replacement updates one producer input and keeps watching the replacement', options, async t => { - const site = await fixture(t, { native: true }) - await site.start() - await site.settleNative() - const source = join(site.src, 'a/page.md') - const oldInode = (await stat(source)).ino - const replacement = join(site.root, 'replacement.md') - await writeFile(replacement, article('Atomic Alpha', 'Atomically replaced body.')) - const beforeEdit = (await site.calls()).length - await rename(replacement, source) - assert.notEqual((await stat(source)).ino, oldInode, 'replacement uses a new inode, not an in-place write') - await waitFor(async () => (await site.calls()).length > beforeEdit, 'native watcher dispatches the atomic replacement') - await site.settleNative() - const calls = (await site.calls()).slice(beforeEdit) - assertNativeEvents(calls, site.event('a/page.md')) - for (const replaced of calls) { - assertDelta(replaced, ['a/page.md']) - assert.deepEqual(replaced.rendered, ['a/page.md']) - } - assert.match(await readFile(join(site.dest, 'a/index.html'), 'utf8'), /Atomically replaced body/) - await site.matchesFresh() - - await site.settleNative() - const beforeNextEdit = (await site.calls()).length - await site.write('a/page.md', article('Atomic Alpha', 'Edited replacement body.')) - await waitFor(async () => (await site.calls()).length > beforeNextEdit, 'native watcher follows the replacement inode') - await site.settleNative() - const nextCalls = (await site.calls()).slice(beforeNextEdit) - assertNativeEvents(nextCalls, site.event('a/page.md')) - for (const edited of nextCalls) { - assertDelta(edited, ['a/page.md']) - assert.deepEqual(edited.rendered, ['a/page.md']) - } - assert.match(await readFile(join(site.dest, 'a/index.html'), 'utf8'), /Edited replacement body/) - await site.matchesFresh() -}) - -test('native chokidar body edits reach the same incremental public API', options, async t => { - const site = await fixture(t, { native: true }) - await site.start() - // Native startup may conservatively report unchanged files; measure only the edit. - await site.settleNative() - const beforeEdit = (await site.calls()).length - await site.write('a/page.md', article('Alpha', 'Native watcher body edit.')) - await waitFor(async () => (await site.calls()).length > beforeEdit, 'native watcher dispatches the source edit') - await site.settleNative() - const calls = (await site.calls()).slice(beforeEdit) - assertNativeEvents(calls, site.event('a/page.md')) - for (const edited of calls) { - assertDelta(edited, ['a/page.md']) - assert.deepEqual(edited.rendered, ['a/page.md']) - } - await site.matchesFresh() + assert.ok((await site.data()).every(row => row.html.startsWith('Original prefix: '))) + await site.write('prefix.json', JSON.stringify({ prefix: 'Updated prefix: ' })) + await waitFor(async () => (await site.data()).some(row => row.html.startsWith('Updated prefix: ')), 'native JSON edit reaches the published index') + + const call = (await site.calls()).at(-1) + assertReset(call) + assert.equal(call.reason, 'global-data-changed') + assert.deepEqual(call.rendered, ['a/page.md', 'b/page.md']) + const data = await site.data() + assert.deepEqual(data.map(row => row.title), ['Alpha', 'Beta']) + assert.ok(data.every(row => row.html.startsWith('Updated prefix: '))) }) - -/** @param {ProducerCall[]} calls @param {InputEvent} expectedEvent */ -function assertNativeEvents (calls, expectedEvent) { - // Native delivery is not exactly-once: an edit can repeat within or across batches. - // Deterministic tests retain exact invocation and input-render counts per batch. - const details = JSON.stringify({ - expectedEvent, - calls: calls.map(({ kind, reason, events, upserted, removed, rendered }) => ({ kind, reason, events, upserted, removed, rendered })), - }, null, 2) - assert.ok(calls.length > 0, `Expected at least one native edit invocation: ${details}`) - for (const call of calls) { - assert.ok(call.events.length > 0, `Native edit invocation has no events: ${details}`) - for (const event of call.events) { - assert.deepEqual(event, expectedEvent, `Unexpected native event or source path: ${details}`) - } - } -} diff --git a/test-cases/page-outputs/watch.test.js b/test-cases/page-outputs/watch.test.js index caccdc94..6091db54 100644 --- a/test-cases/page-outputs/watch.test.js +++ b/test-cases/page-outputs/watch.test.js @@ -1,39 +1,22 @@ -/** - * @import { TestContext } from 'node:test' - */ import { test } from 'node:test' import assert from 'node:assert/strict' import { rename, rm, stat, writeFile } from 'node:fs/promises' import { join } from 'node:path' -import { hook, setup as setupSite, settle, writeFiles } from './helpers.js' +import { hook, setup, settle, writeFiles } from './helpers.js' import { startWatch } from '../watch/helpers.js' -/** @param {TestContext} t @param {Record} files */ -async function setup (t, files) { - const fixture = await setupSite(t, files) - - return { - ...fixture, - async startWatch () { - const report = await startWatch(t, fixture.site, fixture.src) - assert.deepEqual(fixture.logs.filter(line => JSON.parse(line).level >= 50), [], 'startup builds must not fail') - return report - }, - } -} - const rawLayout = `export const vars = { dataDeps: ['navigation'] } export default ({ children, data }) => data.navigation + children export const pageOutputs = async ({ page }) => ({ outputName: 'source.txt', content: await page.readMarkdownContent() })` test('watch updates only changed raw content and retains unchanged ownership without a public manifest', { timeout: 30_000 }, async t => { - const { site, startWatch, src, dest, read, mtime, logs } = await setup(t, { + const { site, src, dest, read, mtime, logs } = await setup(t, { 'global.data.js': "export default { navigation: 'Navigation one' }", 'root.layout.js': rawLayout, 'a/page.md': '# Article A\n', 'b/page.md': '# Article B\n', }) - await startWatch() + await startWatch(t, site, src) const siblingRaw = await mtime('b/source.txt') const siblingHtml = await mtime('b/index.html') const originalRaw = await mtime('a/source.txt') @@ -62,8 +45,8 @@ test('watch updates only changed raw content and retains unchanged ownership wit }) test('watch reconciles companion addition, output rename, hook removal, companion removal and re-addition', { timeout: 30_000 }, async t => { - const { site, startWatch, src, dest, read, logs } = await setup(t, { 'article/page.html': '

Article

' }) - await startWatch() + const { site, src, dest, read, logs } = await setup(t, { 'article/page.html': '

Article

' }) + await startWatch(t, site, src) const companion = join(src, 'article/page.vars.js') await settle(site, logs, async () => { await writeFile(companion, 'export default {}; ' + hook('first.txt', 'first')) @@ -98,12 +81,12 @@ test('watch reconciles companion addition, output rename, hook removal, companio }) test('watch removes sidecars on source rename and draft exclusion', { timeout: 30_000 }, async t => { - const { site, startWatch, src, dest, read, logs } = await setup(t, { + const { site, src, dest, read, logs } = await setup(t, { 'root.layout.js': `export default ({ children }) => children export const pageOutputs = async ({ page }) => ({ outputName: page.outputName + '.txt', content: await page.readMarkdownContent() })`, 'article.md': '# Article\n', }) - await startWatch() + await startWatch(t, site, src) assert.equal(await read('article.html.txt'), '# Article\n') await settle(site, logs, async () => { await rename(join(src, 'article.md'), join(src, 'renamed.md')) @@ -121,13 +104,13 @@ test('watch removes sidecars on source rename and draft exclusion', { timeout: 3 }) test('watch hook failure retains partial writes and recovery removes old and partial outputs', { timeout: 30_000 }, async t => { - const { site, startWatch, src, dest, read, mtime, logs } = await setup(t, { + const { site, src, dest, read, mtime, logs } = await setup(t, { 'page.js': `export default () => 'old main'; export const pageOutputs = () => [ { outputName: 'old.txt', content: 'old sidecar' }, { outputName: 'stale.txt', content: 'retain until recovery' }, ]`, }) - await startWatch() + await startWatch(t, site, src) const oldHtmlTime = await mtime('index.html') await settle(site, logs, async () => { await writeFile(join(src, 'page.js'), `export default () => 'failed main'; export async function* pageOutputs () { @@ -154,11 +137,11 @@ test('watch hook failure retains partial writes and recovery removes old and par for (const change of ['source deletion', 'draft exclusion', 'hook removal', 'companion deletion', 'companion rename']) { test(`watch independently cleans up after ${change}`, { timeout: 15_000 }, async t => { - const { site, startWatch, src, dest, read, logs } = await setup(t, { + const { site, src, dest, read, logs } = await setup(t, { 'article/page.html': 'Article', 'article/page.vars.js': 'export default {}; ' + hook('owned.txt'), }) - await startWatch() + await startWatch(t, site, src) assert.equal(await read('article/owned.txt'), 'sidecar') const page = join(src, 'article/page.html') const companion = join(src, 'article/page.vars.js') @@ -179,11 +162,11 @@ for (const change of ['source deletion', 'draft exclusion', 'hook removal', 'com } test('provider precedence warnings reach the configured logger on initial and incremental watch builds', { timeout: 15_000 }, async t => { - const { site, startWatch, src, read, logs } = await setup(t, { + const { site, src, read, logs } = await setup(t, { 'page.js': "export default () => 'initial'; " + hook('selected.txt', 'page module'), 'page.vars.js': "export default {}; export const pageOutputs = () => { throw Error('ignored companion ran') }", }) - const result = await startWatch() + const result = await startWatch(t, site, src) assert.equal(result.pageBuildResults?.warnings.filter(warning => 'code' in warning && warning.code === 'DOM_STACK_WARNING_DUPLICATE_PAGE_OUTPUTS_PROVIDER').length, 1) const cursor = logs.length await settle(site, logs, async () => { @@ -202,13 +185,13 @@ test('provider precedence warnings reach the configured logger on initial and in }) test('hook-only data subscriptions invalidate their owner but not an unrelated sibling', { timeout: 30_000 }, async t => { - const { site, startWatch, src, read, mtime, logs } = await setup(t, { + const { site, src, read, mtime, logs } = await setup(t, { 'global.data.js': "export default { selected: 'first', unrelated: 'unchanged' }", 'a/page.html': 'A', 'a/page.vars.js': "export default { dataDeps: ['selected'] }; export const pageOutputs = ({ data }) => ({ outputName: 'data.txt', content: data.selected })", 'b/page.html': 'B', }) - await startWatch() + await startWatch(t, site, src) const mainTime = await mtime('a/index.html') const siblingTime = await mtime('b/index.html') await settle(site, logs, async () => { From 7e5d31df226403ef34dfa4e5e3e6ac4875c410bd Mon Sep 17 00:00:00 2001 From: Bret Comnes Date: Tue, 15 Sep 2026 12:12:36 -0700 Subject: [PATCH 11/14] Defer precise watch dependency tracking to a stacked follow-up --- docs/data/README.md | 32 +- docs/implementation/README.md | 16 +- index.js | 85 +-- lib/watch-plan.js | 59 +-- lib/watch-plan.test.js | 485 ++++++++---------- test-cases/incremental-global-data/helpers.js | 36 +- .../incremental-global-data/index.test.js | 40 +- 7 files changed, 327 insertions(+), 426 deletions(-) diff --git a/docs/data/README.md b/docs/data/README.md index 20c15639..5f3aa38d 100644 --- a/docs/data/README.md +++ b/docs/data/README.md @@ -144,8 +144,9 @@ When one layout calls another layout function directly, the composing layout mus - Keeps global data separate from ordinary `vars`, so derived values cannot silently collide with page or layout configuration. - Runs inside the worker process (same as all other dynamic imports) to avoid ESM caching issues. - No producer callback runs if no `global.data.*` file exists. -- In watch mode, DOMStack fingerprints each top-level returned value and rebuilds only consumers subscribed to changed keys. -- Editing `global.data.*` or one of its tracked statically imported helpers in the watched source tree resets retained state and recomputes data; a shared helper also rebuilds its direct page, layout, template, and factory consumers. +- During targeted watch rebuilds, DOMStack fingerprints each top-level returned value and adds only consumers subscribed to changed keys to the rebuild set. +- Editing `global.data.*` resets retained state and recomputes data. + When retaining state, watched module edits conservatively reset it and trigger a full rebuild: rediscover inputs, restart esbuild, and rebuild all pages and templates. - Values composed of JSON-safe primitives, arrays, and plain objects get stable fingerprints; opaque values such as functions, class instances, maps, sets, or cycles conservatively invalidate their subscribers on every page build. - A declaration naming a missing key fails the build, and access to an existing but undeclared key throws a focused error. @@ -359,13 +360,16 @@ After `global.data.ts` returns, consumers receive only the values named by their > Most sites do not need incremental global data. > Start with a regular `global.data.ts` callback and consider incremental indexing when a large number of pages makes watch rebuilds slow. -In watch mode, keep an index so you only recompute entries for affected source pages. +In watch mode, keep an index so Markdown/HTML source edits only require recomputing affected entries. The callback receives four fields: - `pages`: All current source pages, excluding generated pages. -- `previousState`: Your saved state, or `undefined` when starting fresh. It is isolated and safe to modify. -- `changes`: On `kind: 'reset'`, rebuild from `pages`. On `kind: 'delta'`, replace entries for `changes.upserted` and delete the IDs in `changes.removed`. -- `setState(next)`: Save a snapshot for the next successful build. Always call it to retain your updates. +- `previousState`: Your saved state, or `undefined` when starting fresh. + It is isolated and safe to modify. +- `changes`: On `kind: 'reset'`, rebuild from `pages`. + On `kind: 'delta'`, replace entries for `changes.upserted` and delete the IDs in `changes.removed`. +- `setState(next)`: Save a snapshot for the next successful build. + Always call it to retain your updates. Use `page.sourceId` as the index key: a read-only source-relative path such as `docs/data/README.md`, using `/` separators on every platform. `changes.removed` contains these same IDs. @@ -418,13 +422,19 @@ export default function ({ ``` A consuming layout declares `export const vars = { dataDeps: ['docsNavigation'] }` and reads `data.docsNavigation`. -If the returned navigation is unchanged, its subscribers do not rebuild just because the index was updated. +On a Markdown/HTML source edit, unchanged navigation does not cause additional subscriber rebuilds. For a larger example that also caches heading links, see `site/globals/global.data.ts` in this repository. ### Usage notes -- State lasts for the current watch session and is retained only after a successful build. Always handle resets, including after producer or settings changes and failed builds. +- Markdown/HTML source edits can use incremental deltas. + When retaining state, watched module edits—including JavaScript/TypeScript pages, page-variable modules, helpers, browser entries, and settings—reset the index and trigger a full rebuild. + Without retained state, existing watch rebuild scopes are unchanged. +- State lasts for the current watch session and is retained only after a successful build. + Always handle resets, including after producer or settings changes and failed builds. - Store cloneable records, arrays, or maps—not `PageData` instances, functions, or shared memory. -- An upsert may repeat even when content is unchanged. Replace its cached entry, or delete it if the page no longer belongs in your index. -- Relative static imports within the watched source tree are tracked. Re-exports (`export … from`) are not tracked yet ([#328](https://github.com/bcomnes/domstack/issues/328)); use an explicit import followed by a local export, or restart watching after those inputs change. -- Restart watching when other inputs change, such as arbitrary file reads, environment variables, or network data. +- An upsert may repeat even when content is unchanged. + Replace its cached entry, or delete it if the page no longer belongs in your index. +- Dependency tracking supports relative static imports, but the page watcher observes only processed file extensions within the watched source tree, not imported JSON files. + Re-exports (`export … from`) are not tracked yet ([#328](https://github.com/bcomnes/domstack/issues/328)); use an explicit import followed by a local export, or restart watching after those inputs change. +- Restart watching when imported JSON or other inputs change, such as arbitrary file reads, environment variables, or network data. diff --git a/docs/implementation/README.md b/docs/implementation/README.md index 0249ca49..610bde82 100644 --- a/docs/implementation/README.md +++ b/docs/implementation/README.md @@ -229,7 +229,7 @@ Require a full page retry`"] Bundle replanning returns only a page plan or a skip; it does not restart esbuild again. After a page-build failure, the next page-producing plan retries the complete page phase rather than trusting incremental filters. -Manifest-settings changes and service-worker entry additions or removals retain their intentional page-phase skips. +Without retained global-data state, manifest-settings changes and service-worker entry additions or removals retain their intentional page-phase skips. The one-shot manifest pipeline is not part of watch execution. > [!NOTE] @@ -245,11 +245,16 @@ DOMStack uses these rebuild scopes: - **Full page/template rebuild**: DOMStack renders every source-backed and generated page and every template without restarting esbuild. - **Full rebuild**: DOMStack rediscovers the source tree, restarts esbuild, renders all pages and templates, and refreshes its dependency maps. -Like templates, generated-pages modules rebuild when their own source or imported dependencies change. +Like templates, generated-pages modules rebuild when their own source or watched imported dependencies change. When a targeted build recomputes global data, DOMStack compares top-level values with the previous successful build and adds only subscribers of changed keys to the rebuild set. ### What triggers what +The tables below describe the unchanged defaults when no global-data state is retained. +When retaining state, watched module edits—including JavaScript/TypeScript pages, page-variable modules, helpers, browser entries, and manifest or other settings—reset the index and trigger a **full rebuild**: rediscover inputs, restart esbuild, and rebuild all pages and templates. +Direct Markdown/HTML source edits can still use incremental deltas. +Input changes can request a global-data reset without retained state, but do not change these default output rebuild scopes. + | Change | Rebuild scope | |---|---| | Existing `page.ts`, `page.html`, `page.md`, or adjacent `page.vars.ts` | That page, plus subscribers of any changed global-data keys | @@ -280,6 +285,7 @@ When a full page/template rebuild or targeted generated-pages rebuild no longer ### Dependency tracking DOMStack uses [`@11ty/dependency-tree-typescript`](https://github.com/11ty/dependency-tree-typescript) to statically analyze ESM imports. +The page watcher observes only processed file extensions in the watched source tree; imported JSON changes do not trigger page rebuilds. It maintains maps for: - Layout dependencies, source-backed pages using each layout, and generated-page owner layout membership @@ -294,8 +300,8 @@ Dependency analysis is best-effort. When DOMStack cannot safely determine a targeted scope, it falls back to a broader rebuild or skips an unrelated changed module. esbuild tracks browser-entry dependencies independently. -Changing a module imported only by `client.ts` rebundles that entry without rendering page HTML. -When a module has both browser and server-side consumers, the planner unions the server-side consumers rather than skipping the page phase. +Without retained global-data state, changing a module imported only by `client.ts` rebundles that entry without rendering page HTML. +When retaining state, watched module edits trigger the full rebuild described above. ### Stable entry filenames @@ -322,7 +328,7 @@ Page HTML points to stable entry files during watch mode. esbuild can update an ### Manifest behavior Watch mode builds and rebundles the site service worker, but it does not finalize, return, or write the [DOMStack manifest](../../docs/workers/#domstack-manifest). -Changes to `domstack-manifest.settings.ts` therefore do not trigger a watch rebuild. +Changes to `domstack-manifest.settings.ts` do not trigger a watch rebuild unless global-data state is retained. Use `domstack --serve` when testing manifest-driven cache behavior. It runs a one-shot build and serves the result without watch-mode filenames or live-reload HTML injection. diff --git a/index.js b/index.js index 0c4ef9a5..962776bc 100644 --- a/index.js +++ b/index.js @@ -105,15 +105,10 @@ export class DomStack { #pagesFileDepMap = new Map() /** @type {Set} Imported inputs of global.data, including its entry file. */ #globalDataDepPaths = new Set() - /** @type {Set} */ - #globalVarsDepPaths = new Set() - /** @type {Set} */ - #markdownDepPaths = new Set() - #dependencyAnalysisFailed = false + /** @type {Set} absolute filepaths of esbuild entry points */ #esbuildEntryPoints = new Set() - /** @type {Set} Known browser-only helpers can skip the page phase. */ - #esbuildDepPaths = new Set() + /** @type {Map>} source page or *.pages.* filepath → owned absolute output paths */ #pageOutputMap = new Map() /** @type {PageOutputCache} Successful writes, including those before an iterator failure. */ @@ -343,9 +338,7 @@ export class DomStack { const anymatch = (/** @type {string} */name) => ig.ignores(relname(this.#src, name)) const watcher = chokidar.watch(this.#src, { - // Observe non-page extensions too (for example statically imported JSON). - // Route only processed files and known dependencies after maps are ready. - ignored: filePath => anymatch(filePath), + ignored: (filePath, stats) => anymatch(filePath) || Boolean(stats?.isFile() && !isProcessedFile(filePath)), persistent: true, ignoreInitial: true, // Increase the atomic write window so editors that do slow atomic saves @@ -465,46 +458,22 @@ export class DomStack { pagesFileDepMap: this.#pagesFileDepMap, pagesFileLayoutMap: this.#pagesFileLayoutMap, globalDataDepPaths: this.#globalDataDepPaths, - globalVarsDepPaths: this.#globalVarsDepPaths, - markdownDepPaths: this.#markdownDepPaths, - dependencyAnalysisFailed: this.#dependencyAnalysisFailed, + hasGlobalDataState: this.#watchSession?.globalDataBaseline?.state !== undefined, pageBuildFailed: this.#pageBuildFailed, esbuildEntryPoints: this.#esbuildEntryPoints, - esbuildDepPaths: this.#esbuildDepPaths, } } - /** @param {WatchEvent[]} events */ - #filterWatchEvents (events) { - // Unknown inputs may be needed to recover after a failed build or analysis. - if (this.#pageBuildFailed || this.#dependencyAnalysisFailed) return events - - const dependencies = [ - this.#globalDataDepPaths, - this.#globalVarsDepPaths, - this.#markdownDepPaths, - this.#layoutDepMap, - this.#pageDepMap, - this.#templateDepMap, - this.#pagesFileDepMap, - this.#esbuildDepPaths, - ] - return events.filter(({ filepath }) => - isProcessedFile(filepath) || dependencies.some(paths => paths.has(filepath)) - ) - } - /** @param {WatchEvent[]} events */ async #handleWatchBatch (events) { const snapshot = this.#watchSnapshot() if (!snapshot) return - events = this.#filterWatchEvents(events) const event = events[0] if (!event) return const { plan, inputChanges } = planWatchBatch(snapshot, events) - // Keep the existing precise bundle-membership path for a single event. - // Mixed batches use the conservative rediscovery plan instead. - const singlePlan = events.length === 1 && event.type !== 'change' && event.convention?.bundleScope + // Preserve single-event bundle routing unless retained state needs a reset. + const singlePlan = !(snapshot.hasGlobalDataState && inputChanges.resetReason) && + events.length === 1 && event.type !== 'change' && event.convention?.bundleScope ? planWatchEvent(snapshot, event) : null await this.#executeWatchPlan( @@ -739,22 +708,15 @@ export class DomStack { const pageDepMap = /** @type {Map>} */ (new Map()) const templateDepMap = /** @type {Map>} */ (new Map()) const pagesFileDepMap = /** @type {Map>} */ (new Map()) - let dependencyAnalysisFailed = false - /** @param {string | undefined} filepath */ - const rootDependencies = async filepath => { - const paths = new Set(/** @type {string[]} */ ([])) - if (!filepath) return paths - paths.add(resolve(filepath)) + const globalDataDepPaths = new Set() + if (siteData.globalData) { + globalDataDepPaths.add(siteData.globalData.filepath) try { - for (const dep of await find(filepath)) paths.add(resolve(dep)) + for (const dep of await find(siteData.globalData.filepath)) globalDataDepPaths.add(resolve(dep)) } catch { - dependencyAnalysisFailed = true + // Static import analysis is best-effort, as for page and layout helpers. } - return paths } - const globalDataDepPaths = await rootDependencies(siteData.globalData?.filepath) - const globalVarsDepPaths = await rootDependencies(siteData.globalVars?.filepath) - const markdownDepPaths = await rootDependencies(siteData.markdownItSettings?.filepath) // layoutFileMap: layout filepath → layoutName for (const layout of Object.values(siteData.layouts)) { @@ -771,7 +733,7 @@ export class DomStack { layoutDepMap.get(absPath)?.add(layout.layoutName) } } catch { - dependencyAnalysisFailed = true + // dep analysis is best-effort } } @@ -791,7 +753,7 @@ export class DomStack { // pageDepMap: dep filepath → Set for (const pageInfo of siteData.pages) { - const filesToTrack = /\.[cm]?[jt]sx?$/.test(pageInfo.pageFile.filepath) ? [pageInfo.pageFile.filepath] : [] + const filesToTrack = [pageInfo.pageFile.filepath] if (pageInfo.pageVars) filesToTrack.push(pageInfo.pageVars.filepath) for (const file of filesToTrack) { try { @@ -802,7 +764,7 @@ export class DomStack { pageDepMap.get(absPath)?.add(pageInfo) } } catch { - dependencyAnalysisFailed = true + // best-effort } } } @@ -817,7 +779,7 @@ export class DomStack { templateDepMap.get(absPath)?.add(templateInfo) } } catch { - dependencyAnalysisFailed = true + // best-effort } } @@ -831,7 +793,6 @@ export class DomStack { pagesFileDepMap.get(absPath)?.add(pagesFileInfo) } } catch (err) { - dependencyAnalysisFailed = true const message = err instanceof Error ? err.message : String(err) this.#logger.debug(`Could not analyze dependencies for pages file "${pagesFileInfo.pagesFile.relname}": ${message}`) } @@ -848,16 +809,6 @@ export class DomStack { for (const asset of layoutBundleAssets(layout)) esbuildEntryPoints.add(resolve(asset.filepath)) } - const esbuildDepPaths = new Set(/** @type {string[]} */ ([])) - for (const filepath of esbuildEntryPoints) { - if (!/\.[cm]?[jt]sx?$/.test(filepath)) continue - try { - for (const dep of await find(filepath)) esbuildDepPaths.add(resolve(dep)) - } catch { - // Unknown browser helpers still take the conservative reset path. - } - } - this.#layoutDepMap = layoutDepMap this.#layoutPageMap = layoutPageMap this.#pageFileMap = pageFileMap @@ -866,11 +817,7 @@ export class DomStack { this.#templateDepMap = templateDepMap this.#pagesFileDepMap = pagesFileDepMap this.#globalDataDepPaths = globalDataDepPaths - this.#globalVarsDepPaths = globalVarsDepPaths - this.#markdownDepPaths = markdownDepPaths - this.#dependencyAnalysisFailed = dependencyAnalysisFailed this.#esbuildEntryPoints = esbuildEntryPoints - this.#esbuildDepPaths = esbuildDepPaths } /** diff --git a/lib/watch-plan.js b/lib/watch-plan.js index 42bed124..d41efeda 100644 --- a/lib/watch-plan.js +++ b/lib/watch-plan.js @@ -12,12 +12,9 @@ * @property {ReadonlyMap>} pagesFileDepMap * @property {ReadonlyMap>} pagesFileLayoutMap * @property {ReadonlySet} globalDataDepPaths - * @property {ReadonlySet} [globalVarsDepPaths] - * @property {ReadonlySet} [markdownDepPaths] - * @property {boolean} [dependencyAnalysisFailed] + * @property {boolean} [hasGlobalDataState] The watch session retains producer state. * @property {boolean} pageBuildFailed * @property {ReadonlySet} esbuildEntryPoints - * @property {ReadonlySet} [esbuildDepPaths] Known imported browser inputs; server roles still take precedence. * @typedef {ReturnType} WatchEvent * @typedef {object} PagePlan * @property {'pages'} kind @@ -61,7 +58,7 @@ export function planWatchEvent (state, event) { ? { kind: 'restart', message: `"${name}" ${type}, restarting esbuild...` } : { kind: 'full', message: `"${name}" ${type}, triggering full rebuild...` } } - if (convention?.change === 'full' || state.globalVarsDepPaths?.has(filepath)) return { kind: 'full', message: `"${name}" changed, triggering full rebuild...` } + if (convention?.change === 'full') return { kind: 'full', message: `"${name}" changed, triggering full rebuild...` } if (convention?.change === 'manifest' && !hasWatchRole(state, filepath)) { return { kind: 'skip', message: `"${name}" changed but domstack manifests are disabled in watch mode, skipping.` } } @@ -74,7 +71,7 @@ export function planWatchEvent (state, event) { if (directLayout) layouts.add(directLayout) const affected = layoutConsumers(state, layouts) const pages = new Set(state.pageDepMap.get(filepath)) - const markdownChanged = convention?.change === 'markdown' || state.markdownDepPaths?.has(filepath) + const markdownChanged = convention?.change === 'markdown' if (markdownChanged) { for (const page of siteData.pages) { if (page.type === 'md') pages.add(page) @@ -107,7 +104,7 @@ export function planWatchEvent (state, event) { if (globalDataChanged) plan.message = `"${name}" changed, rebuilding data subscribers...` return plan } - if (state.esbuildEntryPoints.has(filepath) || state.esbuildDepPaths?.has(filepath)) { + if (state.esbuildEntryPoints.has(filepath)) { return { kind: 'skip', message: `"${name}" changed, esbuild will handle rebundling.` } } return { kind: 'skip', message: `"${name}" changed but did not match any rebuild rule, skipping.` } @@ -122,18 +119,21 @@ export function planWatchEvent (state, event) { * lines in event order so logging retains every selected consumer and reason. * * A full output plan requires rediscovery, but does not itself reset inputs. - * Structural source events upsert the change-plan's source consumers plus the - * event path (including removed paths). Other structural events upsert all + * Structural source events upsert mapped source consumers plus the event path + * (including removed paths). Other structural events upsert all * current source pages. The worker must reconcile membership after rediscovery; * these candidates are not instructions to read deleted files. Templates and * generated-page owners are output selections, not source-input rows. * - * A reset supersedes the delta. Reasons are page-build-failed, then - * dependency-analysis-failed, then the first applicable event reason, using - * this per-event priority: + * A reset supersedes the delta. Reasons are page-build-failed, then the first + * applicable event reason, using this per-event priority: * unreliable-event, global-data-changed, global-vars-changed, * markdown-settings-changed, global-config-changed, or unknown-event. - * Unknown changes reset; unknown adds/removes can use conservative deltas. + * Only non-module source texts (.md/.html) can use deltas. + * Other inputs may also configure global vars or Markdown rendering. + * Resets force full output rebuilds only when producer state is retained; + * without retained state, legacy event routing still determines output scope. + * Failed builds retain their recovery plan; the executor handles rediscovery. * Empty batches skip without requesting recovery. Inputs are never mutated. * * @param {WatchSnapshot} state @@ -147,9 +147,7 @@ export function planWatchBatch (state, events) { const inputChanges = { upsertedPaths: [], events: [...events] } if (!events.length) return { plan, inputChanges } - let resetReason = state.pageBuildFailed - ? 'page-build-failed' - : state.dependencyAnalysisFailed ? 'dependency-analysis-failed' : undefined + let resetReason = state.pageBuildFailed ? 'page-build-failed' : undefined const sourcePaths = new Set(/** @type {string[]} */ ([])) for (const page of state.siteData.pages) { if (!page.generated) sourcePaths.add(page.pageFile.filepath) @@ -165,10 +163,10 @@ export function planWatchBatch (state, events) { // Only source membership changes need mapped consumers; other structural // events already invalidate every current source input. const changePlan = sourceEvent ? planWatchEvent(state, { ...event, type: 'change' }) : eventPlan - const reason = inputResetReason(state, event, changePlan) + const reason = inputResetReason(state, event) resetReason ??= reason plan = unionWatchPlans(plan, eventPlan) - if (reason === 'unknown-event' || reason === 'unreliable-event') { + if (reason === 'unreliable-event') { plan = { kind: 'full', message: `"${event.name}" cannot be routed reliably, triggering full rebuild...` } } @@ -187,8 +185,10 @@ export function planWatchBatch (state, events) { } if (structural && sourceEvent) upsertedPaths.add(event.filepath) } - if (state.pageBuildFailed || state.dependencyAnalysisFailed) { + if (state.pageBuildFailed) { plan = unionWatchPlans(plan, allPages()) + } else if (state.hasGlobalDataState && resetReason) { + plan = unionWatchPlans(plan, { kind: 'full', message: 'Producer inputs require a reset, triggering full rebuild...' }) } if (resetReason) inputChanges.resetReason = resetReason inputChanges.upsertedPaths = [...upsertedPaths] @@ -198,24 +198,25 @@ export function planWatchBatch (state, events) { /** * @param {WatchSnapshot} state * @param {WatchEvent} event - * @param {WatchPlan} changePlan */ -function inputResetReason (state, { type, filepath, name, convention }, changePlan) { +function inputResetReason (state, { type, filepath, name, convention }) { if (type !== 'change' && type !== 'added' && type !== 'removed') return 'unreliable-event' if (convention?.change === 'pages' || state.globalDataDepPaths.has(filepath)) return 'global-data-changed' - if (fileConventions.globalVars.names.includes(name) || state.globalVarsDepPaths?.has(filepath)) return 'global-vars-changed' - if (convention?.change === 'markdown' || state.markdownDepPaths?.has(filepath)) return 'markdown-settings-changed' + if (fileConventions.globalVars.names.includes(name)) return 'global-vars-changed' + if (convention?.change === 'markdown') return 'markdown-settings-changed' if (convention?.change === 'full') return 'global-config-changed' - if (type === 'change' && changePlan.kind === 'skip' && - convention?.change !== 'manifest' && !state.esbuildEntryPoints.has(filepath) && - !state.esbuildDepPaths?.has(filepath) && !hasWatchRole(state, filepath)) { - return 'unknown-event' - } + const sourcePage = state.pageFileMap.get(filepath) + const sourceText = name.endsWith('.md') || name.endsWith('.html') + if (sourceText && ( + (sourcePage && !sourcePage.generated && sourcePage.pageFile.filepath === filepath) || + (type !== 'change' && (name.endsWith('.md') || convention?.change === 'page')) + )) return + return 'unknown-event' } /** @param {WatchSnapshot} state @param {string} filepath */ function hasWatchRole (state, filepath) { - return state.globalDataDepPaths.has(filepath) || state.globalVarsDepPaths?.has(filepath) || state.markdownDepPaths?.has(filepath) || + return state.globalDataDepPaths.has(filepath) || state.pageFileMap.has(filepath) || state.layoutFileMap.has(filepath) || state.layoutDepMap.has(filepath) || state.pageDepMap.has(filepath) || state.templateDepMap.has(filepath) || state.pagesFileDepMap.has(filepath) || state.siteData.templates.some(template => template.templateFile.filepath === filepath) || diff --git a/lib/watch-plan.test.js b/lib/watch-plan.test.js index 27129a99..b9c285ca 100644 --- a/lib/watch-plan.test.js +++ b/lib/watch-plan.test.js @@ -19,7 +19,7 @@ function page (path, type = 'md') { return { pageFile: file(join(path, `page.${type}`)), type, path, url: `/${path}/`, outputName: 'index.html', outputRelname: join(path, 'index.html'), draft: false } } -function fixture () { +function fixture (hasGlobalDataState = true) { const home = page('') const other = page('other', 'js') const template = { templateFile: file('feed.template.js'), path: '', outputName: 'feed.xml' } @@ -29,19 +29,16 @@ function fixture () { siteData: { pages: [home, other], templates: [template], pagesFiles: [owner], layouts: { root } }, layoutDepMap: new Map([['/site/layout-helper.js', new Set(['root'])]]), layoutPageMap: new Map([['root', new Set([home])]]), - pageFileMap: new Map([[home.pageFile.filepath, home], ['/site/page.vars.js', home]]), + pageFileMap: new Map([[home.pageFile.filepath, home], [other.pageFile.filepath, other], ['/site/page.vars.js', home]]), layoutFileMap: new Map([[root.filepath, 'root']]), pageDepMap: new Map([['/site/page-helper.js', new Set([other])]]), templateDepMap: new Map([['/site/template-helper.js', new Set([template])]]), pagesFileDepMap: new Map([['/site/archive-helper.js', new Set([owner])]]), pagesFileLayoutMap: new Map([[owner.pagesFile.filepath, new Set(['root'])]]), globalDataDepPaths: new Set(['/site/global.data.js']), - globalVarsDepPaths: new Set(), - markdownDepPaths: new Set(), - dependencyAnalysisFailed: false, + hasGlobalDataState, pageBuildFailed: false, esbuildEntryPoints: new Set(['/site/client.jsx', '/site/root.layout.css']), - esbuildDepPaths: new Set(), }) return { state, home, other, template, owner } } @@ -53,97 +50,176 @@ function scope (plan) { return [plan.pageFilterPaths, plan.templateFilterPaths, plan.pagesFileFilterPaths] } -test('settings and untracked changes produce inspectable full, page, and skip plans', () => { +test('source text retains targeted scopes while modules and configuration roots reset retained state', () => { const { state, home } = fixture() - for (const name of ['global.vars.js', 'esbuild.settings.js']) { - assert.equal(planWatchEvent(state, classifyWatchEvent('change', `/site/${name}`)).kind, 'full') + const source = planWatchBatch(state, [classifyWatchEvent('change', home.pageFile.filepath)]) + assert.deepEqual(scope(source.plan), [[home.pageFile.filepath], [], []]) + assert.equal(source.inputChanges.resetReason, undefined) + for (const name of ['page.vars.js', 'other/page.js', 'new/page.mjs']) { + for (const type of /** @type {const} */ (['change', 'added', 'removed'])) { + const batch = planWatchBatch(state, [classifyWatchEvent(type, `/site/${name}`)]) + assert.equal(batch.plan.kind, 'full') + assert.equal(batch.inputChanges.resetReason, 'unknown-event') + } } + state.globalDataDepPaths.clear() assert.deepEqual(scope(planWatchEvent(state, classifyWatchEvent('change', '/site/global.data.js'))), [[], [], []]) - assert.deepEqual(scope(planWatchEvent(state, classifyWatchEvent('change', '/site/markdown-it.settings.js'))), [[home.pageFile.filepath], [], []]) - for (const [name, reason] of [['domstack-manifest.settings.js', 'disabled'], ['client.jsx', 'esbuild'], ['unused.js', 'did not match']]) { - const plan = planWatchEvent(state, classifyWatchEvent('change', `/site/${name}`)) - assert.equal(plan.kind, 'skip') - assert.ok(plan.message?.includes(reason ?? '')) + for (const [name, reason] of [ + ['global.data.js', 'global-data-changed'], + ['global.vars.js', 'global-vars-changed'], + ['markdown-it.settings.js', 'markdown-settings-changed'], + ['esbuild.settings.js', 'global-config-changed'], + ]) { + for (const type of /** @type {const} */ (['change', 'added', 'removed'])) { + const batch = planWatchBatch(state, [classifyWatchEvent(type, `/site/${name}`)]) + assert.equal(batch.inputChanges.resetReason, reason) + assert.equal(batch.plan.kind, 'full') + } } + const skipped = planWatchEvent(state, classifyWatchEvent('change', '/site/domstack-manifest.settings.js')) + assert.equal(skipped.kind, 'skip') + assert.ok(skipped.message?.includes('disabled')) }) -test('layout maps target source pages and generated-page owners', () => { - const { state, home, owner } = fixture() - for (const name of ['root.layout.js', 'layout-helper.js']) { - const plan = planWatchEvent(state, classifyWatchEvent('change', `/site/${name}`)) - assert.deepEqual(scope(plan), [[home.pageFile.filepath], [], [owner.pagesFile.filepath]]) +test('helpers, browser entries, layouts, templates and factories reset and rebuild fully even with known consumers', () => { + const { state } = fixture() + for (const name of ['page-helper.js', 'layout-helper.js', 'template-helper.js', 'archive-helper.js', 'unknown.js', 'client.jsx', 'root.layout.css', 'root.layout.js', 'feed.template.js', 'archive.pages.js', 'domstack-manifest.settings.js']) { + const event = classifyWatchEvent('change', `/site/${name}`) + const batch = planWatchBatch(state, [event]) + assert.equal(batch.plan.kind, 'full', name) + assert.equal(batch.inputChanges.resetReason, 'unknown-event', name) } state.layoutPageMap.clear() - assert.deepEqual(scope(planWatchEvent(state, classifyWatchEvent('change', '/site/root.layout.js'))), [[], [], [owner.pagesFile.filepath]]) state.pagesFileLayoutMap.clear() - const unused = planWatchEvent(state, classifyWatchEvent('change', '/site/root.layout.js')) - assert.equal(unused.kind, 'skip') - assert.ok(unused.message?.includes('did not match any rebuild rule')) + assert.equal(planWatchBatch(state, [classifyWatchEvent('change', '/site/root.layout.js')]).plan.kind, 'full', 'an unused layout can still have an untracked configuration role') +}) + +test('without retained state legacy helper scopes and browser skips survive input reset requests', () => { + for (const omitFlag of [false, true]) { + const { state: snapshot, home, other, template, owner } = fixture(false) + const state = /** @type {WatchSnapshot} */ ({ ...snapshot }) + if (omitFlag) delete state.hasGlobalDataState + for (const [name, expected, reason] of /** @type {const} */ ([ + ['page-helper.js', [[other.pageFile.filepath], [], []], 'unknown-event'], + ['page.vars.js', [[home.pageFile.filepath], [], []], 'unknown-event'], + ['other/page.js', [[other.pageFile.filepath], [], []], 'unknown-event'], + ['layout-helper.js', [[home.pageFile.filepath], [], [owner.pagesFile.filepath]], 'unknown-event'], + ['root.layout.js', [[home.pageFile.filepath], [], [owner.pagesFile.filepath]], 'unknown-event'], + ['template-helper.js', [[], [template.templateFile.filepath], []], 'unknown-event'], + ['archive-helper.js', [[], [], [owner.pagesFile.filepath]], 'unknown-event'], + ['feed.template.js', [[], [template.templateFile.filepath], []], 'unknown-event'], + ['archive.pages.js', [[], [], [owner.pagesFile.filepath]], 'unknown-event'], + ['global.data.js', [[], [], []], 'global-data-changed'], + ['markdown-it.settings.js', [[home.pageFile.filepath], [], []], 'markdown-settings-changed'], + ])) { + const event = classifyWatchEvent('change', `/site/${name}`) + const batch = planWatchBatch(state, [event]) + assert.deepEqual(scope(batch.plan), expected, name) + assert.deepEqual(batch.plan, planWatchEvent(state, event), name) + assert.equal(batch.inputChanges.resetReason, reason, name) + } + for (const name of ['client.jsx', 'root.layout.css', 'unknown.js', 'domstack-manifest.settings.js']) { + const event = classifyWatchEvent('change', `/site/${name}`) + const batch = planWatchBatch(state, [event]) + assert.equal(batch.plan.kind, 'skip', name) + assert.deepEqual(batch.plan, planWatchEvent(state, event), name) + assert.deepEqual(batch.inputChanges, { resetReason: 'unknown-event', upsertedPaths: [], events: [event] }) + } + snapshot.globalDataDepPaths.add('/site/client.jsx') + const imported = planWatchBatch(state, [classifyWatchEvent('change', '/site/client.jsx')]) + assert.deepEqual(scope(imported.plan), [[], [], []], 'producer imports take precedence over browser skips') + assert.equal(imported.inputChanges.resetReason, 'global-data-changed') + } }) -test('page, template, and owner inputs retain their targeted scopes', () => { - const { state, home, other, owner, template } = fixture() - const cases = [ - ['page.md', [home.pageFile.filepath], [], []], - ['page.vars.js', [home.pageFile.filepath], [], []], - ['page-helper.js', [other.pageFile.filepath], [], []], - ['archive.pages.js', [], [], [owner.pagesFile.filepath]], - ['archive-helper.js', [], [], [owner.pagesFile.filepath]], - ['feed.template.js', [], [template.templateFile.filepath], []], - ['template-helper.js', [], [template.templateFile.filepath], []], - ] - for (const [name, ...expected] of cases) { - assert.deepEqual(scope(planWatchEvent(state, classifyWatchEvent('change', `/site/${name}`))), expected) - } +test('direct Markdown settings still union every known consumer before browser skips', () => { + const { state, home, other, template, owner } = fixture(false) + const filepath = '/site/markdown-it.settings.js' + state.pageDepMap.set(filepath, new Set([other])) + state.layoutDepMap.set(filepath, new Set(['root'])) + state.templateDepMap.set(filepath, new Set([template])) + state.pagesFileDepMap.set(filepath, new Set([owner])) + state.esbuildEntryPoints.add(filepath) + const event = classifyWatchEvent('change', filepath) + assert.deepEqual(scope(planWatchEvent(state, event)), [[other.pageFile.filepath, home.pageFile.filepath], [template.templateFile.filepath], [owner.pagesFile.filepath]]) + const batch = planWatchBatch(state, [event]) + assert.deepEqual(batch.plan, planWatchEvent(state, event)) + assert.equal(batch.inputChanges.resetReason, 'markdown-settings-changed') }) -test('planning unions every shared role, deduplicates consumers, and leaves the snapshot untouched', () => { - const { state, home, other, template, owner } = fixture() - state.layoutDepMap.set('/site/shared.js', new Set(['root', 'another'])) - state.layoutPageMap.set('another', new Set([home])) - state.pageDepMap.set('/site/shared.js', new Set(state.siteData.pages)) - state.templateDepMap.set('/site/shared.js', new Set([template])) - state.pagesFileDepMap.set('/site/shared.js', new Set([owner])) - state.globalDataDepPaths.add('/site/shared.js') - state.esbuildEntryPoints.add('/site/shared.js') +test('shared helpers cannot hide configuration roles behind precise server or browser consumers', () => { + const { state, template, owner } = fixture() + const filepath = '/site/shared.js' + state.layoutDepMap.set(filepath, new Set(['root'])) + state.pageDepMap.set(filepath, new Set(state.siteData.pages)) + state.templateDepMap.set(filepath, new Set([template])) + state.pagesFileDepMap.set(filepath, new Set([owner])) + state.esbuildEntryPoints.add(filepath) const before = structuredClone(state) - const event = classifyWatchEvent('change', '/site/shared.js') - const first = planWatchEvent(state, event) - assert.deepEqual(scope(first), [[home.pageFile.filepath, other.pageFile.filepath], [template.templateFile.filepath], [owner.pagesFile.filepath]]) - assert.ok(first.message?.includes('rebuilding data subscribers')) - assert.deepEqual(planWatchEvent(state, event), first) + const event = classifyWatchEvent('change', filepath) + const batch = planWatchBatch(state, [event]) + assert.equal(batch.plan.kind, 'full') + assert.equal(batch.inputChanges.resetReason, 'unknown-event') + assert.deepEqual(planWatchBatch(state, [event]), batch) assert.deepEqual(state, before) }) -test('global-data-only imports trigger subscriber builds even when they are browser entries', () => { - for (const name of ['data-helper.js', 'client.jsx']) { - const { state } = fixture() - const filepath = `/site/${name}` - state.globalDataDepPaths.add(filepath) - const plan = planWatchEvent(state, classifyWatchEvent('change', filepath)) - assert.deepEqual(scope(plan), [[], [], []], 'recompute data without directly selecting unrelated consumers') - assert.ok(plan.message?.includes('rebuilding data subscribers')) +test('global-data imports reset and rebuild fully on every event even with shared roles', () => { + for (const type of /** @type {const} */ (['change', 'added', 'removed'])) { + for (const name of ['data-helper.js', 'client.jsx', 'page.md', 'domstack-manifest.settings.js']) { + const { state, other } = fixture() + const filepath = `/site/${name}` + state.globalDataDepPaths.add(filepath) + state.pageDepMap.set(filepath, new Set([other])) + const batch = planWatchBatch(state, [classifyWatchEvent(type, filepath)]) + assert.equal(batch.plan.kind, 'full', `${type} ${name}`) + assert.equal(batch.inputChanges.resetReason, 'global-data-changed') + } } }) -test('a directly selected layout also rebuilds layouts that import it', () => { - const { state, home, other, owner } = fixture() - state.layoutDepMap.set('/site/root.layout.js', new Set(['child'])) - state.layoutPageMap.set('child', new Set([other])) - const plan = planWatchEvent(state, classifyWatchEvent('change', '/site/root.layout.js')) - assert.deepEqual(scope(plan), [[other.pageFile.filepath, home.pageFile.filepath], [], [owner.pagesFile.filepath]]) +test('manifest settings imported by server consumers do not hide those roles', () => { + const { state, other } = fixture() + const filepath = '/site/domstack-manifest.settings.js' + state.pageDepMap.set(filepath, new Set([other])) + const batch = planWatchBatch(state, [classifyWatchEvent('change', filepath)]) + assert.equal(batch.plan.kind, 'full') + assert.equal(batch.inputChanges.resetReason, 'unknown-event') +}) + +test('direct source text selections union mapped consumers without mutating the snapshot', () => { + const { state, home, other, template, owner } = fixture() + const filepath = home.pageFile.filepath + state.layoutDepMap.set(filepath, new Set(['root', 'another'])) + state.layoutPageMap.set('another', new Set([home])) + state.pageDepMap.set(filepath, new Set([other])) + state.templateDepMap.set(filepath, new Set([template])) + state.pagesFileDepMap.set(filepath, new Set([owner])) + const before = structuredClone(state) + const batch = planWatchBatch(state, [classifyWatchEvent('change', filepath)]) + assert.deepEqual(scope(batch.plan), [[other.pageFile.filepath, home.pageFile.filepath], [template.templateFile.filepath], [owner.pagesFile.filepath]]) + assert.equal(batch.inputChanges.resetReason, undefined) + assert.deepEqual(state, before) }) -test('failed page builds retry fully before incomplete maps or browser entries can skip the work', () => { +test('failed builds preserve recovery plans and override event reset reasons', () => { const { state } = fixture() const failed = { ...state, pageBuildFailed: true } - for (const name of ['unused.js', 'client.jsx', 'root.layout.js', 'global.data.js', 'markdown-it.settings.js']) { - const plan = planWatchEvent(failed, classifyWatchEvent('change', `/site/${name}`)) - assert.deepEqual(scope(plan), [null, null, null]) - assert.ok(plan.message?.includes('retrying all pages after the previous build failure')) + for (const name of ['page.md', 'page.vars.js', 'global.data.js', 'unused.js', 'client.jsx', 'root.layout.js', 'markdown-it.settings.js']) { + const event = classifyWatchEvent('change', `/site/${name}`) + const batch = planWatchBatch(failed, [event]) + assert.deepEqual(scope(batch.plan), [null, null, null]) + assert.ok(batch.plan.message?.includes('retrying all pages after the previous build failure')) + assert.equal(batch.inputChanges.resetReason, 'page-build-failed') + } + for (const name of ['global.vars.js', 'esbuild.settings.js']) { + const batch = planWatchBatch(failed, [classifyWatchEvent('change', `/site/${name}`)]) + assert.equal(batch.plan.kind, 'full') + assert.equal(batch.inputChanges.resetReason, 'page-build-failed') } - assert.equal(planWatchEvent(failed, classifyWatchEvent('change', '/site/global.vars.js')).kind, 'full') - assert.equal(planWatchEvent(failed, classifyWatchEvent('change', '/site/domstack-manifest.settings.js')).kind, 'skip') + const skipped = planWatchBatch(failed, [classifyWatchEvent('change', '/site/domstack-manifest.settings.js')]) + assert.deepEqual(scope(skipped.plan), [null, null, null], 'a nonempty recovery batch cannot skip') + assert.equal(skipped.inputChanges.resetReason, 'page-build-failed') }) test('structural events distinguish esbuild entries from full rediscovery', () => { @@ -203,87 +279,31 @@ test('classification resolves paths without changing the public event shape', () assert.equal(classifyWatchEvent('change', '/site/nested/../page.md').filepath, '/site/page.md') }) -test('snapshots without optional dependency analysis fields remain compatible', () => { - const { state, home } = fixture() - const { globalVarsDepPaths, markdownDepPaths, dependencyAnalysisFailed, esbuildDepPaths, ...legacy } = state - assert.deepEqual(scope(planWatchEvent(legacy, classifyWatchEvent('change', '/site/page.md'))), [[home.pageFile.filepath], [], []]) - assert.equal(planWatchBatch(legacy, [classifyWatchEvent('change', '/site/page.md')]).inputChanges.resetReason, undefined) - assert.equal(planWatchBatch(legacy, [classifyWatchEvent('change', '/site/unknown.js')]).inputChanges.resetReason, 'unknown-event') -}) - -test('markdown roots and imports union server roles before browser or manifest skips', () => { - for (const name of ['markdown-it.settings.js', 'markdown-helper.js', 'client.jsx', 'domstack-manifest.settings.js']) { - const { state, home, other, template, owner } = fixture() - const filepath = `/site/${name}` - if (name !== 'markdown-it.settings.js') state.markdownDepPaths.add(filepath) - state.pageDepMap.set(filepath, new Set([other])) - state.layoutDepMap.set(filepath, new Set(['root'])) - state.templateDepMap.set(filepath, new Set([template])) - state.pagesFileDepMap.set(filepath, new Set([owner])) - state.esbuildEntryPoints.add(filepath) - const event = classifyWatchEvent('change', filepath) - assert.deepEqual(scope(planWatchEvent(state, event)), [[other.pageFile.filepath, home.pageFile.filepath], [template.templateFile.filepath], [owner.pagesFile.filepath]], name) - const batch = planWatchBatch(state, [event]) - assert.deepEqual(scope(batch.plan), scope(planWatchEvent(state, event))) - assert.equal(batch.inputChanges.resetReason, 'markdown-settings-changed') - } -}) - -test('global vars imports dominate every shared role and global data roots need no map entry', () => { - for (const name of ['vars-helper.js', 'client.jsx', 'markdown-it.settings.js', 'domstack-manifest.settings.js']) { - const { state, other } = fixture() - const filepath = `/site/${name}` - state.globalVarsDepPaths.add(filepath) - state.markdownDepPaths.add(filepath) - state.pageDepMap.set(filepath, new Set([other])) - const event = classifyWatchEvent('change', filepath) - assert.equal(planWatchEvent(state, event).kind, 'full', name) - assert.equal(planWatchBatch(state, [event]).inputChanges.resetReason, 'global-vars-changed', name) - } - const { state } = fixture() - state.globalDataDepPaths.clear() - assert.deepEqual(scope(planWatchEvent(state, classifyWatchEvent('change', '/site/global.data.js'))), [[], [], []]) - state.siteData.pages = [] - assert.deepEqual(scope(planWatchEvent(state, classifyWatchEvent('change', '/site/markdown-it.settings.js'))), [[], [], []]) -}) - -test('manifest settings imported by server consumers do not hide those roles', () => { - const { state, other } = fixture() - const filepath = '/site/domstack-manifest.settings.js' - state.pageDepMap.set(filepath, new Set([other])) - state.globalDataDepPaths.add(filepath) - const event = classifyWatchEvent('change', filepath) - assert.deepEqual(scope(planWatchEvent(state, event)), [[other.pageFile.filepath], [], []]) - assert.equal(planWatchBatch(state, [event]).inputChanges.resetReason, 'global-data-changed') -}) - test('batch unions filtered output scopes and source inputs, retaining event order and duplicates', () => { - const { state, home, other, template, owner } = fixture() - const events = ['page-helper.js', 'layout-helper.js', 'template-helper.js', 'archive-helper.js', 'page.md', 'client.jsx', 'page-helper.js'] + const { state, home, other } = fixture() + state.pageDepMap.set(home.pageFile.filepath, new Set([other])) + const events = ['page.md', 'page.md', 'page.md'] .map(name => classifyWatchEvent('change', `/site/${name}`)) const before = structuredClone({ state, events }) events.forEach(Object.freeze) Object.freeze(events) const batch = planWatchBatch(state, events) - assert.deepEqual(scope(batch.plan), [[other.pageFile.filepath, home.pageFile.filepath], [template.templateFile.filepath], [owner.pagesFile.filepath]]) + assert.deepEqual(scope(batch.plan), [[other.pageFile.filepath, home.pageFile.filepath], [], []]) assert.equal(batch.plan.kind, 'pages') if (batch.plan.kind !== 'pages') throw new Error('Expected a page plan') assert.deepEqual(batch.plan.pages, [other, home]) - assert.deepEqual(batch.plan.templates, [template]) - assert.equal(batch.plan.message, undefined, 'browser skip messages do not replace rebuild logging') - assert.deepEqual(batch.inputChanges, { - upsertedPaths: [other.pageFile.filepath, home.pageFile.filepath], - events, - }) + assert.deepEqual(batch.plan.templates, []) + assert.equal(batch.plan.message, undefined, 'source-only batches need no reset message') + assert.deepEqual(batch.inputChanges, { upsertedPaths: [other.pageFile.filepath, home.pageFile.filepath], events }) assert.notEqual(batch.inputChanges.events, events) assert.deepEqual(planWatchBatch(state, events), batch) assert.deepEqual({ state, events }, before) }) -test('single page plans preserve exact logging metadata and messages even when surrounded by skips', () => { - const { state } = fixture() - const skipped = classifyWatchEvent('change', '/site/client.jsx') - for (const name of ['page.md', 'layout-helper.js', 'feed.template.js', 'global.data.js', 'markdown-it.settings.js']) { +test('without retained state single page plans preserve metadata even when surrounded by skips', () => { + const { state } = fixture(false) + const skipped = classifyWatchEvent('change', '/site/domstack-manifest.settings.js') + for (const name of ['page.md', 'page.vars.js', 'global.data.js']) { const event = classifyWatchEvent('change', `/site/${name}`) const expected = planWatchEvent(state, event) assert.equal(expected.kind, 'pages') @@ -298,78 +318,32 @@ test('single page plans preserve exact logging metadata and messages even when s }) test('page-plan unions retain distinct messages in event order without duplicate lines', () => { - const { state, home, template } = fixture() - state.globalDataDepPaths.add('/site/data-helper.js') + const { state, home } = fixture(false) const data = classifyWatchEvent('change', '/site/global.data.js') - const helper = classifyWatchEvent('change', '/site/data-helper.js') + const otherData = classifyWatchEvent('change', '/site/nested/global.data.mjs') const pageEvent = classifyWatchEvent('change', home.pageFile.filepath) - const templateEvent = classifyWatchEvent('change', template.templateFile.filepath) - for (const [first, second] of [[data, helper], [helper, data]]) { + for (const [first, second] of [[data, otherData], [otherData, data]]) { assert.ok(first && second) - const batch = planWatchBatch(state, [first, pageEvent, second, templateEvent, first, second]) + const batch = planWatchBatch(state, [first, pageEvent, second, first, second]) assert.equal(batch.plan.kind, 'pages') if (batch.plan.kind !== 'pages') throw new Error('Expected a page plan') assert.deepEqual(batch.plan.pages, [home]) - assert.deepEqual(batch.plan.templates, [template]) assert.equal(batch.plan.message, [planWatchEvent(state, first).message, planWatchEvent(state, second).message].join('\n')) } }) -test('known browser-only dependencies skip page rebuilding in both planners without resetting inputs', () => { - const { state } = fixture() - for (const filepath of ['/site/browser-helper.js', '/site/nested/client.js']) { - state.esbuildDepPaths.add(filepath) - assert.equal(state.esbuildEntryPoints.has(filepath), false) - const event = classifyWatchEvent('change', filepath) - const single = planWatchEvent(state, event) - assert.equal(single.kind, 'skip') - assert.ok(single.message?.includes('esbuild will handle rebundling')) - const batch = planWatchBatch(state, [event]) - assert.deepEqual(batch.plan, single) - assert.deepEqual(batch.inputChanges, { upsertedPaths: [], events: [event] }) - } - const unknown = planWatchBatch(state, [classifyWatchEvent('change', '/site/unknown.js')]) - assert.equal(unknown.plan.kind, 'full') - assert.equal(unknown.inputChanges.resetReason, 'unknown-event') -}) - -test('template and generated-page owner changes do not invent source inputs', () => { - const { state, template, owner } = fixture() - const batch = planWatchBatch(state, ['feed.template.js', 'archive.pages.js', 'archive-helper.js'] - .map(name => classifyWatchEvent('change', `/site/${name}`))) - assert.deepEqual(scope(batch.plan), [[], [template.templateFile.filepath], [owner.pagesFile.filepath]]) - assert.deepEqual(batch.inputChanges.upsertedPaths, []) - assert.equal(batch.inputChanges.resetReason, undefined) -}) - -test('null filters dominate filtered scopes and failure resets override event reasons', () => { - for (const flag of /** @type {const} */ (['pageBuildFailed', 'dependencyAnalysisFailed'])) { - const { state: snapshot } = fixture() - const state = { ...snapshot, pageBuildFailed: flag === 'pageBuildFailed', dependencyAnalysisFailed: flag === 'dependencyAnalysisFailed' } - const events = ['page.md', 'feed.template.js', 'global.data.js', 'client.jsx'].map(name => classifyWatchEvent('change', `/site/${name}`)) - const batch = planWatchBatch(state, events) - assert.deepEqual(scope(batch.plan), [null, null, null]) - assert.equal(batch.inputChanges.resetReason, flag === 'pageBuildFailed' ? 'page-build-failed' : 'dependency-analysis-failed') - const skipped = planWatchBatch(state, [classifyWatchEvent('change', '/site/domstack-manifest.settings.js')]) - assert.deepEqual(scope(skipped.plan), [null, null, null], 'a nonempty recovery batch cannot skip') - state.pageBuildFailed = state.dependencyAnalysisFailed = true - assert.equal(planWatchBatch(state, events).inputChanges.resetReason, 'page-build-failed') - } -}) - -test('empty and intentionally skipped batches have no input changes', () => { - const { state } = fixture() - for (const events of [[], ['client.jsx', 'domstack-manifest.settings.js'].map(name => classifyWatchEvent('change', `/site/${name}`))]) { - const batch = planWatchBatch(state, events) - assert.equal(batch.plan.kind, 'skip') - assert.deepEqual(batch.inputChanges, { upsertedPaths: [], events }) +test('empty batches never reset or request recovery', () => { + for (const hasGlobalDataState of [true, false]) { + const { state } = fixture(hasGlobalDataState) + for (const pageBuildFailed of [true, false]) { + const batch = planWatchBatch({ ...state, pageBuildFailed }, []) + assert.equal(batch.plan.kind, 'skip') + assert.deepEqual(batch.inputChanges, { upsertedPaths: [], events: [] }) + } } - const failed = { ...state, pageBuildFailed: true, dependencyAnalysisFailed: true } - assert.deepEqual(planWatchBatch(failed, []).inputChanges, { upsertedPaths: [], events: [] }) - assert.equal(planWatchBatch(failed, []).plan.kind, 'skip') }) -test('structural source pages carry membership candidates and change-plan consumers without resetting', () => { +test('structural source pages carry membership candidates and mapped consumers without resetting', () => { for (const type of /** @type {const} */ (['added', 'removed'])) { for (const name of ['page.md', 'new/page.html', 'new/post.md']) { const { state, home, other } = fixture() @@ -386,6 +360,29 @@ test('structural source pages carry membership candidates and change-plan consum } }) +test('Markdown and HTML source edits use deltas only when mapped and not global-data dependencies', () => { + for (const type of /** @type {const} */ (['md', 'html'])) { + const { state } = fixture() + const post = { ...page('posts', type), pageFile: file(`posts/article.${type}`) } + state.siteData.pages.push(post) + state.pageFileMap.set(post.pageFile.filepath, post) + const event = classifyWatchEvent('change', post.pageFile.filepath) + const batch = planWatchBatch(state, [event]) + assert.deepEqual(scope(batch.plan), [[post.pageFile.filepath], [], []]) + assert.deepEqual(batch.inputChanges.upsertedPaths, [post.pageFile.filepath]) + assert.equal(batch.inputChanges.resetReason, undefined) + state.globalDataDepPaths.add(event.filepath) + const imported = planWatchBatch(state, [event]) + assert.equal(imported.plan.kind, 'full') + assert.equal(imported.inputChanges.resetReason, 'global-data-changed') + state.globalDataDepPaths.clear() + state.pageFileMap.delete(event.filepath) + const unmapped = planWatchBatch(state, [event]) + assert.equal(unmapped.plan.kind, 'full') + assert.equal(unmapped.inputChanges.resetReason, 'unknown-event') + } +}) + test('structural events preserve replacement and removal order for membership reconciliation', () => { const { state } = fixture() const events = [ @@ -399,7 +396,7 @@ test('structural events preserve replacement and removal order for membership re assert.deepEqual(batch.inputChanges, { upsertedPaths: ['/site/page.md', '/site/page.html', '/site/new/post.md'], events }) }) -test('structural layout, vars, unknown, template, and owner events conservatively upsert only source pages', () => { +test('structural page vars and other non-source-text inputs reset with source-only candidates', () => { for (const type of /** @type {const} */ (['added', 'removed'])) { for (const name of ['page.vars.js', 'new/page.vars.mjs', 'root.layout.js', 'new.layout.js', 'helper.js', 'feed.template.js', 'archive.pages.js']) { const { state, home, other, owner } = fixture() @@ -407,7 +404,9 @@ test('structural layout, vars, unknown, template, and owner events conservativel const event = classifyWatchEvent(type, `/site/${name}`) const batch = planWatchBatch(state, [event]) assert.equal(batch.plan.kind, 'full') - assert.deepEqual(batch.inputChanges, { upsertedPaths: [home.pageFile.filepath, other.pageFile.filepath], events: [event] }, `${type} ${name}`) + assert.deepEqual(batch.inputChanges.upsertedPaths, [home.pageFile.filepath, other.pageFile.filepath]) + assert.deepEqual(batch.inputChanges.events, [event]) + assert.equal(batch.inputChanges.resetReason, 'unknown-event') } } }) @@ -415,7 +414,7 @@ test('structural layout, vars, unknown, template, and owner events conservativel test('repeated all-source events preserve candidate order and later membership changes', () => { const { state, home, other } = fixture() const events = [ - classifyWatchEvent('change', '/site/page-helper.js'), + classifyWatchEvent('change', '/site/other/page.js'), classifyWatchEvent('added', '/site/first/post.md'), classifyWatchEvent('added', '/site/feed.template.js'), classifyWatchEvent('removed', '/site/root.layout.js'), @@ -428,13 +427,13 @@ test('repeated all-source events preserve candidate order and later membership c assert.equal(batch.plan.kind, 'full') assert.deepEqual(batch.inputChanges, { upsertedPaths: [other.pageFile.filepath, '/site/first/post.md', home.pageFile.filepath, '/site/second/post.md'], - resetReason: 'global-vars-changed', + resetReason: 'unknown-event', events, }) assert.deepEqual(state, before) }) -test('batch bundle restarts become full without resetting, including when mixed with page or full plans', () => { +test('batch bundle restarts become full resets even when mixed with source deltas', () => { for (const name of ['client.jsx', 'root.layout.css', 'global.css', 'service-worker.js']) { const { state, home, other } = fixture() const restart = classifyWatchEvent('removed', `/site/${name}`) @@ -442,65 +441,31 @@ test('batch bundle restarts become full without resetting, including when mixed for (const events of [[restart], [restart, classifyWatchEvent('change', '/site/page.md')], [classifyWatchEvent('added', '/site/new/page.md'), restart], [restart, classifyWatchEvent('added', '/site/new/page.md')]]) { const batch = planWatchBatch(state, events) assert.equal(batch.plan.kind, 'full') - assert.equal(batch.inputChanges.resetReason, undefined) + assert.equal(batch.inputChanges.resetReason, 'unknown-event') assert.ok(batch.inputChanges.upsertedPaths.includes(home.pageFile.filepath)) assert.ok(batch.inputChanges.upsertedPaths.includes(other.pageFile.filepath)) } } }) -test('global roots and imports reset on changes, additions, and removals regardless of shared roles', () => { - const cases = /** @type {const} */ ([ - ['global.data.js', 'globalDataDepPaths', 'global-data-changed'], - ['global.vars.js', 'globalVarsDepPaths', 'global-vars-changed'], - ['markdown-it.settings.js', 'markdownDepPaths', 'markdown-settings-changed'], - ['esbuild.settings.js', null, 'global-config-changed'], - ]) - for (const type of /** @type {const} */ (['change', 'added', 'removed'])) { - for (const [name, depSet, reason] of cases) { - const { state, other } = fixture() - state.globalDataDepPaths.clear() - const root = classifyWatchEvent(type, `/site/${name}`) - assert.equal(planWatchBatch(state, [root]).inputChanges.resetReason, reason, `${type} ${name}`) - if (!depSet) continue - for (const sharedName of ['helper.js', 'client.jsx', 'page.md']) { - const filepath = `/site/${sharedName}` - state[depSet].add(filepath) - state.esbuildDepPaths.add(filepath) - state.pageDepMap.set(filepath, new Set([other])) - const batch = planWatchBatch(state, [classifyWatchEvent(type, filepath)]) - assert.equal(batch.inputChanges.resetReason, reason, `${type} ${sharedName}`) - } - } +test('unsupported event types force full resets even without retained state', () => { + for (const hasGlobalDataState of [true, false]) { + const { state } = fixture(hasGlobalDataState) + const unreliable = { ...classifyWatchEvent('change', '/site/page.md'), type: 'rename' } + // @ts-expect-error Runtime callers with unsupported event types must reset, without widening WatchEvent. + const batch = planWatchBatch(state, [unreliable]) + assert.equal(batch.plan.kind, 'full') + assert.equal(batch.inputChanges.resetReason, 'unreliable-event') } }) -test('unknown changes reset and rebuild but known unused layouts and browser entries can skip', () => { - const { state } = fixture() - const unknown = classifyWatchEvent('change', '/site/unknown.js') - assert.equal(planWatchEvent(state, unknown).kind, 'skip', 'single-event behavior is unchanged') - const batch = planWatchBatch(state, [unknown]) - assert.equal(batch.plan.kind, 'full') - assert.equal(batch.inputChanges.resetReason, 'unknown-event') - state.layoutPageMap.clear() - state.pagesFileLayoutMap.clear() - const unusedLayout = planWatchBatch(state, [classifyWatchEvent('change', '/site/root.layout.js')]) - assert.equal(unusedLayout.plan.kind, 'skip') - assert.equal(unusedLayout.inputChanges.resetReason, undefined) - - const unreliable = { ...unknown, type: 'rename' } - // @ts-expect-error Runtime callers with unsupported event types must reset, without widening WatchEvent. - const invalidBatch = planWatchBatch(state, [unreliable]) - assert.equal(invalidBatch.plan.kind, 'full') - assert.equal(invalidBatch.inputChanges.resetReason, 'unreliable-event') -}) - test('page-plan unions deduplicate paths and logging metadata for distinct objects representing one consumer', () => { const { state, home, template, owner } = fixture() - state.pageDepMap.set('/site/shared.js', new Set([home, { ...home }])) - state.templateDepMap.set('/site/shared.js', new Set([template, { ...template }])) - state.pagesFileDepMap.set('/site/shared.js', new Set([owner, { ...owner }])) - const event = classifyWatchEvent('change', '/site/shared.js') + const filepath = home.pageFile.filepath + state.pageDepMap.set(filepath, new Set([home, { ...home }])) + state.templateDepMap.set(filepath, new Set([template, { ...template }])) + state.pagesFileDepMap.set(filepath, new Set([owner, { ...owner }])) + const event = classifyWatchEvent('change', filepath) assert.deepEqual(planWatchBatch(state, [event]).plan, planWatchEvent(state, event), 'a single plan is preserved exactly') const batch = planWatchBatch(state, [event, event]) assert.deepEqual(scope(batch.plan), [[home.pageFile.filepath], [template.templateFile.filepath], [owner.pagesFile.filepath]]) @@ -511,22 +476,12 @@ test('page-plan unions deduplicate paths and logging metadata for distinct objec assert.deepEqual(batch.inputChanges.upsertedPaths, [home.pageFile.filepath]) }) -test('markdown-only imports select only markdown pages and reset producer inputs', () => { - const { state, home } = fixture() - state.markdownDepPaths.add('/site/markdown-helper.js') - const event = classifyWatchEvent('change', '/site/markdown-helper.js') - assert.deepEqual(scope(planWatchEvent(state, event)), [[home.pageFile.filepath], [], []]) - const batch = planWatchBatch(state, [event]) - assert.deepEqual(scope(batch.plan), [[home.pageFile.filepath], [], []]) - assert.equal(batch.inputChanges.resetReason, 'markdown-settings-changed') -}) - test('a full plan never short-circuits later input deltas or reset reasons', () => { const { state } = fixture() - const restart = classifyWatchEvent('added', '/site/client.jsx') + const structural = classifyWatchEvent('removed', '/site/page.md') const config = classifyWatchEvent('change', '/site/global.vars.js') const added = classifyWatchEvent('added', '/site/new/page.md') - for (const events of [[restart, config, added], [config, added, restart], [added, restart, config]]) { + for (const events of [[structural, config, added], [config, added, structural], [added, structural, config]]) { const batch = planWatchBatch(state, events) assert.equal(batch.plan.kind, 'full') assert.equal(batch.inputChanges.resetReason, 'global-vars-changed') @@ -541,5 +496,7 @@ test('reset reason uses the first applicable event after snapshot failure preced const markdown = classifyWatchEvent('change', '/site/markdown-it.settings.js') assert.equal(planWatchBatch(state, [data, markdown]).inputChanges.resetReason, 'global-data-changed') assert.equal(planWatchBatch(state, [markdown, data]).inputChanges.resetReason, 'markdown-settings-changed') - assert.deepEqual(scope(planWatchBatch(state, [data, markdown]).plan), [['/site/page.md'], [], []], 'reset does not force full output selection') + assert.equal(planWatchBatch(state, [data, markdown]).plan.kind, 'full') + const failed = { ...state, pageBuildFailed: true } + assert.equal(planWatchBatch(failed, [data, markdown]).inputChanges.resetReason, 'page-build-failed') }) diff --git a/test-cases/incremental-global-data/helpers.js b/test-cases/incremental-global-data/helpers.js index b7c246b3..ec14853f 100644 --- a/test-cases/incremental-global-data/helpers.js +++ b/test-cases/incremental-global-data/helpers.js @@ -20,7 +20,6 @@ import { setImmediate as nextTurn, setTimeout as delay } from 'node:timers/promi import chokidar from 'chokidar' import pino from 'pino' import { DomStack } from '../../index.js' -import { startWatch } from '../watch/helpers.js' /** @param {string} heading @param {string} [body] @param {string} [frontmatter] */ export function article (heading, body = 'Original body.', frontmatter = '') { @@ -37,12 +36,11 @@ export async function waitFor (predicate, message) { } /** - * Deterministic tests mock only source event delivery; native tests use real watchers. - * Builds, workers, rendering and cleanup are real in both modes. + * Mock only source event delivery; builds, workers, rendering and cleanup are real. * @param {TestContext} t - * @param {{ native?: boolean, files?: Record }} [options] + * @param {{ files?: Record }} [options] */ -export async function fixture (t, { native = false, files = {} } = {}) { +export async function fixture (t, { files = {} } = {}) { const root = await mkdtemp(join(import.meta.dirname, '.tmp-')) const src = join(root, 'src') const dest = join(root, 'public') @@ -53,21 +51,19 @@ export async function fixture (t, { native = false, files = {} } = {}) { const dom = new DomStack(src, dest, { metafile: false, domstackManifest: false, logger: pino({ level: 'silent' }) }) /** @type {FSWatcher | undefined} */ let watcher - if (!native) { - const watch = chokidar.watch - /** @param {Parameters} args */ - const sourceWatch = (...args) => { - if (args[0] !== src) return watch(...args) - const fake = Object.assign(new EventEmitter(), { - closed: false, - async close () { fake.closed = true }, - }) - watcher = /** @type {FSWatcher} */ (/** @type {unknown} */ (fake)) - setImmediate(() => { if (!fake.closed) fake.emit('ready') }) - return watcher - } - t.mock.method(chokidar, 'watch', sourceWatch) + const watch = chokidar.watch + /** @param {Parameters} args */ + const sourceWatch = (...args) => { + if (args[0] !== src) return watch(...args) + const fake = Object.assign(new EventEmitter(), { + closed: false, + async close () { fake.closed = true }, + }) + watcher = /** @type {FSWatcher} */ (/** @type {unknown} */ (fake)) + setImmediate(() => { if (!fake.closed) fake.emit('ready') }) + return watcher } + t.mock.method(chokidar, 'watch', sourceWatch) t.after(async () => { await rm(gate, { force: true }) try { @@ -177,7 +173,7 @@ export default ({ data }) => { return call }, async start () { - const report = native ? await startWatch(t, dom, src) : await dom.watch({ serve: false }) + const report = await dom.watch({ serve: false }) assert.deepEqual(report.pageBuildResults?.errors, [], 'initial page build succeeded') return report }, diff --git a/test-cases/incremental-global-data/index.test.js b/test-cases/incremental-global-data/index.test.js index e462e998..578de350 100644 --- a/test-cases/incremental-global-data/index.test.js +++ b/test-cases/incremental-global-data/index.test.js @@ -46,7 +46,7 @@ test('cached index adds, updates and removes distinct source IDs without rerende test.todo('re-exported page helpers shared with templates upsert affected source pages (https://github.com/bcomnes/domstack/issues/328)') test.todo('named, star, and namespace re-exports of global-data helpers trigger an index reset (https://github.com/bcomnes/domstack/issues/328)') -test('a shared dependency upserts a transitive source page even when a template imports it directly', options, async t => { +test('a shared page and template helper conservatively resets the index and rebuilds both outputs', options, async t => { const site = await fixture(t, { files: { 'code/page.js': "import { content } from '../page-middle.js'; export const vars = { article: true, title: 'Code' }; export default () => content\n", @@ -58,8 +58,9 @@ test('a shared dependency upserts a transitive source page even when a template await site.start() await site.write('page-leaf.js', "export const content = 'Code two'\n") const call = await site.rebuild([site.event('page-leaf.js')]) - assertDelta(call, ['code/page.js']) - assert.deepEqual(call.rendered, ['code/page.js']) + assertReset(call) + assert.equal(call.reason, 'unknown-event') + assert.deepEqual(call.rendered, ['a/page.md', 'b/page.md', 'code/page.js']) assert.equal(await readFile(join(site.dest, 'shared.txt'), 'utf8'), 'Code two') assert.match(await readFile(join(site.dest, 'code/index.html'), 'utf8'), /Code two/) assert.ok((await site.data()).some(row => row.html.includes('Code two'))) @@ -81,7 +82,7 @@ test('a global producer dependency resets the whole index, including in a mixed assert.ok(data.every(row => row.html.startsWith('New search: '))) }) -test('tracked global-vars and Markdown-settings helpers reset the index with updated output', options, async t => { +test('helpers shared by pages and global configuration reset the index and rebuild unrelated outputs', options, async t => { const site = await fixture(t, { files: { 'global.vars.js': "import { site } from './vars-helper.js'; export default { layout: 'root', site }\n", @@ -89,6 +90,7 @@ test('tracked global-vars and Markdown-settings helpers reset the index with upd 'root.layout.js': "export default ({ vars, children }) => '
' + vars.site + '
' + children\n", 'markdown-it.settings.js': "import { html } from './markdown-helper.js'; export default md => md.set({ html })\n", 'markdown-helper.js': 'export const html = true\n', + 'code/page.js': "import { site } from '../vars-helper.js'; import { html } from '../markdown-helper.js'; export default () => site + ':' + html\n", 'a/page.md': article('Alpha', 'Raw HTML'), }, }) @@ -98,14 +100,18 @@ test('tracked global-vars and Markdown-settings helpers reset the index with upd await site.write('vars-helper.js', "export const site = 'Site two'\n") const vars = await site.rebuild([site.event('vars-helper.js')]) assertReset(vars) - assert.equal(vars.reason, 'global-vars-changed') + assert.equal(vars.reason, 'unknown-event') assert.ok((await site.data()).every(row => row.html.includes('
Site two
'))) + assert.match(await readFile(join(site.dest, 'b/index.html'), 'utf8'), /
Site two<\/header>/) + assert.match(await readFile(join(site.dest, 'code/index.html'), 'utf8'), /Site two:true/) await site.write('markdown-helper.js', 'export const html = false\n') const markdown = await site.rebuild([site.event('markdown-helper.js')]) assertReset(markdown) - assert.equal(markdown.reason, 'markdown-settings-changed') + assert.equal(markdown.reason, 'unknown-event') assert.match((await site.data())[0]?.html ?? '', /<strong>Raw HTML<\/strong>/) + assert.match(await readFile(join(site.dest, 'a/index.html'), 'utf8'), /<strong>Raw HTML<\/strong>/) + assert.match(await readFile(join(site.dest, 'code/index.html'), 'utf8'), /Site two:false/) }) test('in-flight events become one ordered batch with deduplicated inputs', options, async t => { @@ -273,25 +279,3 @@ test('independent sessions and a stopped/restarted instance do not inherit cache assertDelta(await first.rebuild([first.event('a/page.md')]), ['a/page.md']) assert.deepEqual((await first.data()).map(row => row.title), ['Restart Alpha', 'Edited while stopped']) }) - -test('a native imported JSON edit resets the producer and publishes every updated row', options, async t => { - const site = await fixture(t, { - native: true, - files: { - 'producer-leaf.js': "import settings from './prefix.json' with { type: 'json' }; export const prefix = settings.prefix\n", - 'prefix.json': JSON.stringify({ prefix: 'Original prefix: ' }), - }, - }) - await site.start() - assert.ok((await site.data()).every(row => row.html.startsWith('Original prefix: '))) - await site.write('prefix.json', JSON.stringify({ prefix: 'Updated prefix: ' })) - await waitFor(async () => (await site.data()).some(row => row.html.startsWith('Updated prefix: ')), 'native JSON edit reaches the published index') - - const call = (await site.calls()).at(-1) - assertReset(call) - assert.equal(call.reason, 'global-data-changed') - assert.deepEqual(call.rendered, ['a/page.md', 'b/page.md']) - const data = await site.data() - assert.deepEqual(data.map(row => row.title), ['Alpha', 'Beta']) - assert.ok(data.every(row => row.html.startsWith('Updated prefix: '))) -}) From 59fd5765e00b3e31f0bd8cf3c5c586fe869d3534 Mon Sep 17 00:00:00 2001 From: Bret Comnes Date: Tue, 15 Sep 2026 12:54:18 -0700 Subject: [PATCH 12/14] Focus incremental global-data docs on learning and usage --- docs/data/README.md | 61 ++++++++++++++++++----------------- docs/implementation/README.md | 18 +++++------ 2 files changed, 40 insertions(+), 39 deletions(-) diff --git a/docs/data/README.md b/docs/data/README.md index 5f3aa38d..e91d1860 100644 --- a/docs/data/README.md +++ b/docs/data/README.md @@ -18,7 +18,7 @@ For output definitions, see [Generation](../generation/); for ordinary configura The `global.data.ts` file is an optional file that can live anywhere in your `src` tree. The first one found wins and duplicates warn. -Its callback runs **once per page build**, after [source-backed pages](../pages/#page-files) are initialized and before generated-page factories run. +Its callback runs **once each time DOMStack builds pages**, after [source-backed pages](../pages/#page-files) are initialized and before generated-page factories run. > [!NOTE] > `global.data.js` works too. @@ -143,10 +143,8 @@ When one layout calls another layout function directly, the composing layout mus - Gives pages, layouts, templates, and page factories only their declared top-level keys through `data`. - Keeps global data separate from ordinary `vars`, so derived values cannot silently collide with page or layout configuration. - Runs inside the worker process (same as all other dynamic imports) to avoid ESM caching issues. -- No producer callback runs if no `global.data.*` file exists. -- During targeted watch rebuilds, DOMStack fingerprints each top-level returned value and adds only consumers subscribed to changed keys to the rebuild set. -- Editing `global.data.*` resets retained state and recomputes data. - When retaining state, watched module edits conservatively reset it and trigger a full rebuild: rediscover inputs, restart esbuild, and rebuild all pages and templates. +- During targeted watch rebuilds, DOMStack compares each top-level returned value with the previous build and also rebuilds consumers subscribed to values that changed. +- Editing `global.data.*` or its watched imports recomputes the shared data. - Values composed of JSON-safe primitives, arrays, and plain objects get stable fingerprints; opaque values such as functions, class instances, maps, sets, or cycles conservatively invalidate their subscribers on every page build. - A declaration naming a missing key fails the build, and access to an existing but undeclared key throws a focused error. @@ -182,9 +180,7 @@ export default globalData Use `AsyncGlobalDataFunction` instead when the implementation needs to await rendering, network requests, or other asynchronous work. For typed source input, use `GlobalDataFunction` or its async counterpart. Helpers can accept `GlobalDataFunctionParams['pages']` without recovering types from the full global-data result. -For retained state, add the final generic: `GlobalDataFunction`, `AsyncGlobalDataFunction`, or `GlobalDataFunctionParams`. -`State` defaults to `unknown`. -Use `GlobalDataChanges` from `@domstack/static/types.js` when typing a helper that handles reset or delta changes. +For callbacks that cache results between watch builds, see [Incremental global data](#incremental-global-data). ## Global data caveats @@ -281,7 +277,7 @@ The current `page` is a `PageInfo` object with the following properties: - `generated`: Metadata about the `*.pages.ts` file that created a generated page, or `undefined` for a source-backed page. Each `PageData` entry supplied to `global.data.ts` exposes this object as `page.pageInfo`. -Use `page.sourceId` for the source-relative index key and `page.pageInfo.url` for the output URL. +When caching entries by source page, use `page.sourceId` as the key and `page.pageInfo.url` as the output URL. Combine `page.pageInfo.url` with a `siteUrl` from `global.vars.ts` to build an absolute URL: `` `${vars.siteUrl}${page.pageInfo.url}` ``. The [RSS and JSON feed recipe](../cookbook/feeds/) uses this pattern for feed item URLs. @@ -360,16 +356,19 @@ After `global.data.ts` returns, consumers receive only the values named by their > Most sites do not need incremental global data. > Start with a regular `global.data.ts` callback and consider incremental indexing when a large number of pages makes watch rebuilds slow. -In watch mode, keep an index so Markdown/HTML source edits only require recomputing affected entries. +An incremental index caches results for each source page so you can update affected entries instead of processing every page on every watch rebuild. +Keep the index in saved state and return the shared values your pages need, such as a navigation list. +Saved state is private to the callback; it is not exposed through `data`. + The callback receives four fields: - `pages`: All current source pages, excluding generated pages. -- `previousState`: Your saved state, or `undefined` when starting fresh. - It is isolated and safe to modify. -- `changes`: On `kind: 'reset'`, rebuild from `pages`. - On `kind: 'delta'`, replace entries for `changes.upserted` and delete the IDs in `changes.removed`. -- `setState(next)`: Save a snapshot for the next successful build. - Always call it to retain your updates. +- `previousState`: A copy of your saved state, or `undefined` when starting fresh. + You can modify it, but must call `setState` to save your updates. +- `changes`: On `kind: 'reset'`, rebuild the index from `pages`. + On `kind: 'delta'`, update entries for the new or affected pages in `changes.upserted` and delete the IDs in `changes.removed`. +- `setState(next)`: Take a snapshot of your index to reuse on the next build. + DOMStack keeps that snapshot only if the current build succeeds. Use `page.sourceId` as the index key: a read-only source-relative path such as `docs/data/README.md`, using `/` separators on every platform. `changes.removed` contains these same IDs. @@ -421,20 +420,24 @@ export default function ({ } ``` -A consuming layout declares `export const vars = { dataDeps: ['docsNavigation'] }` and reads `data.docsNavigation`. -On a Markdown/HTML source edit, unchanged navigation does not cause additional subscriber rebuilds. -For a larger example that also caches heading links, see `site/globals/global.data.ts` in this repository. +A layout subscribes with `export const vars = { dataDeps: ['docsNavigation'] }` and reads `data.docsNavigation`. +If you edit a Markdown page without changing its title or URL, the navigation stays the same, so other pages do not rebuild just because they subscribe to it. +For an example that also caches heading links, see the [documentation site's index](https://github.com/bcomnes/domstack/blob/master/site/globals/global.data.ts). + +The third type argument in `GlobalDataFunctionParams` describes the saved state—`Index` in this example. +For `GlobalDataFunction` and `AsyncGlobalDataFunction`, `State` is the fourth type argument, after the result, source-vars, and source-content types. +It defaults to `unknown`. +Helpers that process `changes` can use the exported `GlobalDataChanges` type. ### Usage notes -- Markdown/HTML source edits can use incremental deltas. - When retaining state, watched module edits—including JavaScript/TypeScript pages, page-variable modules, helpers, browser entries, and settings—reset the index and trigger a full rebuild. - Without retained state, existing watch rebuild scopes are unchanged. -- State lasts for the current watch session and is retained only after a successful build. - Always handle resets, including after producer or settings changes and failed builds. -- Store cloneable records, arrays, or maps—not `PageData` instances, functions, or shared memory. -- An upsert may repeat even when content is unchanged. +- State lasts for one watch session, not across process restarts. + Always handle `changes.kind === 'reset'`, including after failed builds. +- Markdown and HTML source edits can update individual entries. + Changes to watched modules, including JavaScript/TypeScript pages, helpers, browser entries, and settings, reset the saved index and rebuild the site. +- Store structured-cloneable values such as plain records, arrays, or maps—not `PageData` instances, functions, or shared memory. +- A page can appear in `changes.upserted` even when its content is unchanged. Replace its cached entry, or delete it if the page no longer belongs in your index. -- Dependency tracking supports relative static imports, but the page watcher observes only processed file extensions within the watched source tree, not imported JSON files. - Re-exports (`export … from`) are not tracked yet ([#328](https://github.com/bcomnes/domstack/issues/328)); use an explicit import followed by a local export, or restart watching after those inputs change. -- Restart watching when imported JSON or other inputs change, such as arbitrary file reads, environment variables, or network data. +- Restart watch mode after changing imported JSON, files read with `fs.readFile()`, environment variables, or other inputs outside page dependency tracking. +- Static re-exports (`export … from`) are not followed by dependency tracking ([#328](https://github.com/bcomnes/domstack/issues/328)). + Use an explicit import followed by a local export, or restart watch mode after those dependencies change. diff --git a/docs/implementation/README.md b/docs/implementation/README.md index 610bde82..a25a520f 100644 --- a/docs/implementation/README.md +++ b/docs/implementation/README.md @@ -229,7 +229,7 @@ Require a full page retry`"] Bundle replanning returns only a page plan or a skip; it does not restart esbuild again. After a page-build failure, the next page-producing plan retries the complete page phase rather than trusting incremental filters. -Without retained global-data state, manifest-settings changes and service-worker entry additions or removals retain their intentional page-phase skips. +Manifest-settings changes and service-worker entry additions or removals skip page rendering unless an incremental global-data index needs to be reset. The one-shot manifest pipeline is not part of watch execution. > [!NOTE] @@ -250,10 +250,8 @@ When a targeted build recomputes global data, DOMStack compares top-level values ### What triggers what -The tables below describe the unchanged defaults when no global-data state is retained. -When retaining state, watched module edits—including JavaScript/TypeScript pages, page-variable modules, helpers, browser entries, and manifest or other settings—reset the index and trigger a **full rebuild**: rediscover inputs, restart esbuild, and rebuild all pages and templates. -Direct Markdown/HTML source edits can still use incremental deltas. -Input changes can request a global-data reset without retained state, but do not change these default output rebuild scopes. +The tables below apply when `global.data.ts` does not save state with `setState`. +With an [incremental index](../data/#incremental-global-data), Markdown and HTML edits can update individual entries, while changes to watched modules reset the index and trigger a **full rebuild**. | Change | Rebuild scope | |---|---| @@ -285,8 +283,8 @@ When a full page/template rebuild or targeted generated-pages rebuild no longer ### Dependency tracking DOMStack uses [`@11ty/dependency-tree-typescript`](https://github.com/11ty/dependency-tree-typescript) to statically analyze ESM imports. -The page watcher observes only processed file extensions in the watched source tree; imported JSON changes do not trigger page rebuilds. -It maintains maps for: +Imported JSON changes do not trigger page rebuilds; restart watch mode after editing those files. +DOMStack maintains dependency maps for: - Layout dependencies, source-backed pages using each layout, and generated-page owner layout membership - TypeScript pages and adjacent page-variable dependencies @@ -300,8 +298,8 @@ Dependency analysis is best-effort. When DOMStack cannot safely determine a targeted scope, it falls back to a broader rebuild or skips an unrelated changed module. esbuild tracks browser-entry dependencies independently. -Without retained global-data state, changing a module imported only by `client.ts` rebundles that entry without rendering page HTML. -When retaining state, watched module edits trigger the full rebuild described above. +A module imported only by `client.ts` normally triggers rebundling without rendering page HTML. +If `global.data.ts` has saved an incremental index, the module edit also resets that index and triggers a full rebuild. ### Stable entry filenames @@ -328,7 +326,7 @@ Page HTML points to stable entry files during watch mode. esbuild can update an ### Manifest behavior Watch mode builds and rebundles the site service worker, but it does not finalize, return, or write the [DOMStack manifest](../../docs/workers/#domstack-manifest). -Changes to `domstack-manifest.settings.ts` do not trigger a watch rebuild unless global-data state is retained. +Editing `domstack-manifest.settings.ts` only triggers a rebuild when an incremental global-data index needs to be reset; it does not generate a manifest. Use `domstack --serve` when testing manifest-driven cache behavior. It runs a one-shot build and serves the result without watch-mode filenames or live-reload HTML injection. From 6c23ab1af3220f6f0c38d55514d5f672648a32d5 Mon Sep 17 00:00:00 2001 From: Bret Comnes Date: Tue, 15 Sep 2026 13:32:12 -0700 Subject: [PATCH 13/14] Extract incremental data test modules into fixture files --- .../fixtures/data.json.template.js | 12 +++++ .../fixtures/global.data.js | 51 +++++++++++++++++++ .../fixtures/producer-leaf.js | 1 + .../fixtures/producer-middle.js | 2 + test-cases/incremental-global-data/helpers.js | 50 ++---------------- 5 files changed, 69 insertions(+), 47 deletions(-) create mode 100644 test-cases/incremental-global-data/fixtures/data.json.template.js create mode 100644 test-cases/incremental-global-data/fixtures/global.data.js create mode 100644 test-cases/incremental-global-data/fixtures/producer-leaf.js create mode 100644 test-cases/incremental-global-data/fixtures/producer-middle.js diff --git a/test-cases/incremental-global-data/fixtures/data.json.template.js b/test-cases/incremental-global-data/fixtures/data.json.template.js new file mode 100644 index 00000000..14aa9327 --- /dev/null +++ b/test-cases/incremental-global-data/fixtures/data.json.template.js @@ -0,0 +1,12 @@ +/** @import { IndexRow } from './global.data.js' */ +import { existsSync } from 'node:fs' + +const renderFailure = new URL('../fail-render', import.meta.url) + +export const dataDeps = ['index'] + +/** @param {{ data: { index: IndexRow[] } }} params */ +export default function dataTemplate ({ data }) { + if (existsSync(renderFailure)) throw new Error('intentional later render failure') + return JSON.stringify(data.index) +} diff --git a/test-cases/incremental-global-data/fixtures/global.data.js b/test-cases/incremental-global-data/fixtures/global.data.js new file mode 100644 index 00000000..53a5ab07 --- /dev/null +++ b/test-cases/incremental-global-data/fixtures/global.data.js @@ -0,0 +1,51 @@ +/** + * @import { GlobalDataFunctionParams } from '@domstack/static/types.js' + * @typedef {{ url: string, title: string, html: string }} IndexRow + */ +import { appendFileSync, existsSync } from 'node:fs' +import { setTimeout as delay } from 'node:timers/promises' +import { prefix } from './producer-middle.js' + +const log = new URL('../producer.jsonl', import.meta.url) +const gate = new URL('../hold-producer', import.meta.url) +const producerFailure = new URL('../fail-producer', import.meta.url) + +/** @param {GlobalDataFunctionParams<{ article?: boolean, title: string }, string, Map>} params */ +export default async function globalData ({ pages, previousState, changes, setState }) { + const state = changes.kind === 'reset' ? new Map() : new Map(previousState) + const rendered = [] + if (changes.kind === 'delta') { + for (const sourceId of changes.removed) state.delete(sourceId) + } + for (const page of changes.kind === 'reset' ? pages : changes.upserted) { + if (!page.vars.article) { + state.delete(page.sourceId) + continue + } + rendered.push(page.sourceId) + state.set(page.sourceId, { + url: page.pageInfo.url, + title: page.vars.title, + html: prefix + await page.renderFullPage(), + }) + } + setState(state) + // Capture failure before the gate so a buffered retry can recover. + const fail = existsSync(producerFailure) + appendFileSync(log, JSON.stringify({ + kind: changes.kind, + reason: changes.kind === 'reset' ? changes.reason : undefined, + events: changes.events.map(({ type, filepath }) => ({ type, filepath })), + upserted: changes.kind === 'delta' ? changes.upserted.map(page => page.sourceId).sort() : [], + removed: changes.kind === 'delta' ? [...changes.removed].sort() : [], + rendered: rendered.sort(), + previousKeys: previousState === undefined ? null : [...previousState.keys()].sort(), + }) + '\n') + const deadline = Date.now() + 8000 + while (existsSync(gate)) { + if (Date.now() > deadline) throw new Error('Timed out waiting for test producer gate') + await delay(10) + } + if (fail) throw new Error('intentional producer failure after setState') + return { index: [...state.values()].sort((a, b) => a.url.localeCompare(b.url)) } +} diff --git a/test-cases/incremental-global-data/fixtures/producer-leaf.js b/test-cases/incremental-global-data/fixtures/producer-leaf.js new file mode 100644 index 00000000..86ed7bdb --- /dev/null +++ b/test-cases/incremental-global-data/fixtures/producer-leaf.js @@ -0,0 +1 @@ +export const prefix = 'Search: ' diff --git a/test-cases/incremental-global-data/fixtures/producer-middle.js b/test-cases/incremental-global-data/fixtures/producer-middle.js new file mode 100644 index 00000000..6545044d --- /dev/null +++ b/test-cases/incremental-global-data/fixtures/producer-middle.js @@ -0,0 +1,2 @@ +import { prefix } from './producer-leaf.js' +export { prefix } diff --git a/test-cases/incremental-global-data/helpers.js b/test-cases/incremental-global-data/helpers.js index ec14853f..fbd28bf1 100644 --- a/test-cases/incremental-global-data/helpers.js +++ b/test-cases/incremental-global-data/helpers.js @@ -1,6 +1,7 @@ /** * @import { TestContext } from 'node:test' * @import { FSWatcher } from 'chokidar' + * @import { IndexRow } from './fixtures/global.data.js' * @typedef {{ type: 'added' | 'removed' | 'change', filepath: string }} InputEvent * @typedef {object} ProducerCall * @property {'reset' | 'delta'} kind @@ -10,11 +11,10 @@ * @property {string[]} removed * @property {string[]} rendered * @property {string[] | null} previousKeys - * @typedef {{ url: string, title: string, html: string }} IndexRow */ import assert from 'node:assert/strict' import { EventEmitter } from 'node:events' -import { mkdir, mkdtemp, readFile, rm, writeFile } from 'node:fs/promises' +import { cp, mkdir, mkdtemp, readFile, rm, writeFile } from 'node:fs/promises' import { dirname, join } from 'node:path' import { setImmediate as nextTurn, setTimeout as delay } from 'node:timers/promises' import chokidar from 'chokidar' @@ -79,54 +79,10 @@ export async function fixture (t, { files = {} } = {}) { await mkdir(dirname(path), { recursive: true }) await writeFile(path, content) } - const producer = ` -import { appendFileSync, existsSync } from 'node:fs' -import { setTimeout as delay } from 'node:timers/promises' -import { prefix } from './producer-middle.js' -export default async ({ pages, previousState, changes, setState }) => { - const state = changes.kind === 'reset' ? new Map() : new Map(previousState) - const rendered = [] - for (const sourceId of changes.removed ?? []) state.delete(sourceId) - for (const page of changes.kind === 'reset' ? pages : changes.upserted) { - if (!page.vars.article) { state.delete(page.sourceId); continue } - rendered.push(page.sourceId) - state.set(page.sourceId, { - url: page.pageInfo.url, - title: page.vars.title, - html: prefix + await page.renderFullPage(), - }) - } - setState(state) - // Capture failure before the gate so a buffered retry can recover. - const fail = existsSync(${JSON.stringify(producerFailure)}) - appendFileSync(${JSON.stringify(log)}, JSON.stringify({ - kind: changes.kind, reason: changes.reason, - events: changes.events.map(({ type, filepath }) => ({ type, filepath })), - upserted: (changes.upserted ?? []).map(page => page.sourceId).sort(), - removed: [...(changes.removed ?? [])].sort(), rendered: rendered.sort(), - previousKeys: previousState === undefined ? null : [...previousState.keys()].sort(), - }) + '\\n') - const deadline = Date.now() + 8000 - while (existsSync(${JSON.stringify(gate)})) { - if (Date.now() > deadline) throw new Error('Timed out waiting for test producer gate') - await delay(10) - } - if (fail) throw new Error('intentional producer failure after setState') - return { index: [...state.values()].sort((a, b) => a.url.localeCompare(b.url)) } -} -` + await cp(new URL('./fixtures/', import.meta.url), src, { recursive: true }) const defaults = { - 'global.data.js': producer, - 'producer-middle.js': "import { prefix } from './producer-leaf.js'; export { prefix }\n", - 'producer-leaf.js': "export const prefix = 'Search: '\n", 'a/page.md': article('Alpha'), 'b/page.md': article('Beta'), - 'data.json.template.js': `import { existsSync } from 'node:fs' -export const dataDeps = ['index'] -export default ({ data }) => { - if (existsSync(${JSON.stringify(renderFailure)})) throw new Error('intentional later render failure') - return JSON.stringify(data.index) -}\n`, ...files, } await Promise.all(Object.entries(defaults).map(([name, content]) => write(name, content))) From 78a51fbae22a5976212e75b186a3cd6191c617f6 Mon Sep 17 00:00:00 2001 From: Bret Comnes Date: Tue, 15 Sep 2026 18:35:09 -0700 Subject: [PATCH 14/14] Tighten global-data state isolation and simplify startup --- docs/data/README.md | 2 +- index.js | 13 ++----- lib/build-pages/global-data-state.js | 23 ++++++++---- lib/build-pages/global-data-state.test.js | 45 +++++++++++++++++++++-- lib/build-pages/index.js | 28 ++++++++------ 5 files changed, 79 insertions(+), 32 deletions(-) diff --git a/docs/data/README.md b/docs/data/README.md index e91d1860..0bc902ef 100644 --- a/docs/data/README.md +++ b/docs/data/README.md @@ -435,7 +435,7 @@ Helpers that process `changes` can use the exported `GlobalDataChanges` type. Always handle `changes.kind === 'reset'`, including after failed builds. - Markdown and HTML source edits can update individual entries. Changes to watched modules, including JavaScript/TypeScript pages, helpers, browser entries, and settings, reset the saved index and rebuild the site. -- Store structured-cloneable values such as plain records, arrays, or maps—not `PageData` instances, functions, or shared memory. +- Store structured-cloneable values such as plain records, arrays, or maps—not `PageData` instances, functions, or native objects with shared mutable storage. - A page can appear in `changes.upserted` even when its content is unchanged. Replace its cached entry, or delete it if the page no longer belongs in your index. - Restart watch mode after changing imported JSON, files read with `fs.readFile()`, environment variables, or other inputs outside page dependency tracking. diff --git a/index.js b/index.js index 962776bc..b7e58ae0 100644 --- a/index.js +++ b/index.js @@ -229,9 +229,7 @@ export class DomStack { const { signal } = session.cancellation const preparation = this.#prepareWatch(session) session.startupWork = preparation - const { report, ready } = await preparation - - await ready + const report = await preparation if (signal.aborted) return report await onInitialBuild?.(report) @@ -254,8 +252,7 @@ export class DomStack { const { signal } = session.cancellation // Establish observation before discovery. Initial scan adds are not edits; // subsequent events stay buffered until startup and the user callback finish. - const { watcher, ready } = this.#createSourceWatcher(session) - await ready + await this.#createSourceWatcher(session) // ── Initial build (inline, not via builder()) ──────────────────────── const siteData = await identifyPages(this.#src, this.opts) @@ -327,7 +324,7 @@ export class DomStack { const copyErrors = copyStartup.filter(result => result.status === 'rejected').map(result => result.reason) if (copyErrors.length) throw new AggregateError(copyErrors, 'Copy watch startup failed') - return { report, watcher, ready } + return report } /** @param {WatchSession} session */ @@ -359,11 +356,9 @@ export class DomStack { watcher.on('error', err => errorLogger(err, this.#logger)) // Attach the listener before returning; the watcher can become ready before // the caller resumes. Cancellation settles this wait even without a ready event. - const ready = once(watcher, 'ready', { signal }).catch(error => { + return once(watcher, 'ready', { signal }).catch(error => { if (!signal.aborted || error.name !== 'AbortError') throw error }) - - return { watcher, ready } } /** diff --git a/lib/build-pages/global-data-state.js b/lib/build-pages/global-data-state.js index 55ceb4bc..0364daea 100644 --- a/lib/build-pages/global-data-state.js +++ b/lib/build-pages/global-data-state.js @@ -4,6 +4,11 @@ * @import { WatchEvent } from '../watch-plan.js' */ import { resolve } from 'node:path' +import { BlockList } from 'node:net' +import { createHistogram } from 'node:perf_hooks' + +// Node does not export Histogram; both recordable and event-loop histograms extend it. +const Histogram = Object.getPrototypeOf(createHistogram().constructor) /** * Internal candidate committed by the main build only after successful cleanup. @@ -56,8 +61,8 @@ function cloneState (state, operation) { throw new Error(`global.data ${operation} requires structured-cloneable state. Store plain data, Maps, Sets, or other structured-cloneable values instead of functions, PageData instances, or renderers.`, { cause }) } // Inspect only retained values: cloning evaluates source getters once and drops - // private fields, but SharedArrayBuffer still aliases its original memory. - rejectSharedMemory(cloned, operation) + // private fields, but some native objects still share mutable backing storage. + rejectSharedState(cloned, operation) return cloned } @@ -66,28 +71,32 @@ function cloneState (state, operation) { * @param {string} operation * @param {Set} [seen] */ -function rejectSharedMemory (value, operation, seen = new Set()) { +function rejectSharedState (value, operation, seen = new Set()) { if (value === null || typeof value !== 'object' || seen.has(value)) return seen.add(value) if (value instanceof SharedArrayBuffer || (ArrayBuffer.isView(value) && value.buffer instanceof SharedArrayBuffer)) { throw new Error(`global.data ${operation} cannot retain shared-memory state (SharedArrayBuffer or views backed by it): structuredClone shares their memory. Copy the bytes into a non-shared ArrayBuffer or typed array before storing state.`) } + if ((typeof WebAssembly !== 'undefined' && value instanceof WebAssembly.Memory) || + value instanceof Histogram || value instanceof BlockList) { + throw new Error(`global.data ${operation} cannot retain shared mutable state (WebAssembly.Memory, Histogram, or BlockList): structuredClone shares their backing storage. Store plain records or arrays instead.`) + } // Cloned views have no custom properties; avoid walking every byte or element. if (ArrayBuffer.isView(value)) return if (value instanceof Map) { for (const [key, entry] of value) { - rejectSharedMemory(key, operation, seen) - rejectSharedMemory(entry, operation, seen) + rejectSharedState(key, operation, seen) + rejectSharedState(entry, operation, seen) } return } else if (value instanceof Set) { - for (const entry of value) rejectSharedMemory(entry, operation, seen) + for (const entry of value) rejectSharedState(entry, operation, seen) return } // Include non-enumerable retained properties such as Error.cause. for (const key of Object.getOwnPropertyNames(value)) { const descriptor = Object.getOwnPropertyDescriptor(value, key) - if (descriptor && 'value' in descriptor) rejectSharedMemory(descriptor.value, operation, seen) + if (descriptor && 'value' in descriptor) rejectSharedState(descriptor.value, operation, seen) } } diff --git a/lib/build-pages/global-data-state.test.js b/lib/build-pages/global-data-state.test.js index 3cf621c4..34596896 100644 --- a/lib/build-pages/global-data-state.test.js +++ b/lib/build-pages/global-data-state.test.js @@ -6,6 +6,8 @@ import { test } from 'node:test' import { mkdtemp, mkdir, writeFile, readFile, rm } from 'node:fs/promises' import { tmpdir } from 'node:os' import { join } from 'node:path' +import { BlockList } from 'node:net' +import { createHistogram, monitorEventLoopDelay } from 'node:perf_hooks' import { createGlobalDataState } from './global-data-state.js' import { buildPages, buildPagesDirect } from './index.js' import { identifyPages } from '../identify-pages.js' @@ -68,7 +70,7 @@ test('cyclic state is supported and an uncloneable update leaves the last candid assert.equal(transaction.getBaseline().state, candidate) }) -test('shared memory is rejected throughout nested state without replacing a valid candidate', () => { +test('shared mutable storage is rejected throughout nested state without replacing a valid candidate', () => { const buffer = new SharedArrayBuffer(8) const cycle = { self: /** @type {unknown} */ (null), buffer } cycle.self = cycle @@ -82,16 +84,20 @@ test('shared memory is rejected throughout nested state without replacing a vali new Set([buffer]), cycle, new Error('nested cause', { cause: buffer }), + { memory: new WebAssembly.Memory({ initial: 1, maximum: 1, shared: true }) }, + new Map([['histogram', createHistogram()]]), + monitorEventLoopDelay(), + { blockList: new BlockList() }, ] const transaction = createGlobalDataState({ pages: [] }) transaction.context.setState({ bytes: new Uint8Array([1]) }) const candidate = transaction.getBaseline().state for (const state of states) { - assert.throws(() => transaction.context.setState(state), /SharedArrayBuffer/) + assert.throws(() => transaction.context.setState(state), /shared.*state/) assert.equal(transaction.getBaseline().state, candidate) assert.throws(() => createGlobalDataState({ pages: [], previousGlobalDataBaseline: { state, sourceIds: [] }, - }), /global\.data previous state.*SharedArrayBuffer/) + }), /global\.data previous state.*shared.*state/) } }) @@ -226,8 +232,12 @@ test('worker validation errors remain actionable and never emit a candidate', as for (const [expression, expected] of /** @type {const} */ ([ ['{ render () {} }', /structured-cloneable/], ['new SharedArrayBuffer(8)', /SharedArrayBuffer/], + ['new WebAssembly.Memory({ initial: 1, maximum: 1, shared: true })', /shared.*state/], + ['(await import("node:perf_hooks")).createHistogram()', /shared.*state/], + ['(await import("node:perf_hooks")).monitorEventLoopDelay()', /shared.*state/], + ['new (await import("node:net")).BlockList()', /shared.*state/], ])) { - const { src, dest } = await fixture(t, `export default ({ setState }) => { setState(${expression}); return {} }`) + const { src, dest } = await fixture(t, `export default async ({ setState }) => { setState(${expression}); return { summary: 'stable' } }`) const result = await buildPages(src, dest, await identifyPages(src), { trackWatchDependencies: true }) assert.equal(result.errors.length, 1) const error = result.errors[0] @@ -237,6 +247,33 @@ test('worker validation errors remain actionable and never emit a candidate', as } }) +test('shared native state cannot cross into a worker callback and mutate the retained baseline', async t => { + const { src, dest } = await fixture(t, `export default ({ previousState }) => { + new Uint8Array(previousState.memory.buffer)[0] = 42 + return { summary: 'unchanged' } + }`) + const memory = new WebAssembly.Memory({ initial: 1, maximum: 1, shared: true }) + const result = await buildPages(src, dest, await identifyPages(src), { + trackWatchDependencies: true, + previousGlobalDataBaseline: { state: { memory }, sourceIds: ['page.js'] }, + }) + assert.equal(new Uint8Array(memory.buffer)[0], 0) + assert.equal(result.errors.length, 1) + const error = result.errors[0] + assert.ok(error instanceof Error) + assert.match(error.message, /global\.data previous state.*shared.*state/) + assert.equal(Object.hasOwn(result.report, 'globalDataBaseline'), false) +}) + +test('watch builds without global data do not emit an index baseline', async t => { + const { src, dest } = await fixture(t, 'export default {}') + await rm(join(src, 'global.data.js')) + await writeFile(join(src, 'page.js'), 'export default () => "Home"') + const result = await buildPages(src, dest, await identifyPages(src), { trackWatchDependencies: true }) + assert.deepEqual(result.errors, []) + assert.equal(Object.hasOwn(result.report, 'globalDataBaseline'), false) +}) + test('resolveGlobalData preserves static exports and accepts a missing global-data file', async t => { const { src } = await fixture(t, 'export default { render: () => "public function", value: 1 }') const globalDataPath = join(src, 'global.data.js') diff --git a/lib/build-pages/index.js b/lib/build-pages/index.js index 01988069..29351b90 100644 --- a/lib/build-pages/index.js +++ b/lib/build-pages/index.js @@ -58,7 +58,7 @@ const __dirname = import.meta.dirname * @template [U=any] - Source-page render values. * @template [S=unknown] - Explicitly retained, structured-cloneable state. * @typedef {object} GlobalDataFunctionParams - * @property {PageData[]} pages - Fully initialized source-backed pages, before generated pages are created. Identities are normalized absolute pageInfo.pageFile.filepath values. + * @property {PageData[]} pages - Fully initialized source-backed pages, before generated pages are created. Use page.sourceId (a source-relative path with POSIX separators) as the index key, matching changes.removed. * @property {S | undefined} previousState - Isolated state from the last successful build, or undefined on reset. * @property {GlobalDataChanges} changes - Reset information or source-page input deltas, independent of output filters. * @property {(next: S) => void} setState - Immediately snapshot state for the next successful build. Mutating previousState alone does not commit changes. @@ -633,15 +633,19 @@ export async function buildPagesDirect (_src, dest, siteData, opts) { // Derive collection data from source-backed pages before generated-page factories run. // This keeps generated pages downstream while making shared data available to them. - const globalDataState = createGlobalDataState({ - pages: concretePages, - previousGlobalDataBaseline: opts?.previousGlobalDataBaseline, - globalDataInputChanges: opts?.globalDataInputChanges, - }) - const globalData = /** @type {Record} */ (await resolveGlobalData({ - globalDataPath: siteData.globalData?.filepath, - context: globalDataState.context, - })) + const globalDataState = siteData.globalData + ? createGlobalDataState({ + pages: concretePages, + previousGlobalDataBaseline: opts?.previousGlobalDataBaseline, + globalDataInputChanges: opts?.globalDataInputChanges, + }) + : null + const globalData = /** @type {Record} */ (globalDataState + ? await resolveGlobalData({ + globalDataPath: siteData.globalData?.filepath, + context: globalDataState.context, + }) + : {}) const changedGlobalDataKeys = watchDependencyTracker.updateGlobalDataFingerprints( globalData, opts?.previousWatchDependencies?.globalDataFingerprints @@ -789,7 +793,9 @@ export async function buildPagesDirect (_src, dest, siteData, opts) { ) result.report.watchDependencies = watchDependencyTracker.state } - if (opts?.trackWatchDependencies && result.errors.length === 0) result.report.globalDataBaseline = globalDataState.getBaseline() + if (opts?.trackWatchDependencies && result.errors.length === 0 && globalDataState) { + result.report.globalDataBaseline = globalDataState.getBaseline() + } return result }