diff --git a/.github/workflows/generate.yml b/.github/workflows/generate.yml index e02fdb0d..0d60eed6 100644 --- a/.github/workflows/generate.yml +++ b/.github/workflows/generate.yml @@ -76,36 +76,36 @@ jobs: fail-fast: false matrix: include: - - target: man-page + - target: '@node-core/doc-kit/generators/man-page' input: './node/doc/api/cli.md' - - target: addon-verify + - target: '@node-core/doc-kit/generators/addon-verify' input: './node/doc/api/addons.md' - - target: api-links + - target: '@node-core/doc-kit/generators/api-links' input: './node/lib/*.js' compare: object-assertion - - target: orama-db + - target: '@node-core/doc-kit/generators/orama-db' input: './node/doc/api/*.md' compare: file-size - - target: json-simple + - target: '@node-core/doc-kit/generators/json-simple' input: './node/doc/api/*.md' - - target: legacy-json + - target: '@node-core/doc-kit/generators/legacy-json' input: './node/doc/api/*.md' compare: object-assertion - - target: legacy-html + - target: '@node-core/doc-kit/generators/legacy-html' input: './node/doc/api/*.md' compare: file-size - - target: web + - target: '@node-core/doc-kit/generators/web' input: './node/doc/api/*.md' compare: file-size - - target: llms-txt + - target: '@node-core/doc-kit/generators/llms-txt' input: './node/doc/api/*.md' compare: file-size steps: @@ -140,10 +140,14 @@ jobs: - name: Install dependencies run: npm ci + - name: Get generator name + id: generator + run: echo "name=$(node -e "import('${{ matrix.target }}').then(m => console.log(m.name))")" >> "$GITHUB_OUTPUT" + - name: Create output directory - run: mkdir -p out/${{ matrix.target }} + run: mkdir -p out/${{ steps.generator.outputs.name }} - - name: Generate ${{ matrix.target }} + - name: Generate ${{ steps.generator.outputs.name }} run: | node packages/core/bin/cli.mjs generate \ -t ${{ matrix.target }} \ @@ -157,7 +161,7 @@ jobs: if: ${{ matrix.compare && needs.prepare.outputs.base-run }} uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 with: - name: ${{ matrix.target }} + name: ${{ steps.generator.outputs.name }} path: base run-id: ${{ needs.prepare.outputs.base-run }} github-token: ${{ secrets.GITHUB_TOKEN }} @@ -169,8 +173,8 @@ jobs: run: | node scripts/comparators/${{ matrix.compare }}.mjs > out/comparison.txt - - name: Upload ${{ matrix.target }} artifacts + - name: Upload ${{ steps.generator.outputs.name }} artifacts uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0 with: - name: ${{ matrix.target }} + name: ${{ steps.generator.outputs.name }} path: out diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 78f3337c..b9b29890 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -89,7 +89,7 @@ The steps below will give you a general idea of how to prepare your local enviro For fast iteration during development, target a single Markdown file instead of all API docs: ```bash - node bin/cli.mjs generate \ + node packages/core/bin/cli.mjs generate \ -t legacy-html \ -i ../node/doc/api/fs.md \ -o out \ @@ -110,7 +110,7 @@ The steps below will give you a general idea of how to prepare your local enviro Add `--log-level debug` before the `generate` subcommand to see the full pipeline trace: ```bash - node bin/cli.mjs --log-level debug generate -t legacy-html -i ../node/doc/api/fs.md -o out + node packages/core/bin/cli.mjs --log-level debug generate -t legacy-html -i ../node/doc/api/fs.md -o out ``` > [!TIP] diff --git a/README.md b/README.md index 607cf69f..47a3c9c2 100644 --- a/README.md +++ b/README.md @@ -68,7 +68,7 @@ Options: -v, --version Target Node.js version (default: "v22.14.0") -c, --changelog Changelog URL or path (default: "https://raw.githubusercontent.com/nodejs/node/HEAD/CHANGELOG.md") --git-ref Git ref/commit URL (default: "https://github.com/nodejs/node/tree/HEAD") - -t, --target [modes...] Target generator modes (choices: "json-simple", "legacy-html", "legacy-html-all", "man-page", "legacy-json", "legacy-json-all", "addon-verify", "api-links", "orama-db", "llms-txt") + -t, --target Target generator(s) (e.g. @node-core/doc-kit/generators/web) -h, --help display help for command ``` @@ -91,8 +91,8 @@ To generate a 1:1 match with the [legacy tooling](https://github.com/nodejs/node ```sh npx doc-kit generate \ - -t legacy-html \ - -t legacy-json \ + -t @node-core/doc-kit/generators/legacy-html \ + -t @node-core/doc-kit/generators/legacy-json \ -i "path/to/node/doc/api/*.md" \ -o out \ --index path/to/node/doc/api/index.md @@ -104,8 +104,8 @@ To generate [our redesigned documentation pages](https://nodejs-api-docs-tooling ```sh npx doc-kit generate \ - -t web \ - -t orama-db \ + -t @node-core/doc-kit/generators/web \ + -t @node-core/doc-kit/generators/orama-db \ -i "path/to/node/doc/api/*.md" \ -o out \ --index path/to/node/doc/api/index.md diff --git a/docs/commands.md b/docs/commands.md index 9a78f7af..1703aeed 100644 --- a/docs/commands.md +++ b/docs/commands.md @@ -69,7 +69,7 @@ export default [ ### Step 3: Update CLI Entry Point -The CLI in `bin/cli.mjs` automatically loads commands from `bin/commands/index.mjs`, so no changes are needed there if you followed step 2. +The CLI in `packages/core/bin/cli.mjs` automatically loads commands from `bin/commands/index.mjs`, so no changes are needed there if you followed step 2. ## Command Options diff --git a/docs/configuration.md b/docs/configuration.md index 9a073e74..26353b43 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -100,17 +100,17 @@ Configurations are merged in the following order (earlier sources take precedenc CLI options map to configuration properties: -| CLI Option | Config Property | Example | -| ---------------------- | ------------------ | ------------------------- | -| `--input ` | `global.input` | `--input src/` | -| `--output ` | `global.output` | `--output dist/` | -| `--ignore ` | `global.ignore[]` | `--ignore test/` | -| `--minify` | `global.minify` | `--minify` | -| `--git-ref ` | `global.ref` | `--git-ref v20.0.0` | -| `--version ` | `global.version` | `--version 20.0.0` | -| `--changelog ` | `global.changelog` | `--changelog https://...` | -| `--index ` | `global.index` | `--index file://...` | -| `--type-map ` | `metadata.typeMap` | `--type-map file://...` | -| `--target ` | `target` | `--target json` | -| `--threads ` | `threads` | `--threads 4` | -| `--chunk-size ` | `chunkSize` | `--chunk-size 10` | +| CLI Option | Config Property | Example | +| ---------------------- | ------------------ | ---------------------------------------------------- | +| `--input ` | `global.input` | `--input src/` | +| `--output ` | `global.output` | `--output dist/` | +| `--ignore ` | `global.ignore[]` | `--ignore test/` | +| `--minify` | `global.minify` | `--minify` | +| `--git-ref ` | `global.ref` | `--git-ref v20.0.0` | +| `--version ` | `global.version` | `--version 20.0.0` | +| `--changelog ` | `global.changelog` | `--changelog https://...` | +| `--index ` | `global.index` | `--index file://...` | +| `--type-map ` | `metadata.typeMap` | `--type-map file://...` | +| `--target ` | `target` | `--target @node-core/doc-kit/generators/legacy-json` | +| `--threads ` | `threads` | `--threads 4` | +| `--chunk-size ` | `chunkSize` | `--chunk-size 10` | diff --git a/docs/generators.md b/docs/generators.md index d86f9465..1c64c928 100644 --- a/docs/generators.md +++ b/docs/generators.md @@ -24,22 +24,27 @@ Raw Markdown Files [web] - Generate HTML/CSS/JS bundles ``` -Each generator declares its dependency using the `dependsOn` field, allowing automatic pipeline construction. +Each generator declares its dependency using the `dependsOn` export, allowing automatic pipeline construction. ## Generator Structure -A generator is defined as a module exporting an object conforming to the `GeneratorMetadata` interface. +A generator is a single module (`index.mjs`) that exports its metadata and logic as named exports: + +- `name` - The generator's short name (used for config keys and logging) +- `generate` - The main generation function (required) +- `processChunk` - Worker thread processing function (optional — presence enables parallel processing) +- `dependsOn` - Import specifier of the dependency generator (optional) +- `defaultConfiguration` - Default config values (optional) ## Creating a Basic Generator -### Step 1: Create the Generator Files +### Step 1: Create the Generator Directory Create a new directory in `src/generators/`: ``` src/generators/my-format/ -├── index.mjs # Generator metadata (required) -├── generate.mjs # Generator implementation (required) +├── index.mjs # Generator entry point (required) ├── constants.mjs # Constants (optional) ├── types.d.ts # TypeScript types (required) └── utils/ # Utility functions (optional) @@ -67,51 +72,25 @@ export type Generator = GeneratorMetadata< >; ``` -### Step 3: Define Generator Metadata +### Step 3: Implement the Generator -Create the generator metadata in `index.mjs` using `createLazyGenerator`: +Create `index.mjs` with your generator's metadata and logic: ```javascript // src/generators/my-format/index.mjs -import { createLazyGenerator } from '../../utils/generators.mjs'; - -/** - * Generates output in MyFormat. - * - * @type {import('./types').Generator} - */ -export default createLazyGenerator({ - name: 'my-format', - - version: '1.0.0', - - description: 'Generates documentation in MyFormat', - - // This generator depends on the metadata generator - dependsOn: 'metadata', - - defaultConfiguration: { - // If your generator supports a custom configuration, define the defaults here - myCustomOption: 'myDefaultValue', +'use strict'; - // All generators support options in the GlobalConfiguration object - // To override the defaults, they can be specified here - ref: 'overriddenRef', - }, -}); -``` - -### Step 4: Implement the Generator Logic - -Create the generator implementation in `generate.mjs`: - -```javascript -// src/generators/my-format/generate.mjs import { writeFile } from 'node:fs/promises'; import { join } from 'node:path'; import getConfig from '../../utils/configuration/index.mjs'; +export const name = 'my-format'; +export const dependsOn = '@node-core/doc-kit/generators/metadata'; +export const defaultConfiguration = { + myCustomOption: 'myDefaultValue', +}; + /** * Main generation function * @@ -149,63 +128,23 @@ function transformToMyFormat(entries, version) { } ``` -### Step 5: Register the Generator +### Step 4: Register the Generator -Add your generator to the exports in `src/generators/index.mjs`: - -```javascript -// For public generators (available via CLI) -import myFormat from './my-format/index.mjs'; - -export const publicGenerators = { - 'json-simple': jsonSimple, - 'my-format': myFormat, // Add this - // ... other generators -}; - -// For internal generators (used only as dependencies) -const internalGenerators = { - ast, - metadata, - // ... internal generators -}; -``` +Add an entry to the `exports` map in `packages/core/package.json`. If you follow the `index.mjs` convention, the wildcard pattern `"./generators/*": "./src/generators/*/index.mjs"` handles this automatically — no changes needed. ## Parallel Processing with Workers -For generators processing large datasets, implement parallel processing using worker threads. +For generators processing large datasets, implement parallel processing using worker threads. Export a `processChunk` function from your `index.mjs` — its presence automatically enables parallel processing. ### Implementing Worker-Based Processing -First, define the generator metadata in `index.mjs`: - ```javascript // src/generators/parallel-generator/index.mjs -import { createLazyGenerator } from '../../utils/generators.mjs'; - -/** - * @type {import('./types').Generator} - */ -export default createLazyGenerator({ - name: 'parallel-generator', - - version: '1.0.0', - - description: 'Processes data in parallel', - - dependsOn: 'metadata', - - // Indicates this generator has a processChunk implementation - hasParallelProcessor: true, -}); -``` - -Then, implement both `processChunk` and `generate` in `generate.mjs`: - -```javascript -// src/generators/parallel-generator/generate.mjs import getConfig from '../../utils/configuration/index.mjs'; +export const name = 'parallel-generator'; +export const dependsOn = '@node-core/doc-kit/generators/metadata'; + /** * Process a chunk of items in a worker thread. * This function runs in isolated worker threads. @@ -232,7 +171,7 @@ export async function processChunk(fullInput, itemIndices, deps) { */ export async function* generate(input, worker) { // Configuration for this generator is based on its name - const config = getConfig('my-format'); + const config = getConfig('parallel-generator'); // Prepare serializable dependencies const deps = { @@ -271,34 +210,13 @@ Don't use workers when: ## Streaming Results -Generators can yield results as they're produced using async generators. - -Define the generator metadata in `index.mjs`: +Generators can yield results as they're produced using async generators. Export `processChunk` to enable parallel processing, then use `async function*` for `generate`: ```javascript // src/generators/streaming-generator/index.mjs -import { createLazyGenerator } from '../../utils/generators.mjs'; +export const name = 'streaming-generator'; +export const dependsOn = '@node-core/doc-kit/generators/metadata'; -/** - * @type {import('./types').Generator} - */ -export default createLazyGenerator({ - name: 'streaming-generator', - - version: '1.0.0', - - description: 'Streams results as they are ready', - - dependsOn: 'metadata', - - hasParallelProcessor: true, -}); -``` - -Implement the generator in `generate.mjs`: - -```javascript -// src/generators/streaming-generator/generate.mjs /** * Process a chunk of data * @@ -331,32 +249,13 @@ export async function* generate(input, worker) { ### Non-Streaming Generators -Some generators must collect all input before processing. - -Generator metadata in `index.mjs`: +Some generators must collect all input before processing: ```javascript // src/generators/batch-generator/index.mjs -import { createLazyGenerator } from '../../utils/generators.mjs'; +export const name = 'batch-generator'; +export const dependsOn = '@node-core/doc-kit/generators/jsx-ast'; -/** - * @type {import('./types').Generator} - */ -export default createLazyGenerator({ - name: 'batch-generator', - - version: '1.0.0', - - description: 'Requires all input at once', - - dependsOn: 'jsx-ast', -}); -``` - -Implementation in `generate.mjs`: - -```javascript -// src/generators/batch-generator/generate.mjs /** * Non-streaming - returns Promise instead of AsyncGenerator * @@ -383,54 +282,33 @@ Use non-streaming when: ### Declaring Dependencies -In `index.mjs`: - ```javascript -import { createLazyGenerator } from '../../utils/generators.mjs'; - -export default createLazyGenerator({ - name: 'my-generator', - - dependsOn: 'metadata', // This generator requires metadata output - - // ... other metadata -}); -``` - -In `generate.mjs`: +// src/generators/my-generator/index.mjs +export const name = 'my-generator'; +export const dependsOn = '@node-core/doc-kit/generators/metadata'; -```javascript export async function generate(input, worker) { - // input contains the output from 'metadata' generator + // input contains the output from the metadata generator } ``` ### Dependency Chain Example ```javascript -// Step 1: Parse markdown to AST +// Step 1: Parse markdown to AST (no dependency) // src/generators/ast/index.mjs -export default createLazyGenerator({ - name: 'ast', - dependsOn: undefined, // No dependency - // Processes raw markdown files -}); +export const name = 'ast'; +// No dependsOn — processes raw markdown files // Step 2: Extract metadata from AST // src/generators/metadata/index.mjs -export default createLazyGenerator({ - name: 'metadata', - dependsOn: 'ast', // Depends on AST - // Processes AST output -}); +export const name = 'metadata'; +export const dependsOn = '@node-core/doc-kit/generators/ast'; // Step 3: Generate HTML from metadata // src/generators/html-generator/index.mjs -export default createLazyGenerator({ - name: 'html-generator', - dependsOn: 'metadata', // Depends on metadata - // Processes metadata output -}); +export const name = 'html-generator'; +export const dependsOn = '@node-core/doc-kit/generators/metadata'; ``` ### Multiple Consumers @@ -449,8 +327,6 @@ The framework ensures `metadata` runs once and its output is cached for all cons ### Writing Output Files -In `generate.mjs`: - ```javascript import { mkdir, writeFile } from 'node:fs/promises'; import { join } from 'node:path'; diff --git a/package-lock.json b/package-lock.json index 1871058c..2b4014ee 100644 --- a/package-lock.json +++ b/package-lock.json @@ -8791,7 +8791,7 @@ "yaml": "^2.8.3" }, "bin": { - "doc-kit": "bin/cli.mjs" + "doc-kit": "packages/core/bin/cli.mjs" }, "devDependencies": { "@reporters/github": "^1.12.0", diff --git a/packages/core/bin/commands/generate.mjs b/packages/core/bin/commands/generate.mjs index d4586663..d15bcbbb 100644 --- a/packages/core/bin/commands/generate.mjs +++ b/packages/core/bin/commands/generate.mjs @@ -1,12 +1,10 @@ import { Command, Option } from 'commander'; -import { publicGenerators } from '../../src/generators/index.mjs'; import createGenerator from '../../src/generators.mjs'; +import { resolveGeneratorGraph } from '../../src/loader.mjs'; import { setConfig } from '../../src/utils/configuration/index.mjs'; import { errorWrap } from '../utils.mjs'; -const { runGenerators } = createGenerator(); - /** * @typedef {Object} CLIOptions * @property {string[]} input @@ -31,11 +29,7 @@ export default new Command('generate') .addOption( new Option('-i, --input ', 'Input file patterns (glob)') ) - .addOption( - new Option('-t, --target ', 'Target generator(s)').choices( - Object.keys(publicGenerators) - ) - ) + .addOption(new Option('-t, --target ', 'Target generator(s)')) .addOption( new Option('--ignore ', 'Ignore file patterns (glob)') ) @@ -61,7 +55,12 @@ export default new Command('generate') .action( errorWrap(async opts => { - const config = await setConfig(opts); - await runGenerators(config); + const targets = opts.target ?? []; + const loadedGenerators = await resolveGeneratorGraph(targets); + + const config = await setConfig(opts, loadedGenerators); + + const { runGenerators } = createGenerator(loadedGenerators); + await runGenerators(config, targets); }) ); diff --git a/packages/core/package.json b/packages/core/package.json index c8358237..7f08821e 100644 --- a/packages/core/package.json +++ b/packages/core/package.json @@ -15,12 +15,15 @@ "test:ci": "c8 --reporter=lcov node --test --experimental-test-module-mocks \"src/**/*.test.mjs\" --test-reporter=@reporters/github --test-reporter-destination=stdout --test-reporter=junit --test-reporter-destination=junit.xml --test-reporter=spec --test-reporter-destination=stdout", "test:update-snapshots": "node --test --experimental-test-module-mocks --test-update-snapshots \"src/**/*.test.mjs\"", "test:watch": "node --test --experimental-test-module-mocks --watch \"src/**/*.test.mjs\"", - "run": "node bin/cli.mjs", - "watch": "node --watch bin/cli.mjs" + "run": "node packages/core/bin/cli.mjs", + "watch": "node --watch packages/core/bin/cli.mjs" }, "main": "./src/generators.mjs", + "exports": { + "./generators/*": "./src/generators/*/index.mjs" + }, "bin": { - "doc-kit": "./bin/cli.mjs" + "doc-kit": "./packages/core/bin/cli.mjs" }, "dependencies": { "@actions/core": "^3.0.0", diff --git a/packages/core/src/generators.mjs b/packages/core/src/generators.mjs index 451c39b5..c3cc7b2c 100644 --- a/packages/core/src/generators.mjs +++ b/packages/core/src/generators.mjs @@ -1,6 +1,5 @@ 'use strict'; -import { allGenerators } from './generators/index.mjs'; import logger from './logger/index.mjs'; import { isAsyncGenerator, createStreamingCache } from './streaming.mjs'; import createWorkerPool from './threading/index.mjs'; @@ -12,8 +11,10 @@ const generatorsLogger = logger.child('generators'); * Creates a generator orchestration system that manages the execution of * documentation generators in dependency order, with support for parallel * processing and streaming results. + * + * @param {Map} loadedGenerators - Map of specifier → loaded generator */ -const createGenerator = () => { +const createGenerator = loadedGenerators => { /** @type {{ [key: string]: Promise | AsyncGenerator }} */ const cachedGenerators = {}; @@ -25,7 +26,7 @@ const createGenerator = () => { /** * Gets the collected input from a dependency generator. * - * @param {string | undefined} dependsOn - Dependency generator name + * @param {string | undefined} dependsOn - Dependency generator specifier * @returns {Promise} */ const getDependencyInput = async dependsOn => { @@ -45,36 +46,36 @@ const createGenerator = () => { /** * Schedules a generator and its dependencies for execution. * - * @param {string} generatorName - Generator to schedule - * @param {import('./utils/configuration/types').Configuration} configuration - Runtime options + * @param {string} specifier - Generator specifier to schedule + * @param {object} configuration - Runtime options */ - const scheduleGenerator = async (generatorName, configuration) => { - if (generatorName in cachedGenerators) { + const scheduleGenerator = async (specifier, configuration) => { + if (specifier in cachedGenerators) { return; } - const { dependsOn, generate, hasParallelProcessor } = - allGenerators[generatorName]; + const generator = loadedGenerators.get(specifier); + const { dependsOn, generate, processChunk, name } = generator; // Schedule dependency first if (dependsOn && !(dependsOn in cachedGenerators)) { await scheduleGenerator(dependsOn, configuration); } - generatorsLogger.debug(`Scheduling "${generatorName}"`, { + generatorsLogger.debug(`Scheduling "${name}"`, { dependsOn: dependsOn || 'none', - streaming: hasParallelProcessor, + streaming: !!processChunk, }); // Schedule the generator - cachedGenerators[generatorName] = (async () => { + cachedGenerators[specifier] = (async () => { const dependencyInput = await getDependencyInput(dependsOn); - generatorsLogger.debug(`Starting "${generatorName}"`); + generatorsLogger.debug(`Starting "${name}"`); // Create parallel worker for streaming generators - const worker = hasParallelProcessor - ? createParallelWorker(generatorName, pool, configuration) + const worker = processChunk + ? createParallelWorker(specifier, generator, pool, configuration) : Promise.resolve(null); const result = await generate(dependencyInput, await worker); @@ -82,7 +83,7 @@ const createGenerator = () => { // For streaming generators, "Completed" is logged when collection finishes // (in streamingCache.getOrCollect), not here when the generator returns if (!isAsyncGenerator(result)) { - generatorsLogger.debug(`Completed "${generatorName}"`); + generatorsLogger.debug(`Completed "${name}"`); } return result; @@ -92,14 +93,17 @@ const createGenerator = () => { /** * Runs all requested generators with their dependencies. * - * @param {import('./utils/configuration/types').Configuration} options - Runtime options + * @param {object} configuration - Runtime options + * @param {string[]} targetSpecifiers - Resolved target specifiers * @returns {Promise} Results of all requested generators */ - const runGenerators = async configuration => { - const { target: generators, threads } = configuration; + const runGenerators = async (configuration, targetSpecifiers) => { + const { threads } = configuration; generatorsLogger.debug(`Starting pipeline`, { - generators: generators.join(', '), + generators: targetSpecifiers + .map(s => loadedGenerators.get(s).name) + .join(', '), threads, }); @@ -107,16 +111,16 @@ const createGenerator = () => { pool = createWorkerPool(threads); // Schedule all generators - for (const name of generators) { - await scheduleGenerator(name, configuration); + for (const specifier of targetSpecifiers) { + await scheduleGenerator(specifier, configuration); } // Start all collections in parallel (don't await sequentially) - const resultPromises = generators.map(async name => { - let result = await cachedGenerators[name]; + const resultPromises = targetSpecifiers.map(async specifier => { + let result = await cachedGenerators[specifier]; if (isAsyncGenerator(result)) { - result = await streamingCache.getOrCollect(name, result); + result = await streamingCache.getOrCollect(specifier, result); } return result; diff --git a/packages/core/src/generators/__tests__/index.test.mjs b/packages/core/src/generators/__tests__/index.test.mjs deleted file mode 100644 index 8efd3c9a..00000000 --- a/packages/core/src/generators/__tests__/index.test.mjs +++ /dev/null @@ -1,53 +0,0 @@ -import assert from 'node:assert/strict'; -import { describe, it } from 'node:test'; - -import semver from 'semver'; - -import { allGenerators } from '../index.mjs'; - -const validDependencies = Object.keys(allGenerators); - -const allGeneratorsEntries = Object.entries(allGenerators); - -describe('All Generators', () => { - it('should have keys matching their name property', () => { - allGeneratorsEntries.forEach(([key, generator]) => { - assert.equal( - key, - generator.name, - `Generator key "${key}" does not match its name property "${generator.name}"` - ); - }); - }); - - it('should have valid semver versions', () => { - allGeneratorsEntries.forEach(([key, generator]) => { - const isValid = semver.valid(generator.version); - assert.ok( - isValid, - `Generator "${key}" has invalid semver version: "${generator.version}"` - ); - }); - }); - - it('should have valid dependsOn references', () => { - allGeneratorsEntries.forEach(([key, generator]) => { - if (generator.dependsOn) { - assert.ok( - validDependencies.includes(generator.dependsOn), - `Generator "${key}" depends on "${generator.dependsOn}" which is not a valid generator` - ); - } - }); - }); - - it('should have ast generator as a top-level generator with no dependencies', () => { - const ast = allGenerators.ast; - assert.ok(ast, 'ast generator should exist'); - assert.equal( - ast.dependsOn, - undefined, - 'ast generator should have no dependencies' - ); - }); -}); diff --git a/packages/core/src/generators/addon-verify/generate.mjs b/packages/core/src/generators/addon-verify/generate.mjs deleted file mode 100644 index 91f9b463..00000000 --- a/packages/core/src/generators/addon-verify/generate.mjs +++ /dev/null @@ -1,82 +0,0 @@ -'use strict'; - -import { mkdir } from 'node:fs/promises'; -import { join } from 'node:path'; - -import { visit } from 'unist-util-visit'; - -import { EXTRACT_CODE_FILENAME_COMMENT } from './constants.mjs'; -import { generateFileList } from './utils/generateFileList.mjs'; -import { - generateSectionFolderName, - isBuildableSection, - normalizeSectionName, -} from './utils/section.mjs'; -import getConfig from '../../utils/configuration/index.mjs'; -import { writeFile } from '../../utils/file.mjs'; - -/** - * Generates a file list from code blocks. - * - * @type {import('./types').Generator['generate']} - */ -export async function generate(input) { - const config = getConfig('addon-verify'); - - const sectionsCodeBlocks = input.reduce((addons, node) => { - const sectionName = node.heading.data.name; - - const content = node.content; - - visit(content, childNode => { - if (childNode.type === 'code') { - const filename = childNode.value.match(EXTRACT_CODE_FILENAME_COMMENT); - - if (filename === null) { - return; - } - - if (!addons[sectionName]) { - addons[sectionName] = []; - } - - addons[sectionName].push({ - name: filename[1], - content: childNode.value, - }); - } - }); - - return addons; - }, {}); - - const files = await Promise.all( - Object.entries(sectionsCodeBlocks) - .filter(([, codeBlocks]) => isBuildableSection(codeBlocks)) - .flatMap(async ([sectionName, codeBlocks], index) => { - const files = generateFileList(codeBlocks); - - if (config.output) { - const normalizedSectionName = normalizeSectionName(sectionName); - - const folderName = generateSectionFolderName( - normalizedSectionName, - index - ); - - await mkdir(join(config.output, folderName), { recursive: true }); - - for (const file of files) { - await writeFile( - join(config.output, folderName, file.name), - file.content - ); - } - } - - return files; - }) - ); - - return files; -} diff --git a/packages/core/src/generators/addon-verify/index.mjs b/packages/core/src/generators/addon-verify/index.mjs index fab78599..8b888999 100644 --- a/packages/core/src/generators/addon-verify/index.mjs +++ b/packages/core/src/generators/addon-verify/index.mjs @@ -1,21 +1,84 @@ 'use strict'; -import { createLazyGenerator } from '../../utils/generators.mjs'; +import { mkdir } from 'node:fs/promises'; +import { join } from 'node:path'; +import { visit } from 'unist-util-visit'; + +import { EXTRACT_CODE_FILENAME_COMMENT } from './constants.mjs'; +import { generateFileList } from './utils/generateFileList.mjs'; +import { + generateSectionFolderName, + isBuildableSection, + normalizeSectionName, +} from './utils/section.mjs'; +import getConfig from '../../utils/configuration/index.mjs'; +import { writeFile } from '../../utils/file.mjs'; + +export const name = 'addon-verify'; +export const dependsOn = '@node-core/doc-kit/generators/metadata'; /** - * This generator generates a file list from code blocks extracted from - * `doc/api/addons.md` to facilitate C++ compilation and JavaScript runtime - * validations. + * Generates a file list from code blocks. * - * @type {import('./types').Generator} + * @type {import('./types').Generator['generate']} */ -export default await createLazyGenerator({ - name: 'addon-verify', +export async function generate(input) { + const config = getConfig('addon-verify'); + + const sectionsCodeBlocks = input.reduce((addons, node) => { + const sectionName = node.heading.data.name; + + const content = node.content; + + visit(content, childNode => { + if (childNode.type === 'code') { + const filename = childNode.value.match(EXTRACT_CODE_FILENAME_COMMENT); + + if (filename === null) { + return; + } + + if (!addons[sectionName]) { + addons[sectionName] = []; + } + + addons[sectionName].push({ + name: filename[1], + content: childNode.value, + }); + } + }); + + return addons; + }, {}); + + const files = await Promise.all( + Object.entries(sectionsCodeBlocks) + .filter(([, codeBlocks]) => isBuildableSection(codeBlocks)) + .flatMap(async ([sectionName, codeBlocks], index) => { + const files = generateFileList(codeBlocks); + + if (config.output) { + const normalizedSectionName = normalizeSectionName(sectionName); + + const folderName = generateSectionFolderName( + normalizedSectionName, + index + ); + + await mkdir(join(config.output, folderName), { recursive: true }); - version: '1.0.0', + for (const file of files) { + await writeFile( + join(config.output, folderName, file.name), + file.content + ); + } + } - description: - 'Generates a file list from code blocks extracted from `doc/api/addons.md` to facilitate C++ compilation and JavaScript runtime validations', + return files; + }) + ); - dependsOn: 'metadata', -}); + return files; +} diff --git a/packages/core/src/generators/api-links/__tests__/fixtures.test.mjs b/packages/core/src/generators/api-links/__tests__/fixtures.test.mjs index 6259aeb3..669e3dac 100644 --- a/packages/core/src/generators/api-links/__tests__/fixtures.test.mjs +++ b/packages/core/src/generators/api-links/__tests__/fixtures.test.mjs @@ -3,11 +3,12 @@ import { after, before, describe, it } from 'node:test'; import { globSync } from 'tinyglobby'; +import { loadGenerator } from '../../../loader.mjs'; import createWorkerPool from '../../../threading/index.mjs'; import createParallelWorker from '../../../threading/parallel.mjs'; import { setConfig } from '../../../utils/configuration/index.mjs'; -import { generate as astJsGenerate } from '../../ast-js/generate.mjs'; -import { generate as apiLinksGenerate } from '../generate.mjs'; +import { generate as astJsGenerate } from '../../ast-js/index.mjs'; +import { generate as apiLinksGenerate } from '../index.mjs'; const relativePath = relative(process.cwd(), import.meta.dirname); @@ -15,7 +16,17 @@ const sourceFiles = globSync('*.js', { cwd: new URL(import.meta.resolve('./fixtures')), }); -const config = await setConfig({}); +const astJsSpecifier = '@node-core/doc-kit/generators/ast-js'; +const astJsGenerator = await loadGenerator(astJsSpecifier); +const apiLinksSpecifier = '@node-core/doc-kit/generators/api-links'; +const apiLinksGenerator = await loadGenerator(apiLinksSpecifier); + +const loadedGenerators = new Map([ + [astJsSpecifier, astJsGenerator], + [apiLinksSpecifier, apiLinksGenerator], +]); + +const config = await setConfig({}, loadedGenerators); describe('api links', () => { let pool; @@ -35,7 +46,12 @@ describe('api links', () => { join(relativePath, 'fixtures', sourceFile).replaceAll(sep, '/'), ]; - const worker = await createParallelWorker('ast-js', pool, config); + const worker = await createParallelWorker( + astJsSpecifier, + astJsGenerator, + pool, + config + ); // Collect results from the async generator const astJsResults = []; diff --git a/packages/core/src/generators/api-links/generate.mjs b/packages/core/src/generators/api-links/generate.mjs deleted file mode 100644 index 3890a308..00000000 --- a/packages/core/src/generators/api-links/generate.mjs +++ /dev/null @@ -1,67 +0,0 @@ -'use strict'; - -import { basename, join } from 'node:path'; - -import { checkIndirectReferences } from './utils/checkIndirectReferences.mjs'; -import { extractExports } from './utils/extractExports.mjs'; -import { findDefinitions } from './utils/findDefinitions.mjs'; -import getConfig from '../../utils/configuration/index.mjs'; -import { populate } from '../../utils/configuration/templates.mjs'; -import { withExt, writeFile } from '../../utils/file.mjs'; - -/** - * Generates the `apilinks.json` file. - * - * @type {import('./types').Generator['generate']} - */ -export async function generate(input) { - const config = getConfig('api-links'); - /** - * @type Record - */ - const definitions = {}; - - input.forEach(program => { - /** - * Mapping of definitions to their line number - * - * @type {Record} - * @example { 'someclass.foo': 10 } - */ - const nameToLineNumberMap = {}; - - const fileName = basename(program.path); - const baseName = withExt(fileName); - - const exports = extractExports(program, baseName, nameToLineNumberMap); - - findDefinitions(program, baseName, nameToLineNumberMap, exports); - - checkIndirectReferences(program, exports, nameToLineNumberMap); - - const fullGitUrl = populate(config.sourceURL, { - ...config, - fileName, - }); - - // Add the exports we found in this program to our output - Object.keys(nameToLineNumberMap).forEach(key => { - const lineNumber = nameToLineNumberMap[key]; - - definitions[key] = `${fullGitUrl}#L${lineNumber}`; - }); - }); - - if (config.output) { - const out = join(config.output, 'apilinks.json'); - - await writeFile( - out, - config.minify - ? JSON.stringify(definitions) - : JSON.stringify(definitions, null, 2) - ); - } - - return definitions; -} diff --git a/packages/core/src/generators/api-links/index.mjs b/packages/core/src/generators/api-links/index.mjs index dca3f908..e967b76d 100644 --- a/packages/core/src/generators/api-links/index.mjs +++ b/packages/core/src/generators/api-links/index.mjs @@ -1,31 +1,76 @@ 'use strict'; -import { GITHUB_BLOB_URL } from '../../utils/configuration/templates.mjs'; -import { createLazyGenerator } from '../../utils/generators.mjs'; +import { basename, join } from 'node:path'; + +import { checkIndirectReferences } from './utils/checkIndirectReferences.mjs'; +import { extractExports } from './utils/extractExports.mjs'; +import { findDefinitions } from './utils/findDefinitions.mjs'; +import getConfig from '../../utils/configuration/index.mjs'; +import { + GITHUB_BLOB_URL, + populate, +} from '../../utils/configuration/templates.mjs'; +import { withExt, writeFile } from '../../utils/file.mjs'; + +export const name = 'api-links'; +export const dependsOn = '@node-core/doc-kit/generators/ast-js'; +export const defaultConfiguration = { + sourceURL: `${GITHUB_BLOB_URL}lib/{fileName}`, +}; /** - * This generator is responsible for mapping publicly accessible functions in - * Node.js to their source locations in the Node.js repository. - * - * This is a top-level generator. It takes in the raw AST tree of the JavaScript - * source files. It outputs a `apilinks.json` file into the specified output - * directory. + * Generates the `apilinks.json` file. * - * @type {import('./types').Generator} + * @type {import('./types').Generator['generate']} */ -export default createLazyGenerator({ - name: 'api-links', +export async function generate(input) { + const config = getConfig('api-links'); + /** + * @type Record + */ + const definitions = {}; + + input.forEach(program => { + /** + * Mapping of definitions to their line number + * + * @type {Record} + * @example { 'someclass.foo': 10 } + */ + const nameToLineNumberMap = {}; + + const fileName = basename(program.path); + const baseName = withExt(fileName); + + const exports = extractExports(program, baseName, nameToLineNumberMap); + + findDefinitions(program, baseName, nameToLineNumberMap, exports); + + checkIndirectReferences(program, exports, nameToLineNumberMap); + + const fullGitUrl = populate(config.sourceURL, { + ...config, + fileName, + }); + + // Add the exports we found in this program to our output + Object.keys(nameToLineNumberMap).forEach(key => { + const lineNumber = nameToLineNumberMap[key]; - version: '1.0.0', + definitions[key] = `${fullGitUrl}#L${lineNumber}`; + }); + }); - description: - 'Creates a mapping of publicly accessible functions to their source locations in the Node.js repository.', + if (config.output) { + const out = join(config.output, 'apilinks.json'); - // Unlike the rest of the generators, this utilizes Javascript sources being - // passed into the input field rather than Markdown. - dependsOn: 'ast-js', + await writeFile( + out, + config.minify + ? JSON.stringify(definitions) + : JSON.stringify(definitions, null, 2) + ); + } - defaultConfiguration: { - sourceURL: `${GITHUB_BLOB_URL}lib/{fileName}`, - }, -}); + return definitions; +} diff --git a/packages/core/src/generators/ast-js/generate.mjs b/packages/core/src/generators/ast-js/generate.mjs deleted file mode 100644 index 303e5ec0..00000000 --- a/packages/core/src/generators/ast-js/generate.mjs +++ /dev/null @@ -1,56 +0,0 @@ -'use strict'; - -import { readFile } from 'node:fs/promises'; -import { extname } from 'node:path'; - -import { parse } from 'acorn'; -import { globSync } from 'tinyglobby'; - -import getConfig from '../../utils/configuration/index.mjs'; - -/** - * Process a chunk of JavaScript files in a worker thread. - * Parses JS source files into AST representations. - * - * @type {import('./types').Generator['processChunk']} - */ -export async function processChunk(inputSlice, itemIndices) { - const filePaths = itemIndices.map(idx => inputSlice[idx]); - - const results = []; - - for (const path of filePaths) { - const value = await readFile(path, 'utf-8'); - - const parsed = parse(value, { - allowReturnOutsideFunction: true, - ecmaVersion: 'latest', - locations: true, - }); - - parsed.path = path; - - results.push(parsed); - } - - return results; -} - -/** - * Generates a JavaScript AST from the input files. - * - * @type {import('./types').Generator['generate']} - */ -export async function* generate(_, worker) { - const config = getConfig('ast-js'); - - const files = globSync(config.input, { ignore: config.ignore }).filter( - p => extname(p) === '.js' - ); - - // Parse the Javascript sources into ASTs in parallel using worker threads - // source is both the items list and the fullInput since we use sliceInput - for await (const chunkResult of worker.stream(files)) { - yield chunkResult; - } -} diff --git a/packages/core/src/generators/ast-js/index.mjs b/packages/core/src/generators/ast-js/index.mjs index 06652167..3e666101 100644 --- a/packages/core/src/generators/ast-js/index.mjs +++ b/packages/core/src/generators/ast-js/index.mjs @@ -1,23 +1,58 @@ 'use strict'; -import { createLazyGenerator } from '../../utils/generators.mjs'; +import { readFile } from 'node:fs/promises'; +import { extname } from 'node:path'; + +import { parse } from 'acorn'; +import { globSync } from 'tinyglobby'; + +import getConfig from '../../utils/configuration/index.mjs'; + +export const name = 'ast-js'; /** - * This generator parses Javascript sources passed into the generator's input - * field. This is separate from the Markdown parsing step since it's not as - * commonly used and can take up a significant amount of memory. - * - * Putting this with the rest of the generators allows it to be lazily loaded - * so we're only parsing the Javascript sources when we need to. + * Process a chunk of JavaScript files in a worker thread. + * Parses JS source files into AST representations. * - * @type {import('./types').Generator} + * @type {import('./types').Generator['processChunk']} */ -export default createLazyGenerator({ - name: 'ast-js', +export async function processChunk(inputSlice, itemIndices) { + const filePaths = itemIndices.map(idx => inputSlice[idx]); + + const results = []; + + for (const path of filePaths) { + const value = await readFile(path, 'utf-8'); + + const parsed = parse(value, { + allowReturnOutsideFunction: true, + ecmaVersion: 'latest', + locations: true, + }); - version: '1.0.0', + parsed.path = path; + + results.push(parsed); + } + + return results; +} + +/** + * Generates a JavaScript AST from the input files. + * + * @type {import('./types').Generator['generate']} + */ +export async function* generate(_, worker) { + const config = getConfig('ast-js'); - description: 'Parses Javascript source files passed into the input.', + const files = globSync(config.input, { ignore: config.ignore }).filter( + p => extname(p) === '.js' + ); - hasParallelProcessor: true, -}); + // Parse the Javascript sources into ASTs in parallel using worker threads + // source is both the items list and the fullInput since we use sliceInput + for await (const chunkResult of worker.stream(files)) { + yield chunkResult; + } +} diff --git a/packages/core/src/generators/ast/generate.mjs b/packages/core/src/generators/ast/generate.mjs deleted file mode 100644 index 97724e03..00000000 --- a/packages/core/src/generators/ast/generate.mjs +++ /dev/null @@ -1,73 +0,0 @@ -'use strict'; - -import { readFile } from 'node:fs/promises'; -import { relative, sep } from 'node:path/posix'; - -import globParent from 'glob-parent'; -import { globSync } from 'tinyglobby'; - -import { STABILITY_INDEX_URL } from './constants.mjs'; -import getConfig from '../../utils/configuration/index.mjs'; -import { withExt } from '../../utils/file.mjs'; -import { QUERIES } from '../../utils/queries/index.mjs'; -import { getRemark } from '../../utils/remark.mjs'; - -const remarkProcessor = getRemark(); - -/** - * Process a chunk of markdown files in a worker thread. - * Loads and parses markdown files into AST representations. - * - * @type {import('./types').Generator['processChunk']} - */ -export async function processChunk(inputSlice, itemIndices) { - const filePaths = itemIndices.map(idx => inputSlice[idx]); - - const results = []; - - for (const [path, parent] of filePaths) { - const content = await readFile(path, 'utf-8'); - const value = content - .replace( - QUERIES.standardYamlFrontmatter, - (_, yaml) => '\n' - ) - .replace( - QUERIES.stabilityIndexPrefix, - match => `[${match}](${STABILITY_INDEX_URL})` - ); - - const relativePath = sep + withExt(relative(parent, path)); - - results.push({ - tree: remarkProcessor.parse(value), - // The path is the relative path minus the extension - path: relativePath, - }); - } - - return results; -} - -/** - * Generates AST trees from markdown input files. - * - * @type {import('./types').Generator['generate']} - */ -export async function* generate(_, worker) { - const { ast: config } = getConfig(); - - const files = config.input.flatMap(input => { - const parent = globParent(input); - - return globSync(input, { ignore: config.ignore }).map(child => [ - child, - parent, - ]); - }); - - // Parse markdown files in parallel using worker threads - for await (const chunkResult of worker.stream(files)) { - yield chunkResult; - } -} diff --git a/packages/core/src/generators/ast/index.mjs b/packages/core/src/generators/ast/index.mjs index 4c34b637..46ef0e50 100644 --- a/packages/core/src/generators/ast/index.mjs +++ b/packages/core/src/generators/ast/index.mjs @@ -1,19 +1,75 @@ 'use strict'; -import { createLazyGenerator } from '../../utils/generators.mjs'; +import { readFile } from 'node:fs/promises'; +import { relative, sep } from 'node:path/posix'; + +import globParent from 'glob-parent'; +import { globSync } from 'tinyglobby'; + +import { STABILITY_INDEX_URL } from './constants.mjs'; +import getConfig from '../../utils/configuration/index.mjs'; +import { withExt } from '../../utils/file.mjs'; +import { QUERIES } from '../../utils/queries/index.mjs'; +import { getRemark } from '../../utils/remark.mjs'; + +export const name = 'ast'; + +const remarkProcessor = getRemark(); + +/** + * Process a chunk of markdown files in a worker thread. + * Loads and parses markdown files into AST representations. + * + * @type {import('./types').Generator['processChunk']} + */ +export async function processChunk(inputSlice, itemIndices) { + const filePaths = itemIndices.map(idx => inputSlice[idx]); + + const results = []; + + for (const [path, parent] of filePaths) { + const content = await readFile(path, 'utf-8'); + const value = content + .replace( + QUERIES.standardYamlFrontmatter, + (_, yaml) => '\n' + ) + .replace( + QUERIES.stabilityIndexPrefix, + match => `[${match}](${STABILITY_INDEX_URL})` + ); + + const relativePath = sep + withExt(relative(parent, path)); + + results.push({ + tree: remarkProcessor.parse(value), + // The path is the relative path minus the extension + path: relativePath, + }); + } + + return results; +} /** - * This generator parses Markdown API doc files into AST trees. - * It parallelizes the parsing across worker threads for better performance. + * Generates AST trees from markdown input files. * - * @type {import('./types').Generator} + * @type {import('./types').Generator['generate']} */ -export default createLazyGenerator({ - name: 'ast', +export async function* generate(_, worker) { + const { ast: config } = getConfig(); - version: '1.0.0', + const files = config.input.flatMap(input => { + const parent = globParent(input); - description: 'Parses Markdown API doc files into AST trees', + return globSync(input, { ignore: config.ignore }).map(child => [ + child, + parent, + ]); + }); - hasParallelProcessor: true, -}); + // Parse markdown files in parallel using worker threads + for await (const chunkResult of worker.stream(files)) { + yield chunkResult; + } +} diff --git a/packages/core/src/generators/index.mjs b/packages/core/src/generators/index.mjs deleted file mode 100644 index 24294f92..00000000 --- a/packages/core/src/generators/index.mjs +++ /dev/null @@ -1,47 +0,0 @@ -'use strict'; - -import addonVerify from './addon-verify/index.mjs'; -import apiLinks from './api-links/index.mjs'; -import ast from './ast/index.mjs'; -import astJs from './ast-js/index.mjs'; -import jsonSimple from './json-simple/index.mjs'; -import jsxAst from './jsx-ast/index.mjs'; -import legacyHtml from './legacy-html/index.mjs'; -import legacyHtmlAll from './legacy-html-all/index.mjs'; -import legacyJson from './legacy-json/index.mjs'; -import legacyJsonAll from './legacy-json-all/index.mjs'; -import llmsTxt from './llms-txt/index.mjs'; -import manPage from './man-page/index.mjs'; -import metadata from './metadata/index.mjs'; -import oramaDb from './orama-db/index.mjs'; -import sitemap from './sitemap/index.mjs'; -import web from './web/index.mjs'; - -export const publicGenerators = { - 'json-simple': jsonSimple, - 'legacy-html': legacyHtml, - 'legacy-html-all': legacyHtmlAll, - 'man-page': manPage, - 'legacy-json': legacyJson, - 'legacy-json-all': legacyJsonAll, - 'addon-verify': addonVerify, - 'api-links': apiLinks, - 'orama-db': oramaDb, - 'llms-txt': llmsTxt, - sitemap, - web, -}; - -// These ones are special since they don't produce standard output, -// and hence, we don't expose them to the CLI. -const internalGenerators = { - ast, - metadata, - 'jsx-ast': jsxAst, - 'ast-js': astJs, -}; - -export const allGenerators = { - ...publicGenerators, - ...internalGenerators, -}; diff --git a/packages/core/src/generators/json-simple/generate.mjs b/packages/core/src/generators/json-simple/generate.mjs deleted file mode 100644 index 82d06a99..00000000 --- a/packages/core/src/generators/json-simple/generate.mjs +++ /dev/null @@ -1,43 +0,0 @@ -'use strict'; - -import { join } from 'node:path'; - -import { remove } from 'unist-util-remove'; - -import getConfig from '../../utils/configuration/index.mjs'; -import { writeFile } from '../../utils/file.mjs'; -import { UNIST } from '../../utils/queries/index.mjs'; - -/** - * Generates the simplified JSON version of the API docs - * - * @type {import('./types').Generator['generate']} - */ -export async function generate(input) { - const config = getConfig('json-simple'); - - // Iterates the input (MetadataEntry) and performs a few changes - const mappedInput = input.map(node => { - // Deep clones the content nodes to avoid affecting upstream nodes - const content = JSON.parse(JSON.stringify(node.content)); - - // Removes numerous nodes from the content that should not be on the "body" - // of the JSON version of the API docs as they are already represented in the metadata - remove(content, [UNIST.isStabilityNode, UNIST.isHeading]); - - return { ...node, content }; - }); - - if (config.output) { - // Writes all the API docs stringified content into one file - // Note: The full JSON generator in the future will create one JSON file per top-level API doc file - await writeFile( - join(config.output, 'api-docs.json'), - config.minify - ? JSON.stringify(mappedInput) - : JSON.stringify(mappedInput, null, 2) - ); - } - - return mappedInput; -} diff --git a/packages/core/src/generators/json-simple/index.mjs b/packages/core/src/generators/json-simple/index.mjs index 9cd62537..41c04ff0 100644 --- a/packages/core/src/generators/json-simple/index.mjs +++ b/packages/core/src/generators/json-simple/index.mjs @@ -1,23 +1,46 @@ 'use strict'; -import { createLazyGenerator } from '../../utils/generators.mjs'; +import { join } from 'node:path'; + +import { remove } from 'unist-util-remove'; + +import getConfig from '../../utils/configuration/index.mjs'; +import { writeFile } from '../../utils/file.mjs'; +import { UNIST } from '../../utils/queries/index.mjs'; + +export const name = 'json-simple'; +export const dependsOn = '@node-core/doc-kit/generators/metadata'; /** - * This generator generates a simplified JSON version of the API docs and returns it as a string - * this is not meant to be used for the final API docs, but for debugging and testing purposes + * Generates the simplified JSON version of the API docs * - * This generator is a top-level generator, and it takes the raw AST tree of the API doc files - * and returns a stringified JSON version of the API docs. - * - * @type {import('./types').Generator} + * @type {import('./types').Generator['generate']} */ -export default createLazyGenerator({ - name: 'json-simple', +export async function generate(input) { + const config = getConfig('json-simple'); + + // Iterates the input (MetadataEntry) and performs a few changes + const mappedInput = input.map(node => { + // Deep clones the content nodes to avoid affecting upstream nodes + const content = JSON.parse(JSON.stringify(node.content)); + + // Removes numerous nodes from the content that should not be on the "body" + // of the JSON version of the API docs as they are already represented in the metadata + remove(content, [UNIST.isStabilityNode, UNIST.isHeading]); - version: '1.0.0', + return { ...node, content }; + }); - description: - 'Generates the simple JSON version of the API docs, and returns it as a string', + if (config.output) { + // Writes all the API docs stringified content into one file + // Note: The full JSON generator in the future will create one JSON file per top-level API doc file + await writeFile( + join(config.output, 'api-docs.json'), + config.minify + ? JSON.stringify(mappedInput) + : JSON.stringify(mappedInput, null, 2) + ); + } - dependsOn: 'metadata', -}); + return mappedInput; +} diff --git a/packages/core/src/generators/jsx-ast/generate.mjs b/packages/core/src/generators/jsx-ast/generate.mjs deleted file mode 100644 index 160971fd..00000000 --- a/packages/core/src/generators/jsx-ast/generate.mjs +++ /dev/null @@ -1,70 +0,0 @@ -import { buildSideBarProps } from './utils/buildBarProps.mjs'; -import buildContent from './utils/buildContent.mjs'; -import { getSortedHeadNodes } from './utils/getSortedHeadNodes.mjs'; -import { groupNodesByModule } from '../../utils/generators.mjs'; -import { getRemarkRecma } from '../../utils/remark.mjs'; -import { relative } from '../../utils/url.mjs'; - -const remarkRecma = getRemarkRecma(); - -/** - * Process a chunk of items in a worker thread. - * Transforms metadata entries into JSX AST nodes. - * - * Each item is a SlicedModuleInput containing the head node - * and all entries for that module - no need to recompute grouping. - * - * @type {import('./types').Generator['processChunk']} - */ -export async function processChunk(slicedInput, itemIndices, docPages) { - const results = []; - - for (const idx of itemIndices) { - const { head, entries } = slicedInput[idx]; - - const sideBarProps = buildSideBarProps( - head, - docPages.map(([heading, path]) => [ - heading, - head.path === path - ? `${head.basename}.html` - : `${relative(path, head.path)}.html`, - ]) - ); - - const content = await buildContent( - entries, - head, - sideBarProps, - remarkRecma - ); - - results.push(content); - } - - return results; -} - -/** - * Generates a JSX AST - * - * @type {import('./types').Generator['generate']} - */ -export async function* generate(input, worker) { - const groupedModules = groupNodesByModule(input); - - const headNodes = getSortedHeadNodes(input); - - const docPages = headNodes.map(node => [node.heading.data.name, node.path]); - - // Create sliced input: each item contains head + its module's entries - // This avoids sending all 4700+ entries to every worker - const entries = headNodes.map(head => ({ - head, - entries: groupedModules.get(head.api), - })); - - for await (const chunkResult of worker.stream(entries, docPages)) { - yield chunkResult; - } -} diff --git a/packages/core/src/generators/jsx-ast/index.mjs b/packages/core/src/generators/jsx-ast/index.mjs index a83839f4..a11baab6 100644 --- a/packages/core/src/generators/jsx-ast/index.mjs +++ b/packages/core/src/generators/jsx-ast/index.mjs @@ -1,27 +1,79 @@ -'use strict'; - +import { buildSideBarProps } from './utils/buildBarProps.mjs'; +import buildContent from './utils/buildContent.mjs'; +import { getSortedHeadNodes } from './utils/getSortedHeadNodes.mjs'; import { GITHUB_EDIT_URL } from '../../utils/configuration/templates.mjs'; -import { createLazyGenerator } from '../../utils/generators.mjs'; +import { groupNodesByModule } from '../../utils/generators.mjs'; +import { getRemarkRecma } from '../../utils/remark.mjs'; +import { relative } from '../../utils/url.mjs'; + +export const name = 'jsx-ast'; +export const dependsOn = '@node-core/doc-kit/generators/metadata'; +export const defaultConfiguration = { + ref: 'main', + pageURL: '{baseURL}/latest-{version}/api{path}.html', + editURL: `${GITHUB_EDIT_URL}/doc/api{path}.md`, +}; + +const remarkRecma = getRemarkRecma(); /** - * Generator for converting MDAST to JSX AST. + * Process a chunk of items in a worker thread. + * Transforms metadata entries into JSX AST nodes. * - * @type {import('./types').Generator} + * Each item is a SlicedModuleInput containing the head node + * and all entries for that module - no need to recompute grouping. + * + * @type {import('./types').Generator['processChunk']} */ -export default createLazyGenerator({ - name: 'jsx-ast', +export async function processChunk(slicedInput, itemIndices, docPages) { + const results = []; + + for (const idx of itemIndices) { + const { head, entries } = slicedInput[idx]; + + const sideBarProps = buildSideBarProps( + head, + docPages.map(([heading, path]) => [ + heading, + head.path === path + ? `${head.basename}.html` + : `${relative(path, head.path)}.html`, + ]) + ); - version: '1.0.0', + const content = await buildContent( + entries, + head, + sideBarProps, + remarkRecma + ); + + results.push(content); + } + + return results; +} + +/** + * Generates a JSX AST + * + * @type {import('./types').Generator['generate']} + */ +export async function* generate(input, worker) { + const groupedModules = groupNodesByModule(input); - description: 'Generates JSX AST from the input MDAST', + const headNodes = getSortedHeadNodes(input); - dependsOn: 'metadata', + const docPages = headNodes.map(node => [node.heading.data.name, node.path]); - defaultConfiguration: { - ref: 'main', - pageURL: '{baseURL}/latest-{version}/api{path}.html', - editURL: `${GITHUB_EDIT_URL}/doc/api{path}.md`, - }, + // Create sliced input: each item contains head + its module's entries + // This avoids sending all 4700+ entries to every worker + const entries = headNodes.map(head => ({ + head, + entries: groupedModules.get(head.api), + })); - hasParallelProcessor: true, -}); + for await (const chunkResult of worker.stream(entries, docPages)) { + yield chunkResult; + } +} diff --git a/packages/core/src/generators/jsx-ast/utils/__tests__/buildBarProps.test.mjs b/packages/core/src/generators/jsx-ast/utils/__tests__/buildBarProps.test.mjs index e91ba049..3e5db799 100644 --- a/packages/core/src/generators/jsx-ast/utils/__tests__/buildBarProps.test.mjs +++ b/packages/core/src/generators/jsx-ast/utils/__tests__/buildBarProps.test.mjs @@ -3,6 +3,7 @@ import { describe, it, mock } from 'node:test'; import { SemVer } from 'semver'; +import { loadGenerator } from '../../../../loader.mjs'; import { setConfig } from '../../../../utils/configuration/index.mjs'; import * as generatorsExports from '../../../../utils/generators.mjs'; @@ -30,13 +31,19 @@ const { buildSideBarProps, } = await import('../buildBarProps.mjs'); -await setConfig({ - version: 'v17.0.0', - changelog: [ - { version: new SemVer('16.0.0'), isLts: true, isCurrent: false }, - { version: new SemVer('17.0.0'), isLts: false, isCurrent: true }, - ], -}); +const jsxAstSpecifier = '@node-core/doc-kit/generators/jsx-ast'; +const jsxAstGenerator = await loadGenerator(jsxAstSpecifier); + +await setConfig( + { + version: 'v17.0.0', + changelog: [ + { version: new SemVer('16.0.0'), isLts: true, isCurrent: false }, + { version: new SemVer('17.0.0'), isLts: false, isCurrent: true }, + ], + }, + new Map([[jsxAstSpecifier, jsxAstGenerator]]) +); describe('extractTextContent', () => { it('combines text and code node values from entries', () => { diff --git a/packages/core/src/generators/legacy-html-all/generate.mjs b/packages/core/src/generators/legacy-html-all/generate.mjs deleted file mode 100644 index ad4e2ca1..00000000 --- a/packages/core/src/generators/legacy-html-all/generate.mjs +++ /dev/null @@ -1,75 +0,0 @@ -'use strict'; - -import { readFile, writeFile } from 'node:fs/promises'; -import { join } from 'node:path'; - -import getConfig from '../../utils/configuration/index.mjs'; -import { minifyHTML } from '../../utils/html-minifier.mjs'; -import { getRemarkRehype } from '../../utils/remark.mjs'; -import { replaceTemplateValues } from '../legacy-html/utils/replaceTemplateValues.mjs'; -import tableOfContents from '../legacy-html/utils/tableOfContents.mjs'; - -/** - * Generates the `all.html` file from the `legacy-html` generator - * - * @type {import('./types').Generator['generate']} - */ -export async function generate(input) { - const config = getConfig('legacy-html-all'); - - // Gets a Remark Processor that parses Markdown to minified HTML - const remarkWithRehype = getRemarkRehype(); - - // Reads the API template.html file to be used as a base for the HTML files - const apiTemplate = await readFile(config.templatePath, 'utf-8'); - - // Filter out index entries and extract needed properties - const entries = input.filter(entry => entry.api !== 'index'); - - // Aggregates all individual Table of Contents into one giant string - const aggregatedToC = entries.map(entry => entry.toc).join('\n'); - - // Aggregates all individual content into one giant string - const aggregatedContent = entries.map(entry => entry.content).join('\n'); - - // Creates a "mimic" of an `MetadataEntry` which fulfils the requirements - // for generating the `tableOfContents` with the `tableOfContents.parseNavigationNode` parser - const sideNavigationFromValues = entries.map(entry => ({ - api: entry.api, - heading: { data: { depth: 1, name: entry.section } }, - })); - - // Generates the global Table of Contents (Sidebar Navigation) - const parsedSideNav = remarkWithRehype.processSync( - tableOfContents(sideNavigationFromValues, { - maxDepth: 1, - parser: tableOfContents.parseNavigationNode, - }) - ); - - const templateValues = { - api: 'all', - path: 'all', - added: '', - section: 'All', - version: `v${config.version.version}`, - toc: aggregatedToC, - nav: String(parsedSideNav), - content: aggregatedContent, - }; - - let result = replaceTemplateValues(apiTemplate, templateValues, config, { - skipGitHub: true, - skipGtocPicker: true, - }); - - if (config.minify) { - result = await minifyHTML(result); - } - - if (config.output) { - await writeFile(join(config.output, 'all.html'), result); - } - - return result; -} diff --git a/packages/core/src/generators/legacy-html-all/index.mjs b/packages/core/src/generators/legacy-html-all/index.mjs index 2edea628..43527413 100644 --- a/packages/core/src/generators/legacy-html-all/index.mjs +++ b/packages/core/src/generators/legacy-html-all/index.mjs @@ -1,28 +1,82 @@ 'use strict'; -import { createLazyGenerator } from '../../utils/generators.mjs'; -import legacyHtml from '../legacy-html/index.mjs'; +import { readFile, writeFile } from 'node:fs/promises'; +import { join } from 'node:path'; + +import getConfig from '../../utils/configuration/index.mjs'; +import { minifyHTML } from '../../utils/html-minifier.mjs'; +import { getRemarkRehype } from '../../utils/remark.mjs'; +import { defaultConfiguration as legacyHtmlDefaults } from '../legacy-html/index.mjs'; +import { replaceTemplateValues } from '../legacy-html/utils/replaceTemplateValues.mjs'; +import tableOfContents from '../legacy-html/utils/tableOfContents.mjs'; + +export const name = 'legacy-html-all'; +export const dependsOn = '@node-core/doc-kit/generators/legacy-html'; +export const defaultConfiguration = { + templatePath: legacyHtmlDefaults.templatePath, +}; /** - * This generator generates the legacy HTML pages of the legacy API docs - * for retro-compatibility and while we are implementing the new 'react' and 'html' generators. - * - * This generator is a top-level generator, and it takes the raw AST tree of the API doc files - * and generates the HTML files to the specified output directory from the configuration settings + * Generates the `all.html` file from the `legacy-html` generator * - * @type {import('./types').Generator} + * @type {import('./types').Generator['generate']} */ -export default createLazyGenerator({ - name: 'legacy-html-all', +export async function generate(input) { + const config = getConfig('legacy-html-all'); + + // Gets a Remark Processor that parses Markdown to minified HTML + const remarkWithRehype = getRemarkRehype(); + + // Reads the API template.html file to be used as a base for the HTML files + const apiTemplate = await readFile(config.templatePath, 'utf-8'); + + // Filter out index entries and extract needed properties + const entries = input.filter(entry => entry.api !== 'index'); + + // Aggregates all individual Table of Contents into one giant string + const aggregatedToC = entries.map(entry => entry.toc).join('\n'); + + // Aggregates all individual content into one giant string + const aggregatedContent = entries.map(entry => entry.content).join('\n'); + + // Creates a "mimic" of an `MetadataEntry` which fulfils the requirements + // for generating the `tableOfContents` with the `tableOfContents.parseNavigationNode` parser + const sideNavigationFromValues = entries.map(entry => ({ + api: entry.api, + heading: { data: { depth: 1, name: entry.section } }, + })); + + // Generates the global Table of Contents (Sidebar Navigation) + const parsedSideNav = remarkWithRehype.processSync( + tableOfContents(sideNavigationFromValues, { + maxDepth: 1, + parser: tableOfContents.parseNavigationNode, + }) + ); + + const templateValues = { + api: 'all', + path: 'all', + added: '', + section: 'All', + version: `v${config.version.version}`, + toc: aggregatedToC, + nav: String(parsedSideNav), + content: aggregatedContent, + }; - version: '1.0.0', + let result = replaceTemplateValues(apiTemplate, templateValues, config, { + skipGitHub: true, + skipGtocPicker: true, + }); - description: - 'Generates the `all.html` file from the `legacy-html` generator, which includes all the modules in one single file', + if (config.minify) { + result = await minifyHTML(result); + } - dependsOn: 'legacy-html', + if (config.output) { + await writeFile(join(config.output, 'all.html'), result); + } - defaultConfiguration: { - templatePath: legacyHtml.defaultConfiguration.templatePath, - }, -}); + return result; +} diff --git a/packages/core/src/generators/legacy-html/generate.mjs b/packages/core/src/generators/legacy-html/generate.mjs deleted file mode 100644 index cff2698d..00000000 --- a/packages/core/src/generators/legacy-html/generate.mjs +++ /dev/null @@ -1,140 +0,0 @@ -'use strict'; - -import { readFile, cp } from 'node:fs/promises'; -import { basename, join } from 'node:path'; - -import buildContent from './utils/buildContent.mjs'; -import { replaceTemplateValues } from './utils/replaceTemplateValues.mjs'; -import tableOfContents from './utils/tableOfContents.mjs'; -import getConfig from '../../utils/configuration/index.mjs'; -import { writeFile } from '../../utils/file.mjs'; -import { groupNodesByModule } from '../../utils/generators.mjs'; -import { minifyHTML } from '../../utils/html-minifier.mjs'; -import { getRemarkRehypeWithShiki } from '../../utils/remark.mjs'; - -/** - * Creates a heading object with the given name. - * @param {string} name - The name of the heading - * @returns {HeadingMetadataEntry} The heading object - */ -const getHeading = name => ({ depth: 1, data: { name } }); - -const remarkRehypeProcessor = getRemarkRehypeWithShiki(); - -/** - * Process a chunk of items in a worker thread. - * Builds HTML template objects - FS operations happen in generate(). - * - * Each item is pre-grouped {head, nodes, headNodes} - no need to - * recompute groupNodesByModule for every chunk. - * - * @type {import('./types').Generator['processChunk']} - */ -export async function processChunk(slicedInput, itemIndices, navigation) { - const results = []; - - for (const idx of itemIndices) { - const { head, nodes, headNodes } = slicedInput[idx]; - - const nav = navigation.replace( - `class="nav-${head.api}`, - `class="nav-${head.api} active` - ); - - const toc = String( - remarkRehypeProcessor.processSync( - tableOfContents(nodes, { - maxDepth: 5, - parser: tableOfContents.parseToCNode, - }) - ) - ); - - const content = buildContent(headNodes, nodes, remarkRehypeProcessor); - - const apiAsHeading = head.api.charAt(0).toUpperCase() + head.api.slice(1); - - const template = { - api: head.api, - path: head.path, - added: head.introduced_in ?? '', - section: head.heading.data.name || apiAsHeading, - toc, - nav, - content, - }; - - results.push(template); - } - - return results; -} - -/** - * Generates the legacy version of the API docs in HTML - * - * @type {import('./types').Generator['generate']} - */ -export async function* generate(input, worker) { - const config = getConfig('legacy-html'); - - const apiTemplate = await readFile(config.templatePath, 'utf-8'); - - const groupedModules = groupNodesByModule(input); - - const headNodes = input - .filter(node => node.heading.depth === 1) - .toSorted((a, b) => a.heading.data.name.localeCompare(b.heading.data.name)); - - const indexOfFiles = config.index - ? config.index.map(({ api, section }) => ({ - api, - heading: getHeading(section), - })) - : headNodes; - - const navigation = String( - remarkRehypeProcessor.processSync( - tableOfContents(indexOfFiles, { - maxDepth: 1, - parser: tableOfContents.parseNavigationNode, - }) - ) - ); - - if (config.output) { - for (const path of config.additionalPathsToCopy) { - // Define the output folder for API docs assets - const assetsFolder = join(config.output, basename(path)); - - // Copy all files from assets folder to output - await cp(path, assetsFolder, { recursive: true }); - } - } - - // Create sliced input: each item contains head + its module's entries + headNodes reference - // This avoids sending all ~4900 entries to every worker and recomputing groupings - const entries = headNodes.map(head => ({ - head, - nodes: groupedModules.get(head.api), - headNodes, - })); - - // Stream chunks as they complete - HTML files are written immediately - for await (const chunkResult of worker.stream(entries, navigation)) { - // Write files for this chunk in the generate method (main thread) - if (config.output) { - for (const template of chunkResult) { - let result = replaceTemplateValues(apiTemplate, template, config); - - if (config.minify) { - result = await minifyHTML(result); - } - - await writeFile(join(config.output, `${template.api}.html`), result); - } - } - - yield chunkResult; - } -} diff --git a/packages/core/src/generators/legacy-html/index.mjs b/packages/core/src/generators/legacy-html/index.mjs index 32003ec1..332d0df1 100644 --- a/packages/core/src/generators/legacy-html/index.mjs +++ b/packages/core/src/generators/legacy-html/index.mjs @@ -1,37 +1,151 @@ 'use strict'; -import { join } from 'node:path'; +import { readFile, cp } from 'node:fs/promises'; +import { basename, join } from 'node:path'; +import buildContent from './utils/buildContent.mjs'; +import { replaceTemplateValues } from './utils/replaceTemplateValues.mjs'; +import tableOfContents from './utils/tableOfContents.mjs'; +import getConfig from '../../utils/configuration/index.mjs'; import { GITHUB_EDIT_URL } from '../../utils/configuration/templates.mjs'; -import { createLazyGenerator } from '../../utils/generators.mjs'; +import { writeFile } from '../../utils/file.mjs'; +import { groupNodesByModule } from '../../utils/generators.mjs'; +import { minifyHTML } from '../../utils/html-minifier.mjs'; +import { getRemarkRehypeWithShiki } from '../../utils/remark.mjs'; + +export const name = 'legacy-html'; +export const dependsOn = '@node-core/doc-kit/generators/metadata'; +export const defaultConfiguration = { + templatePath: join(import.meta.dirname, 'template.html'), + additionalPathsToCopy: [join(import.meta.dirname, 'assets')], + ref: 'main', + pageURL: '{baseURL}/latest-{version}/api{path}.html', + editURL: `${GITHUB_EDIT_URL}/doc/api{path}.md`, +}; + +/** + * Creates a heading object with the given name. + * @param {string} name - The name of the heading + * @returns {HeadingMetadataEntry} The heading object + */ +const getHeading = name => ({ depth: 1, data: { name } }); + +const remarkRehypeProcessor = getRemarkRehypeWithShiki(); /** + * Process a chunk of items in a worker thread. + * Builds HTML template objects - FS operations happen in generate(). * - * This generator generates the legacy HTML pages of the legacy API docs - * for retro-compatibility and while we are implementing the new 'react' and 'html' generators. + * Each item is pre-grouped {head, nodes, headNodes} - no need to + * recompute groupNodesByModule for every chunk. * - * This generator is a top-level generator, and it takes the raw AST tree of the API doc files - * and generates the HTML files to the specified output directory from the configuration settings + * @type {import('./types').Generator['processChunk']} + */ +export async function processChunk(slicedInput, itemIndices, navigation) { + const results = []; + + for (const idx of itemIndices) { + const { head, nodes, headNodes } = slicedInput[idx]; + + const nav = navigation.replace( + `class="nav-${head.api}`, + `class="nav-${head.api} active` + ); + + const toc = String( + remarkRehypeProcessor.processSync( + tableOfContents(nodes, { + maxDepth: 5, + parser: tableOfContents.parseToCNode, + }) + ) + ); + + const content = buildContent(headNodes, nodes, remarkRehypeProcessor); + + const apiAsHeading = head.api.charAt(0).toUpperCase() + head.api.slice(1); + + const template = { + api: head.api, + path: head.path, + added: head.introduced_in ?? '', + section: head.heading.data.name || apiAsHeading, + toc, + nav, + content, + }; + + results.push(template); + } + + return results; +} + +/** + * Generates the legacy version of the API docs in HTML * - * @type {import('./types').Generator} + * @type {import('./types').Generator['generate']} */ -export default createLazyGenerator({ - name: 'legacy-html', +export async function* generate(input, worker) { + const config = getConfig('legacy-html'); + + const apiTemplate = await readFile(config.templatePath, 'utf-8'); + + const groupedModules = groupNodesByModule(input); + + const headNodes = input + .filter(node => node.heading.depth === 1) + .toSorted((a, b) => a.heading.data.name.localeCompare(b.heading.data.name)); + + const indexOfFiles = config.index + ? config.index.map(({ api, section }) => ({ + api, + heading: getHeading(section), + })) + : headNodes; + + const navigation = String( + remarkRehypeProcessor.processSync( + tableOfContents(indexOfFiles, { + maxDepth: 1, + parser: tableOfContents.parseNavigationNode, + }) + ) + ); + + if (config.output) { + for (const path of config.additionalPathsToCopy) { + // Define the output folder for API docs assets + const assetsFolder = join(config.output, basename(path)); + + // Copy all files from assets folder to output + await cp(path, assetsFolder, { recursive: true }); + } + } - version: '1.0.0', + // Create sliced input: each item contains head + its module's entries + headNodes reference + // This avoids sending all ~4900 entries to every worker and recomputing groupings + const entries = headNodes.map(head => ({ + head, + nodes: groupedModules.get(head.api), + headNodes, + })); - description: - 'Generates the legacy version of the API docs in HTML, with the assets and styles included as files', + // Stream chunks as they complete - HTML files are written immediately + for await (const chunkResult of worker.stream(entries, navigation)) { + // Write files for this chunk in the generate method (main thread) + if (config.output) { + for (const template of chunkResult) { + let result = replaceTemplateValues(apiTemplate, template, config); - dependsOn: 'metadata', + if (config.minify) { + result = await minifyHTML(result); + } - defaultConfiguration: { - templatePath: join(import.meta.dirname, 'template.html'), - additionalPathsToCopy: [join(import.meta.dirname, 'assets')], - ref: 'main', - pageURL: '{baseURL}/latest-{version}/api{path}.html', - editURL: `${GITHUB_EDIT_URL}/doc/api{path}.md`, - }, + await writeFile(join(config.output, `${template.api}.html`), result); + } + } - hasParallelProcessor: true, -}); + yield chunkResult; + } +} diff --git a/packages/core/src/generators/legacy-json-all/generate.mjs b/packages/core/src/generators/legacy-json-all/generate.mjs deleted file mode 100644 index bc04f9aa..00000000 --- a/packages/core/src/generators/legacy-json-all/generate.mjs +++ /dev/null @@ -1,80 +0,0 @@ -'use strict'; - -import { writeFile } from 'node:fs/promises'; -import { join } from 'node:path'; - -import getConfig from '../../utils/configuration/index.mjs'; -import { legacyToJSON } from '../../utils/generators.mjs'; - -/** - * Generates the legacy JSON `all.json` file. - * - * @type {import('./types').Generator['generate']} - */ -export async function generate(input) { - const config = getConfig('legacy-json-all'); - - /** - * The consolidated output object that will contain - * combined data from all sections in the input. - * - * @type {import('./types.d.ts').Output} - */ - const generatedValue = { - miscs: [], - modules: [], - classes: [], - globals: [], - methods: [], - }; - - /** - * The properties to copy from each section in the input - */ - const propertiesToCopy = Object.keys(generatedValue); - - // Create a map of api name to index position for sorting - const indexOrder = new Map( - config.index?.map(({ api }, position) => [`doc/api/${api}.md`, position]) ?? - [] - ); - - // Sort input by index order (documents not in index go to the end) - const sortedInput = input.toSorted((a, b) => { - const aOrder = indexOrder.get(a.source) ?? Infinity; - const bOrder = indexOrder.get(b.source) ?? Infinity; - - return aOrder - bOrder; - }); - - // Aggregate all sections into the output - for (const section of sortedInput) { - // Skip index.json - it has no useful content, just navigation - if (section.api === 'index') { - continue; - } - - for (const property of propertiesToCopy) { - const items = section[property]; - - if (Array.isArray(items)) { - const enrichedItems = section.source - ? items.map(item => ({ ...item, source: section.source })) - : items; - - generatedValue[property].push(...enrichedItems); - } - } - } - - if (config.output) { - await writeFile( - join(config.output, 'all.json'), - config.minify - ? legacyToJSON(generatedValue) - : legacyToJSON(generatedValue, null, 2) - ); - } - - return generatedValue; -} diff --git a/packages/core/src/generators/legacy-json-all/index.mjs b/packages/core/src/generators/legacy-json-all/index.mjs index dcfe0fab..ae11fddf 100644 --- a/packages/core/src/generators/legacy-json-all/index.mjs +++ b/packages/core/src/generators/legacy-json-all/index.mjs @@ -1,24 +1,86 @@ 'use strict'; -import { createLazyGenerator } from '../../utils/generators.mjs'; +import { writeFile } from 'node:fs/promises'; +import { join } from 'node:path'; + +import getConfig from '../../utils/configuration/index.mjs'; +import { legacyToJSON } from '../../utils/generators.mjs'; + +export const name = 'legacy-json-all'; +export const dependsOn = '@node-core/doc-kit/generators/legacy-json'; +export const defaultConfiguration = { + minify: false, +}; /** - * This generator consolidates data from the `legacy-json` generator into a single - * JSON file (`all.json`). + * Generates the legacy JSON `all.json` file. * - * @type {import('./types.d.ts').Generator} + * @type {import('./types').Generator['generate']} */ -export default createLazyGenerator({ - name: 'legacy-json-all', +export async function generate(input) { + const config = getConfig('legacy-json-all'); + + /** + * The consolidated output object that will contain + * combined data from all sections in the input. + * + * @type {import('./types.d.ts').Output} + */ + const generatedValue = { + miscs: [], + modules: [], + classes: [], + globals: [], + methods: [], + }; + + /** + * The properties to copy from each section in the input + */ + const propertiesToCopy = Object.keys(generatedValue); + + // Create a map of api name to index position for sorting + const indexOrder = new Map( + config.index?.map(({ api }, position) => [`doc/api/${api}.md`, position]) ?? + [] + ); + + // Sort input by index order (documents not in index go to the end) + const sortedInput = input.toSorted((a, b) => { + const aOrder = indexOrder.get(a.source) ?? Infinity; + const bOrder = indexOrder.get(b.source) ?? Infinity; + + return aOrder - bOrder; + }); + + // Aggregate all sections into the output + for (const section of sortedInput) { + // Skip index.json - it has no useful content, just navigation + if (section.api === 'index') { + continue; + } + + for (const property of propertiesToCopy) { + const items = section[property]; - version: '1.0.0', + if (Array.isArray(items)) { + const enrichedItems = section.source + ? items.map(item => ({ ...item, source: section.source })) + : items; - description: - 'Generates the `all.json` file from the `legacy-json` generator, which includes all the modules in one single file.', + generatedValue[property].push(...enrichedItems); + } + } + } - dependsOn: 'legacy-json', + if (config.output) { + await writeFile( + join(config.output, 'all.json'), + config.minify + ? legacyToJSON(generatedValue) + : legacyToJSON(generatedValue, null, 2) + ); + } - defaultConfiguration: { - minify: false, - }, -}); + return generatedValue; +} diff --git a/packages/core/src/generators/legacy-json/generate.mjs b/packages/core/src/generators/legacy-json/generate.mjs deleted file mode 100644 index d5f5fb82..00000000 --- a/packages/core/src/generators/legacy-json/generate.mjs +++ /dev/null @@ -1,66 +0,0 @@ -'use strict'; - -import { writeFile } from 'node:fs/promises'; -import { join } from 'node:path'; - -import { createSectionBuilder } from './utils/buildSection.mjs'; -import getConfig from '../../utils/configuration/index.mjs'; -import { groupNodesByModule, legacyToJSON } from '../../utils/generators.mjs'; - -const buildSection = createSectionBuilder(); - -/** - * Process a chunk of items in a worker thread. - * Builds JSON sections - FS operations happen in generate(). - * - * Each item is pre-grouped {head, nodes} - no need to - * recompute groupNodesByModule for every chunk. - * - * @type {import('./types').Generator['processChunk']} - */ -export async function processChunk(slicedInput, itemIndices) { - const results = []; - - for (const idx of itemIndices) { - const { head, nodes } = slicedInput[idx]; - - results.push(buildSection(head, nodes)); - } - - return results; -} - -/** - * Generates a legacy JSON file. - * - * @type {import('./types').Generator['generate']} - */ -export async function* generate(input, worker) { - const config = getConfig('legacy-json'); - - const groupedModules = groupNodesByModule(input); - - const headNodes = input.filter(node => node.heading.depth === 1); - - // Create sliced input: each item contains head + its module's entries - // This avoids sending all 4900+ entries to every worker - const entries = headNodes.map(head => ({ - head, - nodes: groupedModules.get(head.api), - })); - - for await (const chunkResult of worker.stream(entries)) { - if (config.output) { - for (const section of chunkResult) { - const out = join(config.output, `${section.api}.json`); - - await writeFile( - out, - config.minify ? legacyToJSON(section) : legacyToJSON(section, null, 2) - ); - } - } - - yield chunkResult; - } -} diff --git a/packages/core/src/generators/legacy-json/index.mjs b/packages/core/src/generators/legacy-json/index.mjs index a2b5f5d9..69ec68ff 100644 --- a/packages/core/src/generators/legacy-json/index.mjs +++ b/packages/core/src/generators/legacy-json/index.mjs @@ -1,31 +1,73 @@ 'use strict'; -import { createLazyGenerator } from '../../utils/generators.mjs'; +import { writeFile } from 'node:fs/promises'; +import { join } from 'node:path'; + +import { createSectionBuilder } from './utils/buildSection.mjs'; +import getConfig from '../../utils/configuration/index.mjs'; +import { groupNodesByModule, legacyToJSON } from '../../utils/generators.mjs'; + +export const name = 'legacy-json'; +export const dependsOn = '@node-core/doc-kit/generators/metadata'; +export const defaultConfiguration = { + ref: 'main', + minify: false, +}; + +const buildSection = createSectionBuilder(); /** - * This generator is responsible for generating the legacy JSON files for the - * legacy API docs for retro-compatibility. It is to be replaced while we work - * on the new schema for this file. + * Process a chunk of items in a worker thread. + * Builds JSON sections - FS operations happen in generate(). * - * This is a top-level generator, intaking the raw AST tree of the api docs. - * It generates JSON files to the specified output directory given by the - * config. + * Each item is pre-grouped {head, nodes} - no need to + * recompute groupNodesByModule for every chunk. * - * @type {import('./types').Generator} + * @type {import('./types').Generator['processChunk']} */ -export default createLazyGenerator({ - name: 'legacy-json', +export async function processChunk(slicedInput, itemIndices) { + const results = []; + + for (const idx of itemIndices) { + const { head, nodes } = slicedInput[idx]; + + results.push(buildSection(head, nodes)); + } + + return results; +} + +/** + * Generates a legacy JSON file. + * + * @type {import('./types').Generator['generate']} + */ +export async function* generate(input, worker) { + const config = getConfig('legacy-json'); + + const groupedModules = groupNodesByModule(input); - version: '1.0.0', + const headNodes = input.filter(node => node.heading.depth === 1); - description: 'Generates the legacy version of the JSON API docs.', + // Create sliced input: each item contains head + its module's entries + // This avoids sending all 4900+ entries to every worker + const entries = headNodes.map(head => ({ + head, + nodes: groupedModules.get(head.api), + })); - dependsOn: 'metadata', + for await (const chunkResult of worker.stream(entries)) { + if (config.output) { + for (const section of chunkResult) { + const out = join(config.output, `${section.api}.json`); - defaultConfiguration: { - ref: 'main', - minify: false, - }, + await writeFile( + out, + config.minify ? legacyToJSON(section) : legacyToJSON(section, null, 2) + ); + } + } - hasParallelProcessor: true, -}); + yield chunkResult; + } +} diff --git a/packages/core/src/generators/llms-txt/generate.mjs b/packages/core/src/generators/llms-txt/generate.mjs deleted file mode 100644 index 00852c6a..00000000 --- a/packages/core/src/generators/llms-txt/generate.mjs +++ /dev/null @@ -1,32 +0,0 @@ -'use strict'; - -import { readFile } from 'node:fs/promises'; -import { join } from 'node:path'; - -import { buildApiDocLink } from './utils/buildApiDocLink.mjs'; -import getConfig from '../../utils/configuration/index.mjs'; -import { writeFile } from '../../utils/file.mjs'; - -/** - * Generates a llms.txt file - * - * @type {import('./types').Generator['generate']} - */ -export async function generate(input) { - const config = getConfig('llms-txt'); - - const template = await readFile(config.templatePath, 'utf-8'); - - const apiDocsLinks = input - .filter(entry => entry.heading.depth === 1) - .map(entry => `- ${buildApiDocLink(entry, config)}`) - .join('\n'); - - const filledTemplate = `${template}${apiDocsLinks}`; - - if (config.output) { - await writeFile(join(config.output, 'llms.txt'), filledTemplate); - } - - return filledTemplate; -} diff --git a/packages/core/src/generators/llms-txt/index.mjs b/packages/core/src/generators/llms-txt/index.mjs index 078ec7f1..d37bd214 100644 --- a/packages/core/src/generators/llms-txt/index.mjs +++ b/packages/core/src/generators/llms-txt/index.mjs @@ -1,27 +1,39 @@ 'use strict'; +import { readFile } from 'node:fs/promises'; import { join } from 'node:path'; -import { createLazyGenerator } from '../../utils/generators.mjs'; +import { buildApiDocLink } from './utils/buildApiDocLink.mjs'; +import getConfig from '../../utils/configuration/index.mjs'; +import { writeFile } from '../../utils/file.mjs'; + +export const name = 'llms-txt'; +export const dependsOn = '@node-core/doc-kit/generators/metadata'; +export const defaultConfiguration = { + templatePath: join(import.meta.dirname, 'template.txt'), + pageURL: '{baseURL}/latest/api{path}.md', +}; /** - * This generator generates a llms.txt file to provide information to LLMs at - * inference time + * Generates a llms.txt file * - * @type {import('./types').Generator} + * @type {import('./types').Generator['generate']} */ -export default createLazyGenerator({ - name: 'llms-txt', +export async function generate(input) { + const config = getConfig('llms-txt'); + + const template = await readFile(config.templatePath, 'utf-8'); - version: '1.0.0', + const apiDocsLinks = input + .filter(entry => entry.heading.depth === 1) + .map(entry => `- ${buildApiDocLink(entry, config)}`) + .join('\n'); - description: - 'Generates a llms.txt file to provide information to LLMs at inference time', + const filledTemplate = `${template}${apiDocsLinks}`; - dependsOn: 'metadata', + if (config.output) { + await writeFile(join(config.output, 'llms.txt'), filledTemplate); + } - defaultConfiguration: { - templatePath: join(import.meta.dirname, 'template.txt'), - pageURL: '{baseURL}/latest/api{path}.md', - }, -}); + return filledTemplate; +} diff --git a/packages/core/src/generators/man-page/generate.mjs b/packages/core/src/generators/man-page/generate.mjs deleted file mode 100644 index 4f779f71..00000000 --- a/packages/core/src/generators/man-page/generate.mjs +++ /dev/null @@ -1,78 +0,0 @@ -'use strict'; - -import { readFile } from 'node:fs/promises'; -import { join } from 'node:path'; - -import { - convertOptionToMandoc, - convertEnvVarToMandoc, -} from './utils/converter.mjs'; -import getConfig from '../../utils/configuration/index.mjs'; -import { writeFile } from '../../utils/file.mjs'; - -/** - * @param {Array} components - * @param {number} start - * @param {number} end - * @param {(element: import('../metadata/types').MetadataEntry) => string} convert - * @returns {string} - */ -function extractMandoc(components, start, end, convert) { - return components - .slice(start, end) - .filter(({ heading }) => heading.depth === 3) - .map(convert) - .join(''); -} - -/** - * Generates the Node.js man-page - * - * @type {import('./types').Generator['generate']} - */ -export async function generate(input) { - const config = getConfig('man-page'); - - // Find the appropriate headers - const optionsStart = input.findIndex( - ({ heading }) => heading.data.slug === config.cliOptionsHeaderSlug - ); - - const environmentStart = input.findIndex( - ({ heading }) => heading.data.slug === config.envVarsHeaderSlug - ); - - // The first header that is <3 in depth after environmentStart - const environmentEnd = input.findIndex( - ({ heading }, index) => heading.depth < 3 && index > environmentStart - ); - - const output = { - // Extract the CLI options. - options: extractMandoc( - input, - optionsStart + 1, - environmentStart, - convertOptionToMandoc - ), - // Extract the environment variables. - env: extractMandoc( - input, - environmentStart + 1, - environmentEnd, - convertEnvVarToMandoc - ), - }; - - const template = await readFile(config.templatePath, 'utf-8'); - - const filledTemplate = template - .replace('__OPTIONS__', output.options) - .replace('__ENVIRONMENT__', output.env); - - if (config.output) { - await writeFile(join(config.output, config.fileName), filledTemplate); - } - - return filledTemplate; -} diff --git a/packages/core/src/generators/man-page/index.mjs b/packages/core/src/generators/man-page/index.mjs index 1128720e..89c83337 100644 --- a/packages/core/src/generators/man-page/index.mjs +++ b/packages/core/src/generators/man-page/index.mjs @@ -1,28 +1,87 @@ 'use strict'; +import { readFile } from 'node:fs/promises'; import { join } from 'node:path'; -import { createLazyGenerator } from '../../utils/generators.mjs'; +import { + convertOptionToMandoc, + convertEnvVarToMandoc, +} from './utils/converter.mjs'; +import getConfig from '../../utils/configuration/index.mjs'; +import { writeFile } from '../../utils/file.mjs'; + +export const name = 'man-page'; +export const dependsOn = '@node-core/doc-kit/generators/metadata'; +export const defaultConfiguration = { + fileName: 'node.1', + cliOptionsHeaderSlug: 'options', + envVarsHeaderSlug: 'environment-variables-1', + templatePath: join(import.meta.dirname, 'template.1'), +}; + +/** + * @param {Array} components + * @param {number} start + * @param {number} end + * @param {(element: import('../metadata/types').MetadataEntry) => string} convert + * @returns {string} + */ +function extractMandoc(components, start, end, convert) { + return components + .slice(start, end) + .filter(({ heading }) => heading.depth === 3) + .map(convert) + .join(''); +} /** - * This generator generates a man page version of the CLI.md file. - * See https://man.openbsd.org/mdoc.7 for the formatting. + * Generates the Node.js man-page * - * @type {import('./types').Generator} + * @type {import('./types').Generator['generate']} */ -export default createLazyGenerator({ - name: 'man-page', +export async function generate(input) { + const config = getConfig('man-page'); + + // Find the appropriate headers + const optionsStart = input.findIndex( + ({ heading }) => heading.data.slug === config.cliOptionsHeaderSlug + ); + + const environmentStart = input.findIndex( + ({ heading }) => heading.data.slug === config.envVarsHeaderSlug + ); + + // The first header that is <3 in depth after environmentStart + const environmentEnd = input.findIndex( + ({ heading }, index) => heading.depth < 3 && index > environmentStart + ); + + const output = { + // Extract the CLI options. + options: extractMandoc( + input, + optionsStart + 1, + environmentStart, + convertOptionToMandoc + ), + // Extract the environment variables. + env: extractMandoc( + input, + environmentStart + 1, + environmentEnd, + convertEnvVarToMandoc + ), + }; - version: '1.0.0', + const template = await readFile(config.templatePath, 'utf-8'); - description: 'Generates the Node.js man-page.', + const filledTemplate = template + .replace('__OPTIONS__', output.options) + .replace('__ENVIRONMENT__', output.env); - dependsOn: 'metadata', + if (config.output) { + await writeFile(join(config.output, config.fileName), filledTemplate); + } - defaultConfiguration: { - fileName: 'node.1', - cliOptionsHeaderSlug: 'options', - envVarsHeaderSlug: 'environment-variables-1', - templatePath: join(import.meta.dirname, 'template.1'), - }, -}); + return filledTemplate; +} diff --git a/packages/core/src/generators/metadata/generate.mjs b/packages/core/src/generators/metadata/generate.mjs deleted file mode 100644 index ecb830bb..00000000 --- a/packages/core/src/generators/metadata/generate.mjs +++ /dev/null @@ -1,38 +0,0 @@ -'use strict'; - -import { parseApiDoc } from './utils/parse.mjs'; -import { parseTypeMap } from '../../parsers/json.mjs'; -import getConfig from '../../utils/configuration/index.mjs'; - -/** - * Process a chunk of API doc files in a worker thread. - * Called by chunk-worker.mjs for parallel processing. - * - * @type {import('./types').Generator['processChunk']} - */ -export async function processChunk(fullInput, itemIndices, typeMap) { - const results = []; - - for (const idx of itemIndices) { - results.push(...parseApiDoc(fullInput[idx], typeMap)); - } - - return results; -} - -/** - * Generates a flattened list of metadata entries from API docs. - * - * @type {import('./types').Generator['generate']} - */ -export async function* generate(inputs, worker) { - const { metadata: config } = getConfig(); - - const typeMap = await parseTypeMap(config.typeMap); - - // Stream chunks as they complete - allows dependent generators - // to start collecting/preparing while we're still processing - for await (const chunkResult of worker.stream(inputs, typeMap)) { - yield chunkResult.flat(); - } -} diff --git a/packages/core/src/generators/metadata/index.mjs b/packages/core/src/generators/metadata/index.mjs index 53c426d4..97c23b11 100644 --- a/packages/core/src/generators/metadata/index.mjs +++ b/packages/core/src/generators/metadata/index.mjs @@ -1,20 +1,44 @@ 'use strict'; -import { createLazyGenerator } from '../../utils/generators.mjs'; +import { parseApiDoc } from './utils/parse.mjs'; +import { parseTypeMap } from '../../parsers/json.mjs'; +import getConfig from '../../utils/configuration/index.mjs'; + +export const name = 'metadata'; +export const dependsOn = '@node-core/doc-kit/generators/ast'; +export const defaultConfiguration = { + typeMap: import.meta.resolve('./typeMap.json'), +}; /** - * This generator generates a flattened list of metadata entries from a API doc + * Process a chunk of API doc files in a worker thread. + * Called by chunk-worker.mjs for parallel processing. * - * @type {import('./types').Generator} + * @type {import('./types').Generator['processChunk']} */ -export default createLazyGenerator({ - name: 'metadata', +export async function processChunk(fullInput, itemIndices, typeMap) { + const results = []; + + for (const idx of itemIndices) { + results.push(...parseApiDoc(fullInput[idx], typeMap)); + } - version: '1.0.0', + return results; +} - description: 'generates a flattened list of API doc metadata entries', +/** + * Generates a flattened list of metadata entries from API docs. + * + * @type {import('./types').Generator['generate']} + */ +export async function* generate(inputs, worker) { + const { metadata: config } = getConfig(); - dependsOn: 'ast', + const typeMap = await parseTypeMap(config.typeMap); - hasParallelProcessor: true, -}); + // Stream chunks as they complete - allows dependent generators + // to start collecting/preparing while we're still processing + for await (const chunkResult of worker.stream(inputs, typeMap)) { + yield chunkResult.flat(); + } +} diff --git a/packages/core/src/generators/orama-db/generate.mjs b/packages/core/src/generators/orama-db/generate.mjs deleted file mode 100644 index 39134009..00000000 --- a/packages/core/src/generators/orama-db/generate.mjs +++ /dev/null @@ -1,60 +0,0 @@ -'use strict'; - -import { join } from 'node:path'; - -import { create, save, insertMultiple } from '@orama/orama'; - -import { SCHEMA } from './constants.mjs'; -import { buildHierarchicalTitle } from './utils/title.mjs'; -import getConfig from '../../utils/configuration/index.mjs'; -import { writeFile } from '../../utils/file.mjs'; -import { groupNodesByModule } from '../../utils/generators.mjs'; -import { transformNodeToString } from '../../utils/unist.mjs'; - -/** - * Generates the Orama database. - * - * @type {import('./types').Generator['generate']} - */ -export async function generate(input) { - const config = getConfig('orama-db'); - - const db = create({ schema: SCHEMA }); - - const apiGroups = groupNodesByModule(input); - - // Process all API groups and flatten into a single document array - const documents = Array.from(apiGroups.values()).flatMap(headings => - headings.map((entry, index) => { - const hierarchicalTitle = buildHierarchicalTitle(headings, index); - - const paragraph = entry.content.children.find( - child => child.type === 'paragraph' - ); - - return { - title: hierarchicalTitle, - description: paragraph - ? transformNodeToString(paragraph, true) - : undefined, - href: `${entry.path.slice(1)}.html#${entry.heading.data.slug}`, - siteSection: headings[0].heading.data.name, - }; - }) - ); - - // Insert all documents - await insertMultiple(db, documents); - - const result = save(db); - - // Persist - if (config.output) { - await writeFile( - join(config.output, 'orama-db.json'), - config.minify ? JSON.stringify(result) : JSON.stringify(result, null, 2) - ); - } - - return result; -} diff --git a/packages/core/src/generators/orama-db/index.mjs b/packages/core/src/generators/orama-db/index.mjs index bb6c2755..390c08a7 100644 --- a/packages/core/src/generators/orama-db/index.mjs +++ b/packages/core/src/generators/orama-db/index.mjs @@ -1,19 +1,62 @@ 'use strict'; -import { createLazyGenerator } from '../../utils/generators.mjs'; +import { join } from 'node:path'; +import { create, save, insertMultiple } from '@orama/orama'; + +import { SCHEMA } from './constants.mjs'; +import { buildHierarchicalTitle } from './utils/title.mjs'; +import getConfig from '../../utils/configuration/index.mjs'; +import { writeFile } from '../../utils/file.mjs'; +import { groupNodesByModule } from '../../utils/generators.mjs'; +import { transformNodeToString } from '../../utils/unist.mjs'; + +export const name = 'orama-db'; +export const dependsOn = '@node-core/doc-kit/generators/metadata'; /** - * This generator is responsible for generating the Orama database for the - * API docs. It is based on the legacy-json generator. + * Generates the Orama database. * - * @type {import('./types').Generator} + * @type {import('./types').Generator['generate']} */ -export default createLazyGenerator({ - name: 'orama-db', +export async function generate(input) { + const config = getConfig('orama-db'); + + const db = create({ schema: SCHEMA }); + + const apiGroups = groupNodesByModule(input); + + // Process all API groups and flatten into a single document array + const documents = Array.from(apiGroups.values()).flatMap(headings => + headings.map((entry, index) => { + const hierarchicalTitle = buildHierarchicalTitle(headings, index); + + const paragraph = entry.content.children.find( + child => child.type === 'paragraph' + ); + + return { + title: hierarchicalTitle, + description: paragraph + ? transformNodeToString(paragraph, true) + : undefined, + href: `${entry.path.slice(1)}.html#${entry.heading.data.slug}`, + siteSection: headings[0].heading.data.name, + }; + }) + ); + + // Insert all documents + await insertMultiple(db, documents); - version: '1.0.0', + const result = save(db); - description: 'Generates the Orama database for the API docs.', + // Persist + if (config.output) { + await writeFile( + join(config.output, 'orama-db.json'), + config.minify ? JSON.stringify(result) : JSON.stringify(result, null, 2) + ); + } - dependsOn: 'metadata', -}); + return result; +} diff --git a/packages/core/src/generators/sitemap/generate.mjs b/packages/core/src/generators/sitemap/generate.mjs deleted file mode 100644 index 9ec9786a..00000000 --- a/packages/core/src/generators/sitemap/generate.mjs +++ /dev/null @@ -1,66 +0,0 @@ -'use strict'; - -import { readFile } from 'node:fs/promises'; -import { join } from 'node:path'; - -import { createPageSitemapEntry } from './utils/createPageSitemapEntry.mjs'; -import getConfig from '../../utils/configuration/index.mjs'; -import { populate } from '../../utils/configuration/templates.mjs'; -import { writeFile } from '../../utils/file.mjs'; - -/** - * Generates a sitemap.xml file - * - * @type {import('./types').Generator['generate']} - */ -export async function generate(entries) { - const { sitemap: config } = getConfig(); - - const template = await readFile( - join(import.meta.dirname, 'template.xml'), - 'utf-8' - ); - - const entryTemplate = await readFile( - join(import.meta.dirname, 'entry-template.xml'), - 'utf-8' - ); - - const lastmod = new Date().toISOString().split('T')[0]; - - const apiPages = entries - .filter(entry => entry.heading.depth === 1) - .map(entry => createPageSitemapEntry(entry, config, lastmod)); - - const loc = populate(config.indexURL, config); - - /** - * @typedef {import('./types').SitemapEntry} - */ - const mainPage = { - loc, - lastmod, - changefreq: 'daily', - priority: '1.0', - }; - - apiPages.push(mainPage); - - const urlset = apiPages - .map(page => - entryTemplate - .replace('__LOC__', page.loc) - .replace('__LASTMOD__', page.lastmod) - .replace('__CHANGEFREQ__', page.changefreq) - .replace('__PRIORITY__', page.priority) - ) - .join(''); - - const sitemap = template.replace('__URLSET__', urlset); - - if (config.output) { - await writeFile(join(config.output, 'sitemap.xml'), sitemap, 'utf-8'); - } - - return sitemap; -} diff --git a/packages/core/src/generators/sitemap/index.mjs b/packages/core/src/generators/sitemap/index.mjs index 1f983423..729961e6 100644 --- a/packages/core/src/generators/sitemap/index.mjs +++ b/packages/core/src/generators/sitemap/index.mjs @@ -1,23 +1,73 @@ 'use strict'; -import { createLazyGenerator } from '../../utils/generators.mjs'; +import { readFile } from 'node:fs/promises'; +import { join } from 'node:path'; + +import { createPageSitemapEntry } from './utils/createPageSitemapEntry.mjs'; +import getConfig from '../../utils/configuration/index.mjs'; +import { populate } from '../../utils/configuration/templates.mjs'; +import { writeFile } from '../../utils/file.mjs'; + +export const name = 'sitemap'; +export const dependsOn = '@node-core/doc-kit/generators/metadata'; +export const defaultConfiguration = { + indexURL: '{baseURL}/latest/api/', + pageURL: '{indexURL}{path}.html', +}; /** - * This generator generates a sitemap.xml file for search engine optimization + * Generates a sitemap.xml file * - * @type {import('./types').Generator} + * @type {import('./types').Generator['generate']} */ -export default createLazyGenerator({ - name: 'sitemap', +export async function generate(entries) { + const { sitemap: config } = getConfig(); + + const template = await readFile( + join(import.meta.dirname, 'template.xml'), + 'utf-8' + ); + + const entryTemplate = await readFile( + join(import.meta.dirname, 'entry-template.xml'), + 'utf-8' + ); + + const lastmod = new Date().toISOString().split('T')[0]; + + const apiPages = entries + .filter(entry => entry.heading.depth === 1) + .map(entry => createPageSitemapEntry(entry, config, lastmod)); + + const loc = populate(config.indexURL, config); + + /** + * @typedef {import('./types').SitemapEntry} + */ + const mainPage = { + loc, + lastmod, + changefreq: 'daily', + priority: '1.0', + }; + + apiPages.push(mainPage); - version: '1.0.0', + const urlset = apiPages + .map(page => + entryTemplate + .replace('__LOC__', page.loc) + .replace('__LASTMOD__', page.lastmod) + .replace('__CHANGEFREQ__', page.changefreq) + .replace('__PRIORITY__', page.priority) + ) + .join(''); - description: 'Generates a sitemap.xml file for search engine optimization', + const sitemap = template.replace('__URLSET__', urlset); - dependsOn: 'metadata', + if (config.output) { + await writeFile(join(config.output, 'sitemap.xml'), sitemap, 'utf-8'); + } - defaultConfiguration: { - indexURL: '{baseURL}/latest/api/', - pageURL: '{indexURL}{path}.html', - }, -}); + return sitemap; +} diff --git a/packages/core/src/generators/types.d.ts b/packages/core/src/generators/types.d.ts index 6374d800..1265d77e 100644 --- a/packages/core/src/generators/types.d.ts +++ b/packages/core/src/generators/types.d.ts @@ -1,12 +1,4 @@ -import type { publicGenerators, allGenerators } from './index.mjs'; - declare global { - // Public generators exposed to the CLI - export type AvailableGenerators = typeof publicGenerators; - - // All generators including internal ones (metadata, jsx-ast, ast-js) - export type AllGenerators = typeof allGenerators; - /** * ParallelWorker interface for distributing work across Node.js worker threads. * Streams results as chunks complete, enabling pipeline parallelism. @@ -28,7 +20,7 @@ declare global { } export interface ParallelTaskOptions { - generatorName: keyof AllGenerators; + generatorSpecifier: string; input: unknown[]; itemIndices: number[]; } @@ -60,40 +52,18 @@ declare global { G extends Generate, P extends ProcessChunk | undefined = undefined, > = { - readonly defaultConfiguration: C; - - // The name of the Generator. Must match the Key in AllGenerators - name: keyof AllGenerators; - - version: string; - - description: string; + readonly defaultConfiguration?: C; - hasParallelProcessor: boolean; + // The name of the Generator + name: string; /** - * The immediate generator that this generator depends on. - * For example, the `html` generator depends on the `react` generator. - * - * If a given generator has no "before" generator, it will be considered a top-level - * generator, and run in parallel. - * - * Assume you pass to the `createGenerator`: ['json', 'html'] as the generators, - * this means both the 'json' and the 'html' generators will be executed and generate their - * own outputs in parallel. If the 'html' generator depends on the 'react' generator, then - * the 'react' generator will be executed first, then the 'html' generator. - * - * But both 'json' and 'html' generators will be executed in parallel. - * - * If you pass `createGenerator` with ['react', 'html'], the 'react' generator will be executed first, - * as it is a top level generator and then the 'html' generator would be executed after the 'react' generator. - * - * The 'ast' generator is the top-level parser for markdown files. It has no dependencies. + * The import specifier of the generator this one depends on. + * For example, '@node-core/doc-kit/generators/metadata'. * - * The `ast-js` generator is the top-level parser for JavaScript files. It - * passes the ASTs for any JavaScript files given in the input. + * If undefined, this is a top-level generator with no dependencies. */ - dependsOn: keyof AllGenerators | undefined; + dependsOn?: string; /** * Generators are abstract and the different generators have different sort of inputs and outputs. diff --git a/packages/core/src/generators/web/generate.mjs b/packages/core/src/generators/web/generate.mjs deleted file mode 100644 index 57ca2a75..00000000 --- a/packages/core/src/generators/web/generate.mjs +++ /dev/null @@ -1,54 +0,0 @@ -'use strict'; - -import { readFile } from 'node:fs/promises'; -import { createRequire } from 'node:module'; -import { join } from 'node:path'; - -import createASTBuilder from './utils/generate.mjs'; -import { processJSXEntries } from './utils/processing.mjs'; -import getConfig from '../../utils/configuration/index.mjs'; -import { writeFile } from '../../utils/file.mjs'; - -/** - * Main generation function that processes JSX AST entries into web bundles. - * - * @type {import('./types').Generator['generate']} - */ -export async function generate(input) { - const config = getConfig('web'); - - const template = await readFile(config.templatePath, 'utf-8'); - - // Create AST builders for server and client programs - const astBuilders = createASTBuilder(); - - // Create require function for resolving external packages in server code - const requireFn = createRequire(import.meta.url); - - // Process all entries: convert JSX to HTML/CSS/JS - const { results, css, chunks } = await processJSXEntries( - input, - template, - astBuilders, - requireFn, - config - ); - - // Process all entries together (required for code-split bundles) - if (config.output) { - // Write HTML files - for (const { html, path } of results) { - await writeFile(join(config.output, `${path}.html`), html, 'utf-8'); - } - - // Write code-split JavaScript chunks - for (const chunk of chunks) { - await writeFile(join(config.output, chunk.fileName), chunk.code, 'utf-8'); - } - - // Write CSS bundle - await writeFile(join(config.output, 'styles.css'), css, 'utf-8'); - } - - return results.map(({ html }) => ({ html: html.toString(), css })); -} diff --git a/packages/core/src/generators/web/index.mjs b/packages/core/src/generators/web/index.mjs index f881bebf..293fd177 100644 --- a/packages/core/src/generators/web/index.mjs +++ b/packages/core/src/generators/web/index.mjs @@ -1,41 +1,69 @@ 'use strict'; +import { readFile } from 'node:fs/promises'; +import { createRequire } from 'node:module'; import { join } from 'node:path'; -import { createLazyGenerator } from '../../utils/generators.mjs'; +import createASTBuilder from './utils/generate.mjs'; +import { processJSXEntries } from './utils/processing.mjs'; +import getConfig from '../../utils/configuration/index.mjs'; +import { writeFile } from '../../utils/file.mjs'; + +export const name = 'web'; +export const dependsOn = '@node-core/doc-kit/generators/jsx-ast'; +export const defaultConfiguration = { + templatePath: join(import.meta.dirname, 'template.html'), + title: 'Node.js', + imports: { + '#theme/Logo': '@node-core/ui-components/Common/NodejsLogo', + '#theme/Navigation': join(import.meta.dirname, './ui/components/NavBar'), + '#theme/Sidebar': join(import.meta.dirname, './ui/components/SideBar'), + '#theme/Metabar': join(import.meta.dirname, './ui/components/MetaBar'), + '#theme/Footer': join(import.meta.dirname, './ui/components/NoOp'), + '#theme/Layout': join(import.meta.dirname, './ui/components/Layout'), + }, +}; /** - * Web generator - transforms JSX AST entries into complete web bundles. - * - * This generator processes JSX AST entries and produces: - * - Server-side rendered HTML pages - * - Client-side JavaScript with code splitting - * - Bundled CSS styles - * - * Note: This generator does NOT support streaming/chunked processing because - * processJSXEntries needs all entries together to generate code-split bundles. + * Main generation function that processes JSX AST entries into web bundles. * - * @type {import('./types').Generator} + * @type {import('./types').Generator['generate']} */ -export default createLazyGenerator({ - name: 'web', - - version: '1.0.0', - - description: 'Generates HTML/CSS/JS bundles from JSX AST entries', - - dependsOn: 'jsx-ast', - - defaultConfiguration: { - templatePath: join(import.meta.dirname, 'template.html'), - title: 'Node.js', - imports: { - '#theme/Logo': '@node-core/ui-components/Common/NodejsLogo', - '#theme/Navigation': join(import.meta.dirname, './ui/components/NavBar'), - '#theme/Sidebar': join(import.meta.dirname, './ui/components/SideBar'), - '#theme/Metabar': join(import.meta.dirname, './ui/components/MetaBar'), - '#theme/Footer': join(import.meta.dirname, './ui/components/NoOp'), - '#theme/Layout': join(import.meta.dirname, './ui/components/Layout'), - }, - }, -}); +export async function generate(input) { + const config = getConfig('web'); + + const template = await readFile(config.templatePath, 'utf-8'); + + // Create AST builders for server and client programs + const astBuilders = createASTBuilder(); + + // Create require function for resolving external packages in server code + const requireFn = createRequire(import.meta.url); + + // Process all entries: convert JSX to HTML/CSS/JS + const { results, css, chunks } = await processJSXEntries( + input, + template, + astBuilders, + requireFn, + config + ); + + // Process all entries together (required for code-split bundles) + if (config.output) { + // Write HTML files + for (const { html, path } of results) { + await writeFile(join(config.output, `${path}.html`), html, 'utf-8'); + } + + // Write code-split JavaScript chunks + for (const chunk of chunks) { + await writeFile(join(config.output, chunk.fileName), chunk.code, 'utf-8'); + } + + // Write CSS bundle + await writeFile(join(config.output, 'styles.css'), css, 'utf-8'); + } + + return results.map(({ html }) => ({ html: html.toString(), css })); +} diff --git a/packages/core/src/loader.mjs b/packages/core/src/loader.mjs new file mode 100644 index 00000000..d0904cc2 --- /dev/null +++ b/packages/core/src/loader.mjs @@ -0,0 +1,64 @@ +'use strict'; + +/** @type {Map} */ +const cache = new Map(); + +/** + * Loads a generator by its import specifier. + * Imports the single entry point which exports generate, processChunk, + * name, dependsOn, and defaultConfiguration. + * + * @param {string} specifier - Full import specifier (e.g. '@node-core/doc-kit/generators/ast') + * @returns {Promise} The loaded generator + */ +export const loadGenerator = async specifier => { + if (cache.has(specifier)) { + return cache.get(specifier); + } + + const module = await import(specifier); + + const generator = { + specifier, + ...module, + }; + + cache.set(specifier, generator); + + return generator; +}; + +/** + * Resolves the full dependency chain for a set of target specifiers. + * Recursively follows `dependsOn` chains, loading each generator. + * + * @param {string[]} targets - Target generator specifiers + * @returns {Promise>} All generators needed, keyed by specifier + */ +export const resolveGeneratorGraph = async targets => { + /** @type {Map} */ + const loaded = new Map(); + + /** + * Resolve a generator via it's specifier + * @param {string} specifier + */ + const resolve = async specifier => { + if (loaded.has(specifier)) { + return; + } + + const generator = await loadGenerator(specifier); + loaded.set(specifier, generator); + + if (generator.dependsOn) { + await resolve(generator.dependsOn); + } + }; + + for (const target of targets) { + await resolve(target); + } + + return loaded; +}; diff --git a/packages/core/src/threading/__tests__/parallel.test.mjs b/packages/core/src/threading/__tests__/parallel.test.mjs index 766e3d03..3d0d56d3 100644 --- a/packages/core/src/threading/__tests__/parallel.test.mjs +++ b/packages/core/src/threading/__tests__/parallel.test.mjs @@ -1,6 +1,7 @@ import { deepStrictEqual, ok, strictEqual } from 'node:assert'; import { describe, it } from 'node:test'; +import { loadGenerator } from '../../loader.mjs'; import createWorkerPool from '../index.mjs'; import createParallelWorker from '../parallel.mjs'; @@ -38,10 +39,20 @@ async function collectChunks(generator) { return chunks; } +const metadataSpecifier = '@node-core/doc-kit/generators/metadata'; +const metadataGenerator = await loadGenerator(metadataSpecifier); +const astJsSpecifier = '@node-core/doc-kit/generators/ast-js'; +const astJsGenerator = await loadGenerator(astJsSpecifier); + describe('createParallelWorker', () => { it('should create a ParallelWorker with stream method', async () => { const pool = createWorkerPool(2); - const worker = createParallelWorker('metadata', pool, { threads: 2 }); + const worker = createParallelWorker( + metadataSpecifier, + metadataGenerator, + pool, + { threads: 2 } + ); ok(worker); strictEqual(typeof worker.stream, 'function'); @@ -51,7 +62,7 @@ describe('createParallelWorker', () => { it('should handle empty items array', async () => { const pool = createWorkerPool(2); - const worker = createParallelWorker('ast-js', pool, { + const worker = createParallelWorker(astJsSpecifier, astJsGenerator, pool, { threads: 2, chunkSize: 10, }); @@ -65,10 +76,15 @@ describe('createParallelWorker', () => { it('should distribute items to multiple worker threads', async () => { const pool = createWorkerPool(4); - const worker = createParallelWorker('metadata', pool, { - threads: 4, - chunkSize: 1, - }); + const worker = createParallelWorker( + metadataSpecifier, + metadataGenerator, + pool, + { + threads: 4, + chunkSize: 1, + } + ); const mockInput = [ { @@ -102,10 +118,15 @@ describe('createParallelWorker', () => { it('should yield results as chunks complete', async () => { const pool = createWorkerPool(2); - const worker = createParallelWorker('metadata', pool, { - threads: 2, - chunkSize: 1, - }); + const worker = createParallelWorker( + metadataSpecifier, + metadataGenerator, + pool, + { + threads: 2, + chunkSize: 1, + } + ); const mockInput = [ { @@ -127,10 +148,15 @@ describe('createParallelWorker', () => { it('should work with single thread and items', async () => { const pool = createWorkerPool(2); - const worker = createParallelWorker('metadata', pool, { - threads: 2, - chunkSize: 5, - }); + const worker = createParallelWorker( + metadataSpecifier, + metadataGenerator, + pool, + { + threads: 2, + chunkSize: 5, + } + ); const mockInput = [ { @@ -149,10 +175,15 @@ describe('createParallelWorker', () => { it('should use sliceInput for metadata generator', async () => { const pool = createWorkerPool(2); - const worker = createParallelWorker('metadata', pool, { - threads: 2, - chunkSize: 1, - }); + const worker = createParallelWorker( + metadataSpecifier, + metadataGenerator, + pool, + { + threads: 2, + chunkSize: 1, + } + ); const mockInput = [ { diff --git a/packages/core/src/threading/chunk-worker.mjs b/packages/core/src/threading/chunk-worker.mjs index 08e363ac..03e5a92b 100644 --- a/packages/core/src/threading/chunk-worker.mjs +++ b/packages/core/src/threading/chunk-worker.mjs @@ -1,6 +1,8 @@ -import { allGenerators } from '../generators/index.mjs'; import { setConfig } from '../utils/configuration/index.mjs'; +/** @type {Map>} */ +const generatorCache = new Map(); + /** * Processes a chunk of items using the specified generator's processChunk method. * This is the worker entry point for Piscina. @@ -9,7 +11,7 @@ import { setConfig } from '../utils/configuration/index.mjs'; * @returns {Promise} The processed result */ export default async ({ - generatorName, + generatorSpecifier, input, itemIndices, extra, @@ -17,7 +19,11 @@ export default async ({ }) => { await setConfig(configuration); - const generator = allGenerators[generatorName]; + if (!generatorCache.has(generatorSpecifier)) { + generatorCache.set(generatorSpecifier, import(generatorSpecifier)); + } + + const { processChunk } = await generatorCache.get(generatorSpecifier); - return generator.processChunk(input, itemIndices, extra); + return processChunk(input, itemIndices, extra); }; diff --git a/packages/core/src/threading/parallel.mjs b/packages/core/src/threading/parallel.mjs index d74ba92e..f369cbcf 100644 --- a/packages/core/src/threading/parallel.mjs +++ b/packages/core/src/threading/parallel.mjs @@ -1,6 +1,5 @@ 'use strict'; -import { allGenerators } from '../generators/index.mjs'; import logger from '../logger/index.mjs'; const parallelLogger = logger.child('parallel'); @@ -30,8 +29,9 @@ const createChunks = (count, size) => { * @param {unknown[]} fullInput - Full input array * @param {number[]} indices - Indices to process * @param {Object} extra - Stuff to pass to the worker - * @param {import('../utils/configuration/types').Configuration} configuration - Serialized options - * @param {string} generatorName - Name of the generator + * @param {object} configuration - Serialized options + * @param {string} generatorSpecifier - Import specifier of the generator + * @param {string} generatorName - Short name of the generator * @returns {ParallelTaskOptions} Task data for Piscina */ const createTask = ( @@ -39,10 +39,11 @@ const createTask = ( indices, extra, configuration, + generatorSpecifier, generatorName ) => { return { - generatorName, + generatorSpecifier, // Only send the items needed for this chunk (reduces serialization overhead) input: indices.map(i => fullInput[i]), // Remap indices to 0-based for the sliced array @@ -58,19 +59,20 @@ const createTask = ( /** * Creates a parallel worker that distributes work across a Piscina thread pool. * - * @param {keyof AllGenerators} generatorName - Generator name + * @param {string} generatorSpecifier - Import specifier for the generator + * @param {object} generator - The loaded generator object * @param {import('piscina').Piscina} pool - Piscina instance - * @param {import('../utils/configuration/types').Configuration} configuration - Generator options + * @param {object} configuration - Generator options * @returns {ParallelWorker} */ export default function createParallelWorker( - generatorName, + generatorSpecifier, + generator, pool, configuration ) { const { threads, chunkSize } = configuration; - - const generator = allGenerators[generatorName]; + const { name: generatorName } = generator; return { /** @@ -90,7 +92,12 @@ export default function createParallelWorker( parallelLogger.debug( `Distributing ${items.length} items across ${chunks.length} chunks`, - { generator: generatorName, chunks: chunks.length, chunkSize, threads } + { + generator: generatorName, + chunks: chunks.length, + chunkSize, + threads, + } ); const runInOneGo = threads <= 1 || items.length <= 2; @@ -108,7 +115,14 @@ export default function createParallelWorker( const promise = pool .run( - createTask(items, indices, extra, configuration, generatorName) + createTask( + items, + indices, + extra, + configuration, + generatorSpecifier, + generatorName + ) ) .then(result => ({ promise, result })); diff --git a/packages/core/src/utils/__tests__/generators.test.mjs b/packages/core/src/utils/__tests__/generators.test.mjs index e4c9e1a8..cd0c48eb 100644 --- a/packages/core/src/utils/__tests__/generators.test.mjs +++ b/packages/core/src/utils/__tests__/generators.test.mjs @@ -1,5 +1,5 @@ import assert from 'node:assert/strict'; -import { describe, it, mock, afterEach } from 'node:test'; +import { describe, it } from 'node:test'; import { groupNodesByModule, @@ -7,7 +7,6 @@ import { coerceSemVer, getCompatibleVersions, legacyToJSON, - createLazyGenerator, } from '../generators.mjs'; describe('groupNodesByModule', () => { @@ -124,34 +123,3 @@ describe('legacyToJSON', () => { assert.ok(result.includes('\n')); }); }); - -describe('createLazyGenerator', () => { - afterEach(() => mock.restoreAll()); - - it('spreads metadata properties onto the returned object', () => { - const metadata = { name: 'ast', version: '1.0.0', dependsOn: undefined }; - const gen = createLazyGenerator(metadata); - assert.equal(gen.name, 'ast'); - assert.equal(gen.version, '1.0.0'); - }); - - it('exposes generate and processChunk functions that delegate to the lazily loaded module', async () => { - // Both exports are mocked in a single mock.module() call to avoid ESM import - // cache collisions that occur when re-mocking the same specifier across two it() blocks. - const specifier = import.meta.resolve('../../generators/ast/generate.mjs'); - const fakeGenerate = async input => `processed:${input}`; - const fakeProcessChunk = async (input, indices) => - indices.map(i => input[i]); - mock.module(specifier, { - namedExports: { generate: fakeGenerate, processChunk: fakeProcessChunk }, - }); - - const gen = createLazyGenerator({ name: 'ast' }); - - const generateResult = await gen.generate('hello'); - assert.equal(generateResult, 'processed:hello'); - - const processChunkResult = await gen.processChunk(['a', 'b', 'c'], [0, 2]); - assert.deepStrictEqual(processChunkResult, ['a', 'c']); - }); -}); diff --git a/packages/core/src/utils/configuration/__tests__/index.test.mjs b/packages/core/src/utils/configuration/__tests__/index.test.mjs index 293a468f..942255dc 100644 --- a/packages/core/src/utils/configuration/__tests__/index.test.mjs +++ b/packages/core/src/utils/configuration/__tests__/index.test.mjs @@ -12,15 +12,6 @@ const createMockConfig = (overrides = {}) => ({ }); // Mock modules -mock.module('../../../generators/index.mjs', { - namedExports: { - allGenerators: { - json: { defaultConfiguration: { format: 'json' } }, - html: { defaultConfiguration: { format: 'html' } }, - markdown: {}, - }, - }, -}); mock.module('../../../parsers/markdown.mjs', { namedExports: { parseChangelog: mockParseChangelog, @@ -40,6 +31,20 @@ const { default: getConfig, } = await import('../index.mjs'); +// Create a mock loaded generators map for tests +const createMockGenerators = () => + new Map([ + [ + '@node-core/doc-kit/generators/json', + { name: 'json', defaultConfiguration: { format: 'json' } }, + ], + [ + '@node-core/doc-kit/generators/html', + { name: 'html', defaultConfiguration: { format: 'html' } }, + ], + ['@node-core/doc-kit/generators/markdown', { name: 'markdown' }], + ]); + // Helper to reset all mocks const resetAllMocks = () => { [mockParseChangelog, mockParseIndex, mockImportFromURL].forEach(m => @@ -131,11 +136,14 @@ describe('config.mjs', () => { createMockConfig({ global: { input: 'custom-src/' } }) ); - const config = await createRunConfiguration({ - configFile: 'config.mjs', - output: 'custom-dist/', - threads: 2, - }); + const config = await createRunConfiguration( + { + configFile: 'config.mjs', + output: 'custom-dist/', + threads: 2, + }, + createMockGenerators() + ); assert.strictEqual(config.global.input, 'custom-src/'); assert.strictEqual(config.global.output, 'custom-dist/'); @@ -157,7 +165,10 @@ describe('config.mjs', () => { ); resetAllMocks(); // Clear calls from getDefaultConfig - await createRunConfiguration({ configFile: 'config.mjs' }); + await createRunConfiguration( + { configFile: 'config.mjs' }, + createMockGenerators() + ); // Each should be called at least once for the string value assert.ok( @@ -172,20 +183,26 @@ describe('config.mjs', () => { }); it('should enforce minimum constraints', async () => { - const config = await createRunConfiguration({ - threads: -5, - chunkSize: 0, - }); + const config = await createRunConfiguration( + { + threads: -5, + chunkSize: 0, + }, + createMockGenerators() + ); assert.strictEqual(config.threads, 1); assert.strictEqual(config.chunkSize, 1); }); it('should work without config file', async () => { - const config = await createRunConfiguration({ - version: '20.0.0', - threads: 4, - }); + const config = await createRunConfiguration( + { + version: '20.0.0', + threads: 4, + }, + createMockGenerators() + ); assert.ok(config); assert.strictEqual(config.threads, 4); @@ -200,9 +217,12 @@ describe('config.mjs', () => { }) ); - const config = await createRunConfiguration({ - configFile: 'config.mjs', - }); + const config = await createRunConfiguration( + { + configFile: 'config.mjs', + }, + createMockGenerators() + ); assert.ok(config.json); assert.ok(config.html); @@ -212,7 +232,10 @@ describe('config.mjs', () => { describe('setConfig and getConfig', () => { it('should persist config across calls', async () => { - const config = await setConfig({ version: '20.0.0', threads: 2 }); + const config = await setConfig( + { version: '20.0.0', threads: 2 }, + createMockGenerators() + ); const retrieved = getConfig(); assert.strictEqual(config, retrieved); @@ -243,7 +266,10 @@ describe('config.mjs', () => { ); resetAllMocks(); - await createRunConfiguration({ configFile: 'config.mjs' }); + await createRunConfiguration( + { configFile: 'config.mjs' }, + createMockGenerators() + ); assert.ok(countCallsMatching(mockFn, ([arg]) => arg === value) >= 1); }); diff --git a/packages/core/src/utils/configuration/index.mjs b/packages/core/src/utils/configuration/index.mjs index acedda6c..42990300 100644 --- a/packages/core/src/utils/configuration/index.mjs +++ b/packages/core/src/utils/configuration/index.mjs @@ -4,45 +4,37 @@ import { isMainThread } from 'node:worker_threads'; import { coerce } from 'semver'; import { CHANGELOG_URL, populate } from './templates.mjs'; -import { allGenerators } from '../../generators/index.mjs'; import logger from '../../logger/index.mjs'; import { parseChangelog, parseIndex } from '../../parsers/markdown.mjs'; import { enforceArray } from '../array.mjs'; import { leftHandAssign } from '../generators.mjs'; -import { deepMerge, lazy } from '../misc.mjs'; +import { deepMerge } from '../misc.mjs'; import { importFromURL } from '../url.mjs'; /** - * Get's the default configuration + * Get's the default configuration (global/structural defaults only). + * Per-generator defaults are merged separately via mergeGeneratorDefaults. */ -export const getDefaultConfig = lazy(() => - Object.keys(allGenerators).reduce( - (acc, k) => { - acc[k] = allGenerators[k].defaultConfiguration ?? {}; - return acc; - }, - /** @type {import('./types').Configuration} */ ({ - global: { - version: process.version, - minify: true, - repository: 'nodejs/node', - ref: 'HEAD', - baseURL: 'https://nodejs.org/docs', - changelog: populate(CHANGELOG_URL, { - repository: 'nodejs/node', - ref: 'HEAD', - }), - }, - - // The number of wasm memory instances is severely limited on - // riscv64 with sv39. Running multiple generators that use wasm in - // parallel could cause failures to allocate new wasm instance. - // See also https://github.com/nodejs/node/pull/60591 - threads: process.arch === 'riscv64' ? 1 : cpus().length, - chunkSize: 10, - }) - ) -); +export const getDefaultConfig = () => ({ + global: { + version: process.version, + minify: true, + repository: 'nodejs/node', + ref: 'HEAD', + baseURL: 'https://nodejs.org/docs', + changelog: populate(CHANGELOG_URL, { + repository: 'nodejs/node', + ref: 'HEAD', + }), + }, + + // The number of wasm memory instances is severely limited on + // riscv64 with sv39. Running multiple generators that use wasm in + // parallel could cause failures to allocate new wasm instance. + // See also https://github.com/nodejs/node/pull/60591 + threads: process.arch === 'riscv64' ? 1 : cpus().length, + chunkSize: 10, +}); /** * Loads a configuration file from a URL or file path. @@ -109,9 +101,10 @@ export const createConfigFromCLIOptions = options => ({ * and constraint enforcement for threads and chunk size. * * @param {import('../../../bin/commands/generate.mjs').CLIOptions} options - User-provided configuration options + * @param {Map} loadedGenerators - Map of specifier → loaded generator * @returns {Promise} The configuration */ -export const createRunConfiguration = async options => { +export const createRunConfiguration = async (options, loadedGenerators) => { const config = await loadConfigFile(options.configFile); config.target &&= enforceArray(config.target); @@ -137,18 +130,30 @@ export const createRunConfiguration = async options => { // Transform global config if it wasn't already done await transformConfig(merged.global); - // Now assign to each generator config (they inherit from global) - await Promise.all( - Object.keys(allGenerators).map(async k => { - const value = merged[k]; + // Merge per-generator defaults from loaded generators and apply global config + if (loadedGenerators) { + await Promise.all( + [...loadedGenerators.values()].map(async generator => { + const { name, defaultConfiguration } = generator; - // Transform generator-specific overrides - await transformConfig(value); + // Initialize generator config section if it doesn't exist + if (!merged[name]) { + merged[name] = {}; + } - // Assign from global (this populates missing values from global) - leftHandAssign(value, merged.global); - }) - ); + // Merge generator's default configuration + if (defaultConfiguration) { + leftHandAssign(merged[name], defaultConfiguration); + } + + // Transform generator-specific overrides + await transformConfig(merged[name]); + + // Assign from global (this populates missing values from global) + leftHandAssign(merged[name], merged.global); + }) + ); + } return merged; }; @@ -159,10 +164,13 @@ let config; /** * Configuration setter * @param {import('./types').Configuration | import('../../../bin/commands/generate.mjs').CLIOptions} options + * @param {Map} [loadedGenerators] * @returns {Promise} */ -export const setConfig = async options => - (config = isMainThread ? await createRunConfiguration(options) : options); +export const setConfig = async (options, loadedGenerators) => + (config = isMainThread + ? await createRunConfiguration(options, loadedGenerators) + : options); /** * Configuration getter @@ -170,6 +178,7 @@ export const setConfig = async options => * @param {T} generator * @returns {T extends keyof import('./types').Configuration ? import('./types').Configuration[T] : import('./types').Configuration} */ -const getConfig = generator => (generator ? config[generator] : config); +const getConfig = generator => + generator ? (config[generator] ?? config.global) : config; export default getConfig; diff --git a/packages/core/src/utils/configuration/types.d.ts b/packages/core/src/utils/configuration/types.d.ts index 3536ba05..35c0f452 100644 --- a/packages/core/src/utils/configuration/types.d.ts +++ b/packages/core/src/utils/configuration/types.d.ts @@ -8,17 +8,14 @@ export type Configuration = { // This is considered a "sorted" list of generators, in the sense that // if the last entry of this list contains a generated value, we will return // the value of the last generator in the list, if any. - target: Array; + target: Array; // The number of threads the process is allowed to use threads: number; // Number of items to process per worker thread chunkSize: number; -} & { - [K in keyof AllGenerators]: GlobalConfiguration & - AllGenerators[K]['defaultConfiguration']; -}; +} & Record>; export type GlobalConfiguration = { // The repository diff --git a/packages/core/src/utils/generators.mjs b/packages/core/src/utils/generators.mjs index 955619b7..837c345a 100644 --- a/packages/core/src/utils/generators.mjs +++ b/packages/core/src/utils/generators.mjs @@ -2,8 +2,6 @@ import { coerce, major } from 'semver'; -import { lazy } from './misc.mjs'; - /** * Groups all the API metadata nodes by module (`api` property) so that we can process each different file * based on the module it belongs to. @@ -135,30 +133,3 @@ export const legacyToJSON = ( }, ...args ); - -/** - * Creates a generator with the provided metadata. - * @template T - * @param {T} metadata - The metadata object - * @returns {Promise} The metadata object with generator methods - */ -export const createLazyGenerator = metadata => { - const generator = lazy( - () => import(`../generators/${metadata.name}/generate.mjs`) - ); - return { - ...metadata, - /** - * Processes a chunk using the lazily-loaded generator. - * @param {...any} args - Arguments to pass to the processChunk method - * @returns {Promise} Result from the generator's processChunk method - */ - processChunk: async (...args) => (await generator()).processChunk(...args), - /** - * Generates output using the lazily-loaded generator. - * @param {...any} args - Arguments to pass to the generate method - * @returns {Promise} Result from the generator's generate method - */ - generate: async (...args) => (await generator()).generate(...args), - }; -}; diff --git a/scripts/vercel-build.sh b/scripts/vercel-build.sh index f8dc7e58..4628191a 100755 --- a/scripts/vercel-build.sh +++ b/scripts/vercel-build.sh @@ -1,8 +1,8 @@ node packages/core/bin/cli.mjs generate \ - -t orama-db \ - -t legacy-json \ - -t llms-txt \ - -t web \ + -t @node-core/doc-kit/generators/orama-db \ + -t @node-core/doc-kit/generators/legacy-json \ + -t @node-core/doc-kit/generators/llms-txt \ + -t @node-core/doc-kit/generators/web \ -i "./node/doc/api/*.md" \ -o "./out" \ -c "./node/CHANGELOG.md" \