diff --git a/infra/docs-gen/EXTRACTION_PIPELINE.md b/infra/docs-gen/EXTRACTION_PIPELINE.md new file mode 100644 index 000000000..e1789f189 --- /dev/null +++ b/infra/docs-gen/EXTRACTION_PIPELINE.md @@ -0,0 +1,42 @@ +# Extension Extraction Pipeline + +```mermaid +flowchart TD + CLI["extract-extension-data.mjs
Parse CLI options"] --> Extractor["ExtensionExtractor
src/extractor/index.mjs"] + Extractor --> Categories["Extension categories
src/config.mjs"] + Categories --> ScanAll["scanAll()
Skip internal extensions"] + ScanAll --> Scan["scanExtension()
src/extractor/scan.mjs"] + + Scan --> Files["readExtensionSources()
src/utils.mjs"] + Files --> Selectors["Source file selectors
src/extractor/source-files.mjs"] + Selectors --> SourceText["Production source text"] + Selectors --> TestFiles["Test files"] + Selectors --> SerializerFiles["Serializer and Specs files"] + + SourceText --> Constants["extractConstants()
src/extractor/constants.mjs"] + SourceText --> RegexScanners["Source scanners
src/extractor/regex.mjs + patterns.mjs"] + SourceText --> Options["Option declarations
src/extractor/options.mjs"] + SerializerFiles --> SerializerHints["Serializer hints"] + TestFiles --> MarkupExamples["Markup examples
src/extractor/examples.mjs"] + + Constants --> Schema["Schema names
nodes and marks"] + RegexScanners --> ExtractedFields["Actions, keymaps, input rules,
plugins, md plugins"] + Options --> ExtractedFields + Schema --> IR["Extension IR record"] + ExtractedFields --> IR + SerializerHints --> IR + MarkupExamples --> IR + + Extractor --> Presets["Preset membership
src/extractor/presets.mjs"] + Presets --> IR + IR --> JsonOut["extensions.json
src/extractor/output.mjs"] + IR --> MarkdownOut["raw/*.md
output.mjs -> markdown-gen.mjs"] +``` + +The extractor keeps orchestration and parsing separate: + +- `index.mjs` decides which extension directories are scanned and when output is written. +- `scan.mjs` builds one extension record from source files and parser results. +- `source-files.mjs` owns file selection rules. +- `regex.mjs`, `options.mjs`, `examples.mjs`, `constants.mjs`, and `patterns.mjs` own source parsing details. +- `output.mjs` and `markdown-gen.mjs` own generated artifacts. diff --git a/infra/docs-gen/README.md b/infra/docs-gen/README.md new file mode 100644 index 000000000..9f012645a --- /dev/null +++ b/infra/docs-gen/README.md @@ -0,0 +1,48 @@ +# Docs Generator + +`infra/docs-gen` contains tooling for two documentation flows: + +- building Diplodoc input from `docs/*.md`; +- extracting raw extension metadata from `packages/editor/src/extensions`. + +## Commands + +- `pnpm --filter @markdown-editor/docs-gen run build` creates `tmp/docs-src` and builds `dist/docs`. +- `pnpm --filter @markdown-editor/docs-gen run extract` creates `tmp/docs-gen/extensions.json` and `tmp/docs-gen/raw/*.md`. +- `pnpm --filter @markdown-editor/docs-gen run test` runs the colocated Node unit tests. + +## Files + +- `package.json` defines the local docs-gen package, scripts, and Nx target. +- `EXTRACTION_PIPELINE.md` documents the raw extension extraction flow with a Mermaid diagram. +- `src/config.mjs` stores shared paths, extension categories, ignored internal extensions, preset definitions, and generator regex constants. +- `src/extract-extension-data.mjs` provides the CLI entry point for raw extension extraction. +- `src/generate-docs.mjs` builds the Diplodoc source tree from repository markdown files. +- `src/logger.mjs` contains the small console logger used by docs-gen CLIs. +- `src/utils.mjs` contains filesystem helpers shared by the build and extraction flows. + +## Extractor Files + +- `src/extractor/index.mjs` contains `ExtensionExtractor`, the high-level orchestrator that scans extension categories, enriches records with presets, and writes output. +- `src/extractor/scan.mjs` scans one extension directory and assembles the raw extension IR record. +- `src/extractor/source-files.mjs` selects source, spec, serializer, root index, specs index, and test files from an extension directory. +- `src/extractor/output.mjs` writes extracted JSON and raw Markdown artifacts. +- `src/extractor/markdown-gen.mjs` renders one raw extension Markdown file from extracted metadata. +- `src/extractor/presets.mjs` parses editor preset files and resolves inherited preset membership. +- `src/extractor/constants.mjs` extracts string constants, enum values, object scalar members, and resolves references between them. +- `src/extractor/options.mjs` extracts extension option fields from local TypeScript option declarations. +- `src/extractor/examples.mjs` extracts serializer test markup examples from `same(...)` calls and simple local string expressions. +- `src/extractor/regex.mjs` contains shared scanner helpers and the source scanners for schema names, actions, keymaps, plugins, input rules, markdown-it plugins, and serializer hints. +- `src/extractor/patterns.mjs` keeps the regular expressions used by extractor modules. +- `src/extractor/actions.test.mjs` covers action extraction behavior. +- `src/extractor/cli.test.mjs` covers extraction CLI argument parsing. +- `src/extractor/constants.test.mjs` covers constant extraction and reference resolution behavior. +- `src/extractor/examples.test.mjs` covers markup example extraction behavior. +- `src/extractor/keymaps.test.mjs` covers keymap extraction behavior. +- `src/extractor/options.test.mjs` covers option declaration extraction behavior. + +## Output + +- `tmp/docs-gen/extensions.json` is the machine-readable raw IR for all extracted extensions. +- `tmp/docs-gen/raw/*.md` is the raw Markdown view of each extension record. +- `tmp/docs-src` is the generated Diplodoc input tree used by the docs build. diff --git a/infra/docs-gen/package.json b/infra/docs-gen/package.json index de6aa64c6..e63f0d947 100644 --- a/infra/docs-gen/package.json +++ b/infra/docs-gen/package.json @@ -14,7 +14,9 @@ } }, "scripts": { - "build": "node src/generate-docs.mjs && yfm -i ../../tmp/docs-src -o ../../dist/docs" + "build": "node src/generate-docs.mjs && yfm -i ../../tmp/docs-src -o ../../dist/docs", + "extract": "node src/extract-extension-data.mjs", + "test": "node --test src/extractor/*.test.mjs" }, "dependencies": { "@diplodoc/cli": "5.43.0" diff --git a/infra/docs-gen/src/config.mjs b/infra/docs-gen/src/config.mjs new file mode 100644 index 000000000..d46270c17 --- /dev/null +++ b/infra/docs-gen/src/config.mjs @@ -0,0 +1,37 @@ +import {dirname, join, resolve} from 'node:path'; +import {fileURLToPath} from 'node:url'; + +export const REPO_ROOT = resolve(dirname(fileURLToPath(import.meta.url)), '../../..'); +export const DOCS_DIR = join(REPO_ROOT, 'docs'); +export const DOCS_SRC_DIR = join(REPO_ROOT, 'tmp/docs-src'); +export const DOCS_GEN_DIR = join(REPO_ROOT, 'tmp/docs-gen'); +export const EDITOR_PKG_DIR = join(REPO_ROOT, 'packages/editor'); + +export const GITHUB_RAW_RE = + /https:\/\/raw\.githubusercontent\.com\/gravity-ui\/markdown-editor\/(?:refs\/heads\/[^/]+|[^/]+)\/docs\//g; +export const HEADER_RE = /^#{5}\s+(.+)$/; + +export const EXTENSION_CATEGORIES = ['base', 'behavior', 'markdown', 'yfm', 'additional']; + +export const INTERNAL_EXTENSIONS = [ + 'BaseInputRules', + 'BaseKeymap', + 'BaseStyles', + 'ReactRenderer', + 'SharedState', +]; + +export const PRESET_DEFS = [ + {name: 'ZeroPreset', file: 'zero.ts', parent: null}, + {name: 'CommonMarkPreset', file: 'commonmark.ts', parent: 'ZeroPreset'}, + {name: 'DefaultPreset', file: 'default.ts', parent: 'CommonMarkPreset'}, + {name: 'YfmPreset', file: 'yfm.ts', parent: 'DefaultPreset'}, + {name: 'FullPreset', file: 'full.ts', parent: 'YfmPreset'}, +]; + +/** + * Checks whether an extension is internal-only infrastructure. + */ +export function isInternalExtension(name) { + return INTERNAL_EXTENSIONS.includes(name); +} diff --git a/infra/docs-gen/src/extract-extension-data.mjs b/infra/docs-gen/src/extract-extension-data.mjs new file mode 100644 index 000000000..3c9614392 --- /dev/null +++ b/infra/docs-gen/src/extract-extension-data.mjs @@ -0,0 +1,133 @@ +#!/usr/bin/env node +import {isAbsolute, join} from 'node:path'; +import process from 'node:process'; +import {fileURLToPath} from 'node:url'; + +import {DOCS_GEN_DIR, EDITOR_PKG_DIR, REPO_ROOT} from './config.mjs'; +import {ExtensionExtractor} from './extractor/index.mjs'; +import {logger} from './logger.mjs'; + +/** + * Resolves a path from the repository root. + */ +function resolveFromRoot(path) { + return isAbsolute(path) ? path : join(REPO_ROOT, path); +} + +/** + * Creates default CLI options. + */ +function createDefaultOptions() { + return { + editorPkg: EDITOR_PKG_DIR, + outDir: DOCS_GEN_DIR, + only: null, + }; +} + +/** + * Reads a required option value. + */ +function readOptionValue(args, index, optionName) { + const value = args[index + 1]; + if (!value || value.startsWith('--')) { + throw new Error(`Missing value for ${optionName}`); + } + + return value; +} + +/** + * Parses comma-separated extension names. + */ +function parseOnlyOption(value) { + return value + .split(',') + .map((item) => item.trim()) + .filter(Boolean); +} + +/** + * Applies one CLI option to a new options object. + */ +function applyOption(opts, args, index) { + const arg = args[index]; + + switch (arg) { + case '--editor-pkg': + return { + nextIndex: index + 1, + opts: {...opts, editorPkg: resolveFromRoot(readOptionValue(args, index, arg))}, + }; + case '--out-dir': + return { + nextIndex: index + 1, + opts: {...opts, outDir: resolveFromRoot(readOptionValue(args, index, arg))}, + }; + case '--only': + return { + nextIndex: index + 1, + opts: {...opts, only: parseOnlyOption(readOptionValue(args, index, arg))}, + }; + case '--help': + return {nextIndex: index, opts: {...opts, help: true}}; + default: + throw new Error(`Unknown option: ${arg}`); + } +} + +/** + * Parses CLI options for extension data extraction. + */ +export function parseArgs(args = process.argv.slice(2)) { + let opts = createDefaultOptions(); + + for (let index = 0; index < args.length; index++) { + const arg = args[index]; + if (arg === '--') return opts; + + const parsedOption = applyOption(opts, args, index); + opts = parsedOption.opts; + index = parsedOption.nextIndex; + } + + return opts; +} + +/** + * Prints command usage information. + */ +function printHelp() { + logger.info('Usage: pnpm --filter @markdown-editor/docs-gen run extract [options]'); + logger.info(''); + logger.info('Options:'); + logger.info(' --only Bold,Link Extract selected extension names'); + logger.info(' --out-dir tmp/docs-gen Override output directory'); + logger.info(' --editor-pkg path Override packages/editor path'); +} + +/** + * Runs extension data extraction from CLI arguments. + */ +export function main(args = process.argv.slice(2)) { + const opts = parseArgs(args); + if (opts.help) { + printHelp(); + return; + } + + new ExtensionExtractor({ + editorPkg: opts.editorPkg, + outDir: opts.outDir, + repoRoot: REPO_ROOT, + }).run({only: opts.only}); +} + +if (process.argv[1] && fileURLToPath(import.meta.url) === process.argv[1]) { + try { + main(); + } catch (error) { + logger.error(error); + process.exit(1); + } +} diff --git a/infra/docs-gen/src/extractor/actions.test.mjs b/infra/docs-gen/src/extractor/actions.test.mjs new file mode 100644 index 000000000..d80ed5004 --- /dev/null +++ b/infra/docs-gen/src/extractor/actions.test.mjs @@ -0,0 +1,25 @@ +import assert from 'node:assert/strict'; +import {test} from 'node:test'; + +import {extractActions} from './regex.mjs'; + +test('extractActions captures direct and chained builder.addAction calls', () => { + const content = [ + "builder.addAction('bold', () => boldAction);", + 'builder', + ' .addAction(BoldAction.Toggle, () => toggle)', + ' .addAction(BoldAction.Off, () => off);', + ].join('\n'); + + assert.deepEqual(extractActions(content), ['bold', 'BoldAction.Toggle', 'BoldAction.Off']); +}); + +test('extractActions ignores non-builder addAction calls', () => { + const content = [ + "tr.addAction('shouldNotMatch', cb);", + "service.addAction('alsoSkip', cb);", + "builder.addAction('keepMe', cb);", + ].join('\n'); + + assert.deepEqual(extractActions(content), ['keepMe']); +}); diff --git a/infra/docs-gen/src/extractor/cli.test.mjs b/infra/docs-gen/src/extractor/cli.test.mjs new file mode 100644 index 000000000..02f8f9e4c --- /dev/null +++ b/infra/docs-gen/src/extractor/cli.test.mjs @@ -0,0 +1,17 @@ +import assert from 'node:assert/strict'; +import {test} from 'node:test'; + +import {parseArgs} from '../extract-extension-data.mjs'; + +test('parseArgs parses selected extensions', () => { + assert.deepEqual(parseArgs(['--only', 'Bold, Link']).only, ['Bold', 'Link']); +}); + +test('parseArgs stops option parsing after separator', () => { + assert.equal(parseArgs(['--help', '--', '--unknown']).help, true); +}); + +test('parseArgs rejects missing option values', () => { + assert.throws(() => parseArgs(['--out-dir']), /Missing value for --out-dir/u); + assert.throws(() => parseArgs(['--only', '--help']), /Missing value for --only/u); +}); diff --git a/infra/docs-gen/src/extractor/constants.mjs b/infra/docs-gen/src/extractor/constants.mjs new file mode 100644 index 000000000..597b571c9 --- /dev/null +++ b/infra/docs-gen/src/extractor/constants.mjs @@ -0,0 +1,245 @@ +import { + CONST_REF_RE, + ENUM_ENTRY_RE, + ENUM_START_RE, + IDENTIFIER_RE, + IDENTIFIER_VALUE_RE, + OBJECT_CONST_START_RE, + STRING_CONST_RE, + STRING_VALUE_RE, +} from './patterns.mjs'; +import {readBalanced} from './regex.mjs'; + +/** + * Resets shared global regex cursors. + */ +function resetConstantPatterns() { + for (const pattern of [ + STRING_CONST_RE, + ENUM_START_RE, + ENUM_ENTRY_RE, + OBJECT_CONST_START_RE, + CONST_REF_RE, + ]) { + pattern.lastIndex = 0; + } +} + +/** + * Extracts string-valued constants, enums, and scalar object members. + */ +export function extractConstants(content) { + resetConstantPatterns(); + + const names = new Map(); + let match; + + while ((match = STRING_CONST_RE.exec(content))) { + names.set(match[1], match[2]); + } + + while ((match = ENUM_START_RE.exec(content))) { + const enumName = match[1]; + const block = readBalanced(content, content.indexOf('{', match.index), '{', '}'); + if (!block) continue; + + ENUM_ENTRY_RE.lastIndex = 0; + const entries = block.body.matchAll(ENUM_ENTRY_RE); + for (const entry of entries) { + names.set(`${enumName}.${entry[1]}`, entry[2]); + } + + ENUM_START_RE.lastIndex = block.endIndex + 1; + } + + while ((match = OBJECT_CONST_START_RE.exec(content))) { + const objName = match[1]; + const braceIndex = content.indexOf('{', match.index); + const block = readBalanced(content, braceIndex, '{', '}'); + if (!block) continue; + + for (const [key, value] of extractTopLevelScalarProps(block.body)) { + names.set(`${objName}.${key}`, value); + } + + OBJECT_CONST_START_RE.lastIndex = block.endIndex + 1; + } + + const refs = []; + while ((match = CONST_REF_RE.exec(content))) { + refs.push([match[1], match[2]]); + } + + for (let pass = 0; pass < 3; pass++) { + for (const [target, source] of refs) { + if (!names.has(target) && names.has(source)) { + names.set(target, names.get(source)); + } + } + + for (const [key, value] of names) { + if (typeof value === 'string' && names.has(value) && key !== value) { + names.set(key, names.get(value)); + } + } + } + + return names; +} + +/** + * Resolves one raw identifier through the constants map. + */ +export function resolveConstant(raw, constants) { + if (!raw) return raw; + if (raw.startsWith("'") || raw.startsWith('"')) return raw.slice(1, -1); + + if (constants.has(raw)) { + const value = constants.get(raw); + return constants.has(value) ? constants.get(value) : value; + } + + for (const [key, value] of constants) { + if (key.endsWith(`.${raw}`) || key === raw) return value; + } + + return raw; +} + +/** + * Resolves a list of raw identifiers and expands constant namespaces. + */ +export function resolveAllConstants(rawList, constants) { + const resolved = []; + + for (const raw of rawList) { + let value = resolveConstant(raw, constants); + + if (value === raw && raw.includes('.') && constants.has(raw)) { + value = constants.get(raw); + } + + let depth = 0; + while (constants.has(value) && depth < 5) { + value = constants.get(value); + depth++; + } + + if (value === raw && constants.size > 0) { + const prefix = raw + '.'; + const members = []; + for (const [key, memberValue] of constants) { + if (!key.startsWith(prefix)) continue; + + let expandedValue = memberValue; + let memberDepth = 0; + while (constants.has(expandedValue) && memberDepth < 5) { + expandedValue = constants.get(expandedValue); + memberDepth++; + } + members.push(expandedValue); + } + + if (members.length > 0) { + resolved.push(...members); + continue; + } + } + + resolved.push(value); + } + + return [...new Set(resolved)]; +} + +/** + * Yields scalar properties from top-level object literal segments. + */ +function* extractTopLevelScalarProps(body) { + let parenDepth = 0; + let braceDepth = 0; + let bracketDepth = 0; + let quote = null; + let inLineComment = false; + let inBlockComment = false; + let segmentStart = 0; + const segments = []; + + for (let index = 0; index < body.length; index++) { + const char = body[index]; + const next = body[index + 1]; + + if (inLineComment) { + if (char === '\n') inLineComment = false; + continue; + } + if (inBlockComment) { + if (char === '*' && next === '/') { + inBlockComment = false; + index++; + } + continue; + } + if (quote) { + if (char === '\\') { + index++; + continue; + } + if (char === quote) quote = null; + continue; + } + if (char === '/' && next === '/') { + inLineComment = true; + index++; + continue; + } + if (char === '/' && next === '*') { + inBlockComment = true; + index++; + continue; + } + if (char === '"' || char === "'" || char === '`') { + quote = char; + continue; + } + if (char === '(') parenDepth++; + else if (char === ')') parenDepth--; + else if (char === '{') braceDepth++; + else if (char === '}') braceDepth--; + else if (char === '[') bracketDepth++; + else if (char === ']') bracketDepth--; + + if (char === ',' && parenDepth === 0 && braceDepth === 0 && bracketDepth === 0) { + segments.push(body.slice(segmentStart, index)); + segmentStart = index + 1; + } + } + + const tail = body.slice(segmentStart); + if (tail.trim()) segments.push(tail); + + for (const segment of segments) { + const trimmed = segment.trim(); + if (!trimmed || trimmed.startsWith('...')) continue; + + const colon = trimmed.indexOf(':'); + if (colon === -1) continue; + + const rawKey = trimmed.slice(0, colon).trim(); + if (!rawKey || rawKey.startsWith('[') || !IDENTIFIER_RE.test(rawKey)) { + continue; + } + + const rawValue = trimmed.slice(colon + 1).trim(); + const stringMatch = rawValue.match(STRING_VALUE_RE); + if (stringMatch) { + yield [rawKey, stringMatch[1]]; + continue; + } + + const identMatch = rawValue.match(IDENTIFIER_VALUE_RE); + if (identMatch) { + yield [rawKey, identMatch[1]]; + } + } +} diff --git a/infra/docs-gen/src/extractor/constants.test.mjs b/infra/docs-gen/src/extractor/constants.test.mjs new file mode 100644 index 000000000..30fbd40e0 --- /dev/null +++ b/infra/docs-gen/src/extractor/constants.test.mjs @@ -0,0 +1,47 @@ +import assert from 'node:assert/strict'; +import {test} from 'node:test'; + +import {extractConstants, resolveAllConstants} from './constants.mjs'; + +test('extractConstants captures top-level scalar object props with nested objects', () => { + const content = [ + 'export const CLASSNAMES = {', + " Inline: { Container: 'a', Sharp: 'b' },", + " Block: 'mathblock',", + " Display: 'display',", + '} as const;', + ].join('\n'); + + const map = extractConstants(content); + + assert.equal(map.get('CLASSNAMES.Block'), 'mathblock'); + assert.equal(map.get('CLASSNAMES.Display'), 'display'); +}); + +test('extractConstants handles enums after objects with nested braces', () => { + const content = [ + 'const NESTED = { Inner: { Foo: "bar" }, Outer: "value" };', + 'export enum NodeName { Para = "paragraph", Doc = "doc" }', + ].join('\n'); + + const map = extractConstants(content); + + assert.equal(map.get('NESTED.Outer'), 'value'); + assert.equal(map.get('NodeName.Para'), 'paragraph'); + assert.equal(map.get('NodeName.Doc'), 'doc'); +}); + +test('extractConstants resolves identifier-valued props through reference chains', () => { + const content = ["const NAME = 'bold';", 'export const Specs = { Bold: NAME };'].join('\n'); + + const map = extractConstants(content); + + assert.equal(map.get('Specs.Bold'), 'bold'); +}); + +test('resolveAllConstants expands a prefix into all known members', () => { + const content = 'enum NodeName { Para = "paragraph", Doc = "doc" }'; + const map = extractConstants(content); + + assert.deepEqual(resolveAllConstants(['NodeName'], map).sort(), ['doc', 'paragraph']); +}); diff --git a/infra/docs-gen/src/extractor/examples.mjs b/infra/docs-gen/src/extractor/examples.mjs new file mode 100644 index 000000000..241e9759c --- /dev/null +++ b/infra/docs-gen/src/extractor/examples.mjs @@ -0,0 +1,242 @@ +import {IDENTIFIER_RE, SAME_CALL_RE, STRING_BINDING_RE} from './patterns.mjs'; +import { + readBalanced, + readExpression, + resetPattern, + splitTopLevel, + splitTopLevelBy, +} from './regex.mjs'; + +const STRING_ESCAPES = new Map([ + ['\\', '\\'], + ["'", "'"], + ['"', '"'], + ['`', '`'], + ['n', '\n'], + ['r', '\r'], + ['t', '\t'], + ['b', '\b'], + ['f', '\f'], + ['v', '\v'], +]); + +/** + * Decodes simple JavaScript string escapes. + */ +function decodeStringEscapes(content) { + return content.replace(/\\([\s\S])/gu, (_, char) => STRING_ESCAPES.get(char) ?? char); +} + +/** + * Resolves a quoted string literal. + */ +function resolveQuotedString(expression) { + const trimmed = expression.trim(); + const quote = trimmed[0]; + if ((quote !== "'" && quote !== '"') || trimmed.at(-1) !== quote) return null; + + return decodeStringEscapes(trimmed.slice(1, -1)); +} + +/** + * Applies a known string method to a value. + */ +function applyStringMethod(value, method) { + if (method === 'trim') return value.trim(); + if (method === 'trimStart') return value.trimStart(); + if (method === 'trimEnd') return value.trimEnd(); + return value; +} + +/** + * Removes one trailing string method call. + */ +function peelStringMethod(expression) { + const match = /^(?[\s\S]+)\.(?trim|trimStart|trimEnd)\(\)\s*$/u.exec( + expression.trim(), + ); + + return match?.groups || null; +} + +/** + * Resolves chained string trim calls. + */ +function resolveStringMethodChain(expression, bindings) { + const peeled = peelStringMethod(expression); + if (!peeled) return null; + + const value = resolveStringExpression(peeled.base, bindings); + return value === null ? null : applyStringMethod(value, peeled.method); +} + +/** + * Resolves a template literal expression. + */ +function resolveTemplateString(expression, bindings) { + const trimmed = expression.trim(); + if (!trimmed.startsWith('`') || !trimmed.endsWith('`')) return null; + + let result = ''; + for (let index = 1; index < trimmed.length - 1; index++) { + const char = trimmed[index]; + const next = trimmed[index + 1]; + + if (char === '\\') { + result += decodeStringEscapes(trimmed.slice(index, index + 2)); + index++; + continue; + } + + if (char === '$' && next === '{') { + const interpolation = readBalanced(trimmed, index + 1, '{', '}'); + if (!interpolation) return null; + + const value = resolveStringExpression(interpolation.body, bindings); + if (value === null) return null; + + result += value; + index = interpolation.endIndex; + continue; + } + + result += char; + } + + return result; +} + +/** + * Resolves an array join expression. + */ +function resolveArrayJoinExpression(expression, bindings) { + const trimmed = expression.trim(); + if (!trimmed.startsWith('[')) return null; + + const arrayBody = readBalanced(trimmed, 0, '[', ']'); + if (!arrayBody) return null; + + const tail = trimmed.slice(arrayBody.endIndex + 1).trim(); + if (!tail.startsWith('.join(')) return null; + + const joinArgs = readBalanced(tail, '.join'.length, '(', ')'); + if (!joinArgs || joinArgs.endIndex !== tail.length - 1) return null; + + const delimiterExpression = joinArgs.body.trim() || "','"; + const delimiter = resolveStringExpression(delimiterExpression, bindings); + if (delimiter === null) return null; + + const parts = splitTopLevel(arrayBody.body).map((part) => + resolveStringExpression(part, bindings), + ); + if (parts.some((part) => part === null)) return null; + + return parts.join(delimiter); +} + +/** + * Resolves a concatenated string expression. + */ +function resolveStringConcatenation(expression, bindings) { + const parts = splitTopLevelBy(expression, ['+']); + if (parts.length < 2) return null; + + const values = parts.map((part) => resolveStringExpression(part, bindings)); + if (values.some((value) => value === null)) return null; + + return values.join(''); +} + +/** + * Resolves a string identifier from known bindings. + */ +function resolveStringIdentifier(expression, bindings) { + const trimmed = expression.trim(); + return IDENTIFIER_RE.test(trimmed) ? (bindings.get(trimmed) ?? null) : null; +} + +/** + * Removes wrapping parentheses from a string expression. + */ +function unwrapStringExpression(expression) { + const trimmed = expression.trim(); + if (!trimmed.startsWith('(')) return trimmed; + + const body = readBalanced(trimmed, 0, '(', ')'); + if (!body || body.endIndex !== trimmed.length - 1) return trimmed; + + return unwrapStringExpression(body.body); +} + +/** + * Resolves supported string expressions. + */ +function resolveStringExpression(expression, bindings) { + const trimmed = unwrapStringExpression(expression); + + return ( + resolveStringMethodChain(trimmed, bindings) ?? + resolveArrayJoinExpression(trimmed, bindings) ?? + resolveStringConcatenation(trimmed, bindings) ?? + resolveTemplateString(trimmed, bindings) ?? + resolveQuotedString(trimmed) ?? + resolveStringIdentifier(trimmed, bindings) + ); +} + +/** + * Collects resolvable local string bindings. + */ +function collectStringBindings(content) { + const bindings = new Map(); + const re = resetPattern(STRING_BINDING_RE); + let match; + + while ((match = re.exec(content))) { + const expression = readExpression(content, re.lastIndex, [';']); + const value = resolveStringExpression(expression.body, bindings); + + if (value !== null) { + bindings.set(match[1], value); + } + + re.lastIndex = expression.endIndex + 1; + } + + return bindings; +} + +/** + * Extracts first arguments from same(...) calls. + */ +function extractSameFirstArguments(content) { + const args = []; + const re = resetPattern(SAME_CALL_RE); + let match; + + while ((match = re.exec(content))) { + const openIndex = content.indexOf('(', match.index); + const callBody = readBalanced(content, openIndex, '(', ')'); + if (!callBody) continue; + + const [firstArg] = splitTopLevel(callBody.body); + if (firstArg) { + args.push({expression: firstArg, index: match.index}); + } + + re.lastIndex = callBody.endIndex + 1; + } + + return args; +} + +/** + * Extracts markdown examples from serializer test helpers. + */ +export function extractTestExamples(content) { + return extractSameFirstArguments(content) + .map(({expression, index}) => + resolveStringExpression(expression, collectStringBindings(content.slice(0, index))), + ) + .filter((example) => example !== null); +} diff --git a/infra/docs-gen/src/extractor/examples.test.mjs b/infra/docs-gen/src/extractor/examples.test.mjs new file mode 100644 index 000000000..82bf82bd0 --- /dev/null +++ b/infra/docs-gen/src/extractor/examples.test.mjs @@ -0,0 +1,46 @@ +import assert from 'node:assert/strict'; +import {readFileSync} from 'node:fs'; +import {test} from 'node:test'; + +import {extractTestExamples} from './examples.mjs'; + +/** + * Reads a repository file as UTF-8 text. + */ +function readRepoFile(relativePath) { + return readFileSync(new URL(`../../../../${relativePath}`, import.meta.url), 'utf-8'); +} + +test('extractTestExamples resolves literals, local bindings, joins, and template bindings', () => { + const content = [ + "const formula = 'x + y';", + 'const markup = `', + '# title', + '`.trimStart();', + "same(['Term', ': Description'].join('\\n'), doc());", + 'checker.same(markup, doc());', + 'same(`$$${formula}$$\\n\\n`, doc());', + ].join('\n'); + + assert.deepEqual(extractTestExamples(content), [ + 'Term\n: Description', + '# title\n', + '$$x + y$$\n\n', + ]); +}); + +test('extractTestExamples captures joined Deflist markup', () => { + const examples = extractTestExamples( + readRepoFile('packages/editor/src/extensions/markdown/Deflist/Deflist.test.ts'), + ); + + assert.ok(examples.includes('Term\n: Description')); +}); + +test('extractTestExamples captures markup stored in local constants', () => { + const examples = extractTestExamples( + readRepoFile('packages/editor/src/extensions/yfm/YfmTable/YfmTable.test.ts'), + ); + + assert.ok(examples.some((example) => example.includes('nested table'))); +}); diff --git a/infra/docs-gen/src/extractor/index.mjs b/infra/docs-gen/src/extractor/index.mjs new file mode 100644 index 000000000..23aefa3a7 --- /dev/null +++ b/infra/docs-gen/src/extractor/index.mjs @@ -0,0 +1,86 @@ +import {existsSync, mkdirSync, rmSync} from 'node:fs'; +import {join} from 'node:path'; + +import {EXTENSION_CATEGORIES, isInternalExtension} from '../config.mjs'; +import {logger} from '../logger.mjs'; +import {listDirs, readText} from '../utils.mjs'; + +import {writeExtensionsJson, writeRawMarkdownFiles} from './output.mjs'; +import {getPresetsForExtension, parsePresets} from './presets.mjs'; +import {scanExtension} from './scan.mjs'; + +export class ExtensionExtractor { + /** + * Creates an extension extractor for editor source paths. + */ + constructor({editorPkg, outDir, repoRoot}) { + this.editorPkg = editorPkg; + this.repoRoot = repoRoot; + this.extensionsDir = join(editorPkg, 'src/extensions'); + this.presetsDir = join(editorPkg, 'src/presets'); + this.outDir = outDir; + this.rawDir = join(outDir, 'raw'); + } + + /** + * Scans one extension directory into raw metadata. + */ + scan(extDir, category) { + return scanExtension({extDir, category, repoRoot: this.repoRoot}); + } + + /** + * Scans all configured extension categories. + */ + scanAll({only} = {}) { + const onlySet = only?.length ? new Set(only) : null; + const extensions = []; + + for (const category of EXTENSION_CATEGORIES) { + const categoryDir = join(this.extensionsDir, category); + for (const dirName of listDirs(categoryDir)) { + if (isInternalExtension(dirName) || (onlySet && !onlySet.has(dirName))) continue; + + extensions.push(this.scan(join(categoryDir, dirName), category)); + } + } + + return extensions; + } + + /** + * Writes extracted JSON IR and raw Markdown files. + */ + run({only} = {}) { + logger.info('Extracting raw extension data...'); + + if (existsSync(this.rawDir)) { + rmSync(this.rawDir, {recursive: true, force: true}); + } + mkdirSync(this.rawDir, {recursive: true}); + + const version = this.getEditorVersion(); + const presetMap = parsePresets(this.presetsDir); + const extensions = this.scanAll({only}); + + for (const extension of extensions) { + extension.presets = getPresetsForExtension(presetMap, extension.name); + } + + writeExtensionsJson(this.outDir, version, extensions); + writeRawMarkdownFiles(this.rawDir, extensions, presetMap, version); + + logger.success(`Raw data written to ${this.outDir}`); + logger.info(`Extensions: ${extensions.length}`); + + return {version, extensions}; + } + + /** + * Reads the editor package version. + */ + getEditorVersion() { + const pkg = JSON.parse(readText(join(this.editorPkg, 'package.json'))); + return pkg.version; + } +} diff --git a/infra/docs-gen/src/extractor/keymaps.test.mjs b/infra/docs-gen/src/extractor/keymaps.test.mjs new file mode 100644 index 000000000..941fdc251 --- /dev/null +++ b/infra/docs-gen/src/extractor/keymaps.test.mjs @@ -0,0 +1,64 @@ +import assert from 'node:assert/strict'; +import {readFileSync} from 'node:fs'; +import {test} from 'node:test'; + +import {extractKeymaps} from './regex.mjs'; + +/** + * Reads a repository file as UTF-8 text. + */ +function readRepoFile(relativePath) { + return readFileSync(new URL(`../../../../${relativePath}`, import.meta.url), 'utf-8'); +} + +test('extractKeymaps handles direct object returns and ignores computed keys', () => { + const content = [ + 'builder.addKeymap(() => ({', + ' Tab: handleTab,', + " 'Shift-Tab': handleShiftTab,", + ' [dynamicKey]: ignoreMe,', + '}));', + ].join('\n'); + + assert.deepEqual(extractKeymaps(content), ['Tab', 'Shift-Tab']); +}); + +test('extractKeymaps merges returned object literals with spread bindings', () => { + const content = [ + 'builder.addKeymap(() => {', + ' const bindings: Keymap = {Backspace: resetHeading};', + ' return {', + ' Tab: handleTab,', + ' ...bindings,', + " 'Shift-Tab': handleShiftTab,", + ' };', + '});', + ].join('\n'); + + assert.deepEqual(extractKeymaps(content), ['Tab', 'Backspace', 'Shift-Tab']); +}); + +test('extractKeymaps captures static keymaps from the Lists extension', () => { + const content = readRepoFile('packages/editor/src/extensions/markdown/Lists/index.ts'); + + assert.deepEqual(extractKeymaps(content), [ + 'Tab', + 'Shift-Tab', + 'Backspace', + 'Mod-[', + 'Mod-]', + 'Enter', + ]); +}); + +test('extractKeymaps captures static block-body bindings and ignores dynamic ones', () => { + const headingContent = readRepoFile('packages/editor/src/extensions/markdown/Heading/index.ts'); + const historyContent = readRepoFile('packages/editor/src/extensions/behavior/History/index.ts'); + const editorModeContent = readRepoFile( + 'packages/editor/src/extensions/behavior/EditorModeKeymap/index.ts', + ); + + assert.deepEqual(extractKeymaps(headingContent), ['Backspace']); + assert.deepEqual(extractKeymaps(historyContent), []); + assert.deepEqual(extractKeymaps(editorModeContent), []); +}); diff --git a/infra/docs-gen/src/extractor/markdown-gen.mjs b/infra/docs-gen/src/extractor/markdown-gen.mjs new file mode 100644 index 000000000..310348a3e --- /dev/null +++ b/infra/docs-gen/src/extractor/markdown-gen.mjs @@ -0,0 +1,87 @@ +import {getPresetsForExtension} from './presets.mjs'; + +/** + * Formats a value as inline Markdown code. + */ +function code(value) { + return `\`${String(value).replace(/\|/g, '\\|').replace(/\n/g, '\\n')}\``; +} + +/** + * Adds a Markdown list section when values are present. + */ +function addList(lines, title, values) { + if (values.length === 0) return; + + lines.push(`## ${title}`, ''); + for (const value of values) { + lines.push(`- ${code(value)}`); + } + lines.push(''); +} + +/** + * Generates a raw Markdown page for extracted extension data. + */ +export function generateRawMd(extension, presetMap, version) { + const presets = getPresetsForExtension(presetMap, extension.name); + const lines = [ + '---', + `extension: ${extension.name}`, + `version: ${version}`, + `category: ${extension.category}`, + `source: ${extension.sourcePath}`, + '---', + '', + `# ${extension.name}`, + '', + '## Source', + '', + `- ${code(extension.sourcePath)}`, + '', + '## Presets', + '', + ]; + + if (presets.length > 0) { + for (const preset of presets) lines.push(`- ${preset}`); + } else { + lines.push('Not included in standard presets.'); + } + lines.push(''); + + if (extension.nodes.length > 0 || extension.marks.length > 0) { + lines.push('## Schema', ''); + for (const node of extension.nodes) { + lines.push(`- Node: ${code(node)}`); + } + for (const mark of extension.marks) { + lines.push(`- Mark: ${code(mark)}`); + } + lines.push(''); + } + + addList(lines, 'Actions', extension.actions); + addList(lines, 'Keymaps', extension.keymaps); + addList(lines, 'Input Rules', extension.inputRules); + addList(lines, 'Markdown-It Plugins', extension.mdPlugins); + addList(lines, 'ProseMirror Plugins', extension.plugins); + addList(lines, 'Serializer Hints', extension.serializerHints); + + if (extension.options.length > 0) { + lines.push('## Options', '', '| Option | Type |', '|--------|------|'); + for (const option of extension.options) { + lines.push(`| ${code(option.name)} | ${code(option.type)} |`); + } + lines.push(''); + } + + if (extension.markupExamples.length > 0) { + lines.push('## Markup Examples', ''); + for (const example of extension.markupExamples.slice(0, 10)) { + lines.push('```markdown', example, '```', ''); + } + } + + return lines.join('\n'); +} diff --git a/infra/docs-gen/src/extractor/options.mjs b/infra/docs-gen/src/extractor/options.mjs new file mode 100644 index 000000000..22c171e8e --- /dev/null +++ b/infra/docs-gen/src/extractor/options.mjs @@ -0,0 +1,381 @@ +import {OPTIONS_DECL_RE, OPTION_FIELD_RE} from './patterns.mjs'; +import { + readBalanced, + readExpression, + resetPattern, + skipWhitespace, + splitTopLevelBy, +} from './regex.mjs'; + +/** + * Normalizes whitespace in extracted type snippets. + */ +function normalizeWhitespace(content) { + return content.trim().replace(/\s+/g, ' '); +} + +/** + * Removes TypeScript comments from a declaration body. + */ +function stripTypeComments(content) { + return content.replace(/\/\*[\s\S]*?\*\//g, '').replace(/\/\/.*$/gm, ''); +} + +/** + * Trims syntax left by top-level field splitting. + */ +function trimFieldType(content) { + return normalizeWhitespace(content.replace(/[;,]\s*$/u, '')); +} + +/** + * Parses fields from an inline object type body. + */ +function parseOptionFieldsFromBody(body) { + const fields = []; + const cleanBody = stripTypeComments(body); + + for (const segment of splitTopLevelBy(cleanBody, [';', ','], {trackAngles: true})) { + const fieldMatch = OPTION_FIELD_RE.exec(segment.trim()); + if (!fieldMatch) continue; + + fields.push({ + name: fieldMatch[1], + type: trimFieldType(fieldMatch[2]), + }); + } + + return fields; +} + +/** + * Creates a type declaration record. + */ +function createOptionDeclaration(name, kind, expression, body, endIndex) { + return { + name, + kind, + expression, + body, + endIndex, + }; +} + +/** + * Reads an exported interface declaration. + */ +function readInterfaceDeclaration(content, name, cursor) { + const bodyStart = content.indexOf('{', cursor); + if (bodyStart === -1) return null; + + const body = readBalanced(content, bodyStart, '{', '}'); + if (!body) return null; + + return createOptionDeclaration( + name, + 'interface', + content.slice(cursor, bodyStart).trim(), + body.body, + body.endIndex, + ); +} + +/** + * Finds the alias assignment after optional generic parameters. + */ +function findTypeAliasEquals(content, cursor) { + let angleDepth = 0; + let quote = null; + let inBlockComment = false; + let inLineComment = false; + + for (let index = cursor; index < content.length; index++) { + const char = content[index]; + const next = content[index + 1]; + + if (inLineComment) { + if (char === '\n') inLineComment = false; + continue; + } + + if (inBlockComment) { + if (char === '*' && next === '/') { + inBlockComment = false; + index++; + } + continue; + } + + if (quote) { + if (char === '\\') { + index++; + continue; + } + if (char === quote) quote = null; + continue; + } + + if (char === '/' && next === '/') { + inLineComment = true; + index++; + continue; + } + + if (char === '/' && next === '*') { + inBlockComment = true; + index++; + continue; + } + + if (char === '"' || char === "'" || char === '`') { + quote = char; + continue; + } + + if (char === '<') angleDepth++; + else if (char === '>' && content[index - 1] !== '=' && angleDepth > 0) angleDepth--; + else if (char === '=' && angleDepth === 0) return index; + else if (char === ';' && angleDepth === 0) return -1; + } + + return -1; +} + +/** + * Reads an exported type alias declaration. + */ +function readTypeDeclaration(content, name, cursor) { + const equalsIndex = findTypeAliasEquals(content, cursor); + if (equalsIndex === -1) return null; + + const expressionStart = skipWhitespace(content, equalsIndex + 1); + const expression = readExpression(content, expressionStart, [';']); + return createOptionDeclaration(name, 'type', expression.body, null, expression.endIndex); +} + +/** + * Reads an exported options declaration. + */ +function readOptionDeclaration(content, match) { + const [, kind, name] = match; + const cursor = match.index + match[0].length; + + return kind === 'interface' + ? readInterfaceDeclaration(content, name, cursor) + : readTypeDeclaration(content, name, cursor); +} + +/** + * Parses exported *Options declarations. + */ +function parseOptionDeclarations(content) { + const declarations = new Map(); + const re = resetPattern(OPTIONS_DECL_RE); + let match; + + while ((match = re.exec(content))) { + const declaration = readOptionDeclaration(content, match); + if (!declaration) continue; + + declarations.set(declaration.name, declaration); + re.lastIndex = declaration.endIndex + 1; + } + + return declarations; +} + +/** + * Deduplicates option fields by name. + */ +function uniqueOptionFields(fields) { + const result = new Map(); + + for (const field of fields) { + if (!result.has(field.name)) { + result.set(field.name, field); + } + } + + return [...result.values()]; +} + +/** + * Removes wrapping parentheses from a type expression. + */ +function unwrapTypeExpression(expression) { + const trimmed = expression.trim(); + if (!trimmed.startsWith('(')) return trimmed; + + const body = readBalanced(trimmed, 0, '(', ')'); + if (!body || body.endIndex !== trimmed.length - 1) return trimmed; + + return unwrapTypeExpression(body.body); +} + +/** + * Reads an inline object type expression. + */ +function readObjectTypeFields(expression) { + const trimmed = expression.trim(); + if (!trimmed.startsWith('{')) return null; + + const body = readBalanced(trimmed, 0, '{', '}'); + if (!body) return null; + + return parseOptionFieldsFromBody(body.body); +} + +/** + * Extracts a plain type reference name. + */ +function extractTypeReferenceName(expression) { + const match = /^([A-Za-z_$][\w$]*)\b(?:<[\s\S]*>)?$/u.exec(expression.trim()); + return match?.[1] || null; +} + +/** + * Parses utility type arguments. + */ +function parseUtilityTypeArguments(expression, utilityName) { + const trimmed = expression.trim(); + const prefix = `${utilityName}<`; + if (!trimmed.startsWith(prefix)) return null; + + const body = readBalanced(trimmed, utilityName.length, '<', '>'); + if (!body || body.endIndex !== trimmed.length - 1) return null; + + return splitTopLevelBy(body.body, [','], {trackAngles: true}); +} + +/** + * Extracts string literal names from a union type. + */ +function parseFieldNameUnion(expression) { + return splitTopLevelBy(expression, ['|'], {trackAngles: true}) + .map((part) => part.trim()) + .map((part) => { + const match = /^['"]([^'"]+)['"]$/u.exec(part); + return match?.[1] || null; + }) + .filter(Boolean); +} + +/** + * Filters fields to names from a utility type. + */ +function pickOptionFields(fields, names) { + const allowedNames = new Set(names); + return fields.filter((field) => allowedNames.has(field.name)); +} + +/** + * Excludes fields by names from a utility type. + */ +function omitOptionFields(fields, names) { + const omittedNames = new Set(names); + return fields.filter((field) => !omittedNames.has(field.name)); +} + +/** + * Resolves a Pick type expression. + */ +function resolvePickExpression(expression, declarations, seen) { + const args = parseUtilityTypeArguments(expression, 'Pick'); + if (!args || args.length < 2) return null; + + const fields = resolveTypeExpression(args[0], declarations, seen); + return pickOptionFields(fields, parseFieldNameUnion(args[1])); +} + +/** + * Resolves an Omit type expression. + */ +function resolveOmitExpression(expression, declarations, seen) { + const args = parseUtilityTypeArguments(expression, 'Omit'); + if (!args || args.length < 2) return null; + + const fields = resolveTypeExpression(args[0], declarations, seen); + return omitOptionFields(fields, parseFieldNameUnion(args[1])); +} + +/** + * Resolves a type reference to another local declaration. + */ +function resolveTypeReference(expression, declarations, seen) { + const typeName = extractTypeReferenceName(expression); + if (!typeName || !declarations.has(typeName)) return null; + + return resolveOptionDeclaration(typeName, declarations, seen); +} + +/** + * Resolves supported TypeScript option type expressions. + */ +function resolveTypeExpression(expression, declarations, seen) { + const trimmed = unwrapTypeExpression(expression); + if (!trimmed) return []; + + const intersectionParts = splitTopLevelBy(trimmed, ['&'], {trackAngles: true}); + if (intersectionParts.length > 1) { + return intersectionParts.flatMap((part) => resolveTypeExpression(part, declarations, seen)); + } + + return ( + readObjectTypeFields(trimmed) || + resolvePickExpression(trimmed, declarations, seen) || + resolveOmitExpression(trimmed, declarations, seen) || + resolveTypeReference(trimmed, declarations, seen) || + [] + ); +} + +/** + * Resolves an interface extends clause. + */ +function resolveInterfaceExtends(expression, declarations, seen) { + const trimmed = expression.trim(); + if (!trimmed.startsWith('extends ')) return []; + + return resolveTypeExpression(trimmed.slice('extends '.length), declarations, seen); +} + +/** + * Resolves fields from a declaration by name. + */ +function resolveOptionDeclaration(name, declarations, seen = new Set()) { + if (seen.has(name)) return []; + seen.add(name); + + const declaration = declarations.get(name); + if (!declaration) return []; + + const inheritedFields = + declaration.kind === 'interface' + ? resolveInterfaceExtends(declaration.expression, declarations, seen) + : resolveTypeExpression(declaration.expression, declarations, seen); + const ownFields = declaration.body ? parseOptionFieldsFromBody(declaration.body) : []; + + return uniqueOptionFields([...inheritedFields, ...ownFields]); +} + +/** + * Selects preferred option declaration names. + */ +function selectOptionNames(declarations, preferredNames) { + const existingPreferredNames = preferredNames.filter((name) => declarations.has(name)); + return existingPreferredNames.length > 0 ? existingPreferredNames : [...declarations.keys()]; +} + +/** + * Extracts exported extension options fields. + */ +export function extractOptionsType(content, preferredNames = []) { + const declarations = parseOptionDeclarations(content); + + for (const name of selectOptionNames(declarations, preferredNames)) { + const fields = resolveOptionDeclaration(name, declarations); + if (fields.length > 0) return fields; + } + + return []; +} diff --git a/infra/docs-gen/src/extractor/options.test.mjs b/infra/docs-gen/src/extractor/options.test.mjs new file mode 100644 index 000000000..6531db73e --- /dev/null +++ b/infra/docs-gen/src/extractor/options.test.mjs @@ -0,0 +1,79 @@ +import assert from 'node:assert/strict'; +import {readFileSync} from 'node:fs'; +import {test} from 'node:test'; + +import {extractOptionsType} from './options.mjs'; + +/** + * Reads a repository file as UTF-8 text. + */ +function readRepoFile(relativePath) { + return readFileSync(new URL(`../../../../${relativePath}`, import.meta.url), 'utf-8'); +} + +/** + * Maps option fields by name. + */ +function mapFields(fields) { + return new Map(fields.map((field) => [field.name, field.type])); +} + +test('extractOptionsType resolves local aliases', () => { + const content = [ + readRepoFile('packages/editor/src/extensions/markdown/Image/index.ts'), + readRepoFile('packages/editor/src/extensions/markdown/Image/imageUrlPaste/index.ts'), + ].join('\n'); + + assert.deepEqual(extractOptionsType(content, ['ImageOptions']), [ + {name: 'parseInsertedUrlAsImage', type: 'ParseInsertedUrlAsImage'}, + ]); +}); + +test('extractOptionsType resolves Pick utility types', () => { + const content = [ + readRepoFile('packages/editor/src/extensions/behavior/Clipboard/index.ts'), + readRepoFile('packages/editor/src/extensions/behavior/Clipboard/clipboard.ts'), + ].join('\n'); + + assert.deepEqual(extractOptionsType(content, ['ClipboardOptions']), [ + {name: 'pasteFileHandler', type: '(file: File) => void'}, + ]); +}); + +test('extractOptionsType keeps nested object option types intact', () => { + const fields = mapFields( + extractOptionsType( + readRepoFile('packages/editor/src/extensions/additional/Mermaid/index.ts'), + ['MermaidOptions'], + ), + ); + + assert.equal(fields.get('autoSave'), '{ enabled: boolean; delay?: number; }'); + assert.equal( + fields.get('theme'), + "{ dark: MermaidConfig['theme']; light: MermaidConfig['theme']; }", + ); +}); + +test('extractOptionsType reads interface declarations with local fields', () => { + const fields = mapFields( + extractOptionsType( + readRepoFile('packages/editor/src/extensions/additional/YfmHtmlBlock/index.ts'), + ['YfmHtmlBlockOptions'], + ), + ); + + assert.equal(fields.get('useConfig'), '() => IHTMLIFrameElementConfig | undefined'); + assert.equal(fields.get('autoSave'), '{ enabled: boolean; delay?: number; }'); +}); + +test('extractOptionsType merges intersections without corrupting nested fields', () => { + const content = [ + readRepoFile('packages/editor/src/extensions/markdown/CodeBlock/index.ts'), + readRepoFile('packages/editor/src/extensions/markdown/CodeBlock/CodeBlockSpecs/index.ts'), + ].join('\n'); + const fields = mapFields(extractOptionsType(content, ['CodeBlockOptions'])); + + assert.equal(fields.get('codeBlockKey'), 'string | null'); + assert.equal(fields.get('lineWrapping'), '{ enabled?: boolean; }'); +}); diff --git a/infra/docs-gen/src/extractor/output.mjs b/infra/docs-gen/src/extractor/output.mjs new file mode 100644 index 000000000..d4d021ca2 --- /dev/null +++ b/infra/docs-gen/src/extractor/output.mjs @@ -0,0 +1,26 @@ +import {writeFileSync} from 'node:fs'; +import {join} from 'node:path'; + +import {generateRawMd} from './markdown-gen.mjs'; + +/** + * Writes extension IR as JSON. + */ +export function writeExtensionsJson(outDir, version, extensions) { + writeFileSync( + join(outDir, 'extensions.json'), + JSON.stringify({version, extensions}, null, 2) + '\n', + ); +} + +/** + * Writes raw Markdown files for extensions. + */ +export function writeRawMarkdownFiles(rawDir, extensions, presetMap, version) { + for (const extension of extensions) { + writeFileSync( + join(rawDir, `${extension.name}.md`), + generateRawMd(extension, presetMap, version), + ); + } +} diff --git a/infra/docs-gen/src/extractor/patterns.mjs b/infra/docs-gen/src/extractor/patterns.mjs new file mode 100644 index 000000000..d736e3d45 --- /dev/null +++ b/infra/docs-gen/src/extractor/patterns.mjs @@ -0,0 +1,35 @@ +export const BUILDER_ADD_NODE_RE = + /builder\s*\.addNode\(\s*(?:(\w+\.\w+)|(\w+)|['"]([^'"]+)['"])\s*,\s*(?:\(|$)/gm; +export const CHAINED_ADD_NODE_RE = + /\)\s*\.addNode\(\s*(?:(\w+\.\w+)|(\w+)|['"]([^'"]+)['"])\s*,\s*(?:\(|$)/gm; +export const BUILDER_ADD_MARK_RE = + /builder\s*\.addMark\(\s*\n?\s*(?:(\w+\.\w+)|(\w+)|['"]([^'"]+)['"])\s*,/g; +export const CHAINED_ADD_MARK_RE = + /\)\s*\n\s*\.addMark\(\s*\n?\s*(?:(\w+\.\w+)|(\w+)|['"]([^'"]+)['"])\s*,/g; +export const NODE_SPEC_RE = /\.addNodeSpec\(\s*\{\s*name:\s*(?:(\w+\.\w+)|(\w+)|['"]([^'"]+)['"])/g; +export const MARK_SPEC_RE = /\.addMarkSpec\(\s*\{\s*name:\s*(?:(\w+\.\w+)|(\w+)|['"]([^'"]+)['"])/g; +export const BUILDER_ADD_ACTION_RE = + /builder\s*\.addAction\(\s*(?:(\w+\.\w+)|(\w+)|['"]([^'"]+)['"])/g; +export const CHAINED_ADD_ACTION_RE = /\)\s*\.addAction\(\s*(?:(\w+\.\w+)|(\w+)|['"]([^'"]+)['"])/g; +export const ADD_PLUGIN_RE = /\.addPlugin\(\s*(\w+)/g; +export const INPUT_RULE_RE = + /(?:markInputRule|textblockTypeInputRule|nodeInputRule|wrappingInputRule|inlineNodeInputRule)\s*\(\s*(?:\/([^/]+)\/|{[^}]*open:\s*'([^']*)'[^}]*close:\s*'([^']*)'[^}]*})/g; +export const MD_PLUGIN_RE = /md\.use\(\s*(\w+)/g; +export const OPTIONS_DECL_RE = /export\s+(type|interface)\s+(\w+Options)\b/g; +export const OPTION_FIELD_RE = /^(?:readonly\s+)?([A-Za-z_$][\w$]*)\??\s*:\s*(.+)$/s; +export const SAME_CALL_RE = /\bsame\s*\(/g; +export const STRING_BINDING_RE = /(?:const|let)\s+([A-Za-z_$][\w$]*)(?::[^=;]+)?\s*=\s*/g; +export const STATE_WRITE_RE = /state\.write\(\s*[`'"]([^`'"]*)[`'"]/g; +export const STATE_TEXT_RE = /state\.text\(\s*[`'"]([^`'"]*)[`'"]/g; +export const STRING_CONST_RE = /(?:export\s+)?const\s+(\w+)\s*=\s*['"]([^'"]+)['"]/g; +export const ENUM_START_RE = /(?:export\s+)?enum\s+(\w+)\s*\{/g; +export const ENUM_ENTRY_RE = /(\w+)\s*=\s*['"]([^'"]+)['"]/g; +export const OBJECT_CONST_START_RE = /(?:export\s+)?const\s+(\w+)\s*=\s*\{/g; +export const CONST_REF_RE = /(?:export\s+)?const\s+(\w+)\s*=\s*(\w+)\s*;/g; +export const IDENTIFIER_RE = /^[A-Za-z_$][\w$]*$/; +export const IDENTIFIER_VALUE_RE = /^([A-Za-z_$][\w$.]*)/; +export const STRING_VALUE_RE = /^['"]([^'"]*)['"]/; +export const KEYMAP_OBJECT_DECL_RE = /(?:const|let|var)\s+(\w+)(?::[^=;]+)?=\s*\{/g; +export const STATIC_COMPUTED_ASSIGNMENT_RE = /(\w+)\s*\[\s*(?:'([^']+)'|"([^"]+)")\s*\]\s*=/g; +export const STATIC_MEMBER_ASSIGNMENT_RE = /(\w+)\.(\w+)\s*=/g; +export const PRESET_USE_RE = /\.use\(\s*(\w+)/g; diff --git a/infra/docs-gen/src/extractor/presets.mjs b/infra/docs-gen/src/extractor/presets.mjs new file mode 100644 index 000000000..90f6201b4 --- /dev/null +++ b/infra/docs-gen/src/extractor/presets.mjs @@ -0,0 +1,51 @@ +import {existsSync} from 'node:fs'; +import {join} from 'node:path'; + +import {PRESET_DEFS} from '../config.mjs'; +import {readText} from '../utils.mjs'; + +import {PRESET_USE_RE} from './patterns.mjs'; + +/** + * Parses preset files into inherited extension membership. + */ +export function parsePresets(presetsDir) { + const presetMap = new Map(); + + for (const def of PRESET_DEFS) { + const filePath = join(presetsDir, def.file); + if (!existsSync(filePath)) continue; + + const content = readText(filePath); + const directUses = []; + let match; + + PRESET_USE_RE.lastIndex = 0; + while ((match = PRESET_USE_RE.exec(content))) { + const extensionName = match[1]; + if (!extensionName.endsWith('Preset') && !extensionName.endsWith('Specs')) { + directUses.push(extensionName); + } + } + + const inherited = def.parent ? presetMap.get(def.parent) || [] : []; + presetMap.set(def.name, [...new Set([...inherited, ...directUses])]); + } + + return presetMap; +} + +/** + * Finds presets that include an extension. + */ +export function getPresetsForExtension(presetMap, extensionName) { + const presets = []; + + for (const [presetName, extensions] of presetMap) { + if (extensions.includes(extensionName)) { + presets.push(presetName); + } + } + + return presets; +} diff --git a/infra/docs-gen/src/extractor/regex.mjs b/infra/docs-gen/src/extractor/regex.mjs new file mode 100644 index 000000000..9ae88f139 --- /dev/null +++ b/infra/docs-gen/src/extractor/regex.mjs @@ -0,0 +1,708 @@ +import { + ADD_PLUGIN_RE, + BUILDER_ADD_ACTION_RE, + BUILDER_ADD_MARK_RE, + BUILDER_ADD_NODE_RE, + CHAINED_ADD_ACTION_RE, + CHAINED_ADD_MARK_RE, + CHAINED_ADD_NODE_RE, + IDENTIFIER_RE, + INPUT_RULE_RE, + KEYMAP_OBJECT_DECL_RE, + MARK_SPEC_RE, + MD_PLUGIN_RE, + NODE_SPEC_RE, + STATE_TEXT_RE, + STATE_WRITE_RE, + STATIC_COMPUTED_ASSIGNMENT_RE, + STATIC_MEMBER_ASSIGNMENT_RE, +} from './patterns.mjs'; + +/** + * Resets a global regular expression before scanning. + */ +export function resetPattern(pattern) { + pattern.lastIndex = 0; + return pattern; +} + +/** + * Reads the first defined capture group from a match. + */ +function firstCapture(match) { + return match[3] || match[1] || match[2]; +} + +/** + * Extracts names from one or more regular expressions. + */ +function extractCapturedNames(content, patterns) { + const names = []; + + for (const pattern of patterns) { + let match; + const re = resetPattern(pattern); + while ((match = re.exec(content))) { + names.push(firstCapture(match)); + } + } + + return names; +} + +/** + * Extracts ProseMirror node registrations from builder calls. + */ +export function extractAddNode(content) { + return extractCapturedNames(content, [BUILDER_ADD_NODE_RE, CHAINED_ADD_NODE_RE]); +} + +/** + * Extracts ProseMirror mark registrations from builder calls. + */ +export function extractAddMark(content) { + return extractCapturedNames(content, [BUILDER_ADD_MARK_RE, CHAINED_ADD_MARK_RE]); +} + +/** + * Extracts node names from node spec registrations. + */ +export function extractNodeSpecs(content) { + return extractCapturedNames(content, [NODE_SPEC_RE]); +} + +/** + * Extracts mark names from mark spec registrations. + */ +export function extractMarkSpecs(content) { + return extractCapturedNames(content, [MARK_SPEC_RE]); +} + +/** + * Extracts editor action identifiers from builder calls. + */ +export function extractActions(content) { + return extractCapturedNames(content, [BUILDER_ADD_ACTION_RE, CHAINED_ADD_ACTION_RE]); +} + +/** + * Extracts ProseMirror plugin factory names. + */ +export function extractPlugins(content) { + const plugins = []; + let match; + const re = resetPattern(ADD_PLUGIN_RE); + while ((match = re.exec(content))) { + plugins.push(match[1]); + } + return plugins; +} + +/** + * Finds the next non-whitespace character index. + */ +export function skipWhitespace(content, index) { + let cursor = index; + while (cursor < content.length && /\s/.test(content[cursor])) { + cursor++; + } + return cursor; +} + +/** + * Reads a balanced bracketed block while ignoring comments and strings. + */ +export function readBalanced(content, startIndex, openChar, closeChar) { + let depth = 0; + let quote = null; + let inBlockComment = false; + let inLineComment = false; + + for (let index = startIndex; index < content.length; index++) { + const char = content[index]; + const next = content[index + 1]; + + if (inLineComment) { + if (char === '\n') inLineComment = false; + continue; + } + + if (inBlockComment) { + if (char === '*' && next === '/') { + inBlockComment = false; + index++; + } + continue; + } + + if (quote) { + if (char === '\\') { + index++; + continue; + } + if (char === quote) quote = null; + continue; + } + + if (char === '/' && next === '/') { + inLineComment = true; + index++; + continue; + } + + if (char === '/' && next === '*') { + inBlockComment = true; + index++; + continue; + } + + if (char === '"' || char === "'" || char === '`') { + quote = char; + continue; + } + + if (char === openChar) { + depth++; + continue; + } + + if (char === closeChar) { + depth--; + if (depth === 0) { + return {body: content.slice(startIndex + 1, index), endIndex: index}; + } + } + } + + return null; +} + +/** + * Reads an expression until a top-level stop character. + */ +export function readExpression(content, startIndex, stopChars) { + let parenDepth = 0; + let braceDepth = 0; + let bracketDepth = 0; + let quote = null; + let inBlockComment = false; + let inLineComment = false; + + for (let index = startIndex; index < content.length; index++) { + const char = content[index]; + const next = content[index + 1]; + + if (inLineComment) { + if (char === '\n') inLineComment = false; + continue; + } + + if (inBlockComment) { + if (char === '*' && next === '/') { + inBlockComment = false; + index++; + } + continue; + } + + if (quote) { + if (char === '\\') { + index++; + continue; + } + if (char === quote) quote = null; + continue; + } + + if (char === '/' && next === '/') { + inLineComment = true; + index++; + continue; + } + + if (char === '/' && next === '*') { + inBlockComment = true; + index++; + continue; + } + + if (char === '"' || char === "'" || char === '`') { + quote = char; + continue; + } + + if (char === '(') parenDepth++; + else if (char === ')') parenDepth--; + else if (char === '{') braceDepth++; + else if (char === '}') braceDepth--; + else if (char === '[') bracketDepth++; + else if (char === ']') bracketDepth--; + + if ( + parenDepth === 0 && + braceDepth === 0 && + bracketDepth === 0 && + stopChars.includes(char) + ) { + return {body: content.slice(startIndex, index).trim(), endIndex: index}; + } + } + + return {body: content.slice(startIndex).trim(), endIndex: content.length}; +} + +/** + * Splits content by top-level separators. + */ +export function splitTopLevelBy(content, separators, {trackAngles = false} = {}) { + const parts = []; + let segmentStart = 0; + let parenDepth = 0; + let braceDepth = 0; + let bracketDepth = 0; + let angleDepth = 0; + let quote = null; + let inBlockComment = false; + let inLineComment = false; + + for (let index = 0; index < content.length; index++) { + const char = content[index]; + const next = content[index + 1]; + + if (inLineComment) { + if (char === '\n') inLineComment = false; + continue; + } + + if (inBlockComment) { + if (char === '*' && next === '/') { + inBlockComment = false; + index++; + } + continue; + } + + if (quote) { + if (char === '\\') { + index++; + continue; + } + if (char === quote) quote = null; + continue; + } + + if (char === '/' && next === '/') { + inLineComment = true; + index++; + continue; + } + + if (char === '/' && next === '*') { + inBlockComment = true; + index++; + continue; + } + + if (char === '"' || char === "'" || char === '`') { + quote = char; + continue; + } + + if (char === '(') parenDepth++; + else if (char === ')') parenDepth--; + else if (char === '{') braceDepth++; + else if (char === '}') braceDepth--; + else if (char === '[') bracketDepth++; + else if (char === ']') bracketDepth--; + else if (trackAngles && char === '<') angleDepth++; + else if (trackAngles && char === '>' && content[index - 1] !== '=' && angleDepth > 0) { + angleDepth--; + } + + if ( + separators.includes(char) && + parenDepth === 0 && + braceDepth === 0 && + bracketDepth === 0 && + angleDepth === 0 + ) { + parts.push(content.slice(segmentStart, index).trim()); + segmentStart = index + 1; + } + } + + const tail = content.slice(segmentStart).trim(); + if (tail) parts.push(tail); + + return parts; +} + +/** + * Splits content by top-level commas. + */ +export function splitTopLevel(content) { + return splitTopLevelBy(content, [',']); +} + +/** + * Finds a top-level arrow token. + */ +function findArrowIndex(content, startIndex) { + let parenDepth = 0; + let braceDepth = 0; + let bracketDepth = 0; + let quote = null; + let inBlockComment = false; + let inLineComment = false; + + for (let index = startIndex; index < content.length; index++) { + const char = content[index]; + const next = content[index + 1]; + + if (inLineComment) { + if (char === '\n') inLineComment = false; + continue; + } + + if (inBlockComment) { + if (char === '*' && next === '/') { + inBlockComment = false; + index++; + } + continue; + } + + if (quote) { + if (char === '\\') { + index++; + continue; + } + if (char === quote) quote = null; + continue; + } + + if (char === '/' && next === '/') { + inLineComment = true; + index++; + continue; + } + + if (char === '/' && next === '*') { + inBlockComment = true; + index++; + continue; + } + + if (char === '"' || char === "'" || char === '`') { + quote = char; + continue; + } + + if (char === '(') parenDepth++; + else if (char === ')') parenDepth--; + else if (char === '{') braceDepth++; + else if (char === '}') braceDepth--; + else if (char === '[') bracketDepth++; + else if (char === ']') bracketDepth--; + + if ( + parenDepth === 0 && + braceDepth === 0 && + bracketDepth === 0 && + char === '=' && + next === '>' + ) { + return index; + } + } + + return -1; +} + +/** + * Extracts static keys from an object literal. + */ +function extractObjectLiteralKeys(content, knownObjects = new Map()) { + let objectBody = content.trim(); + if (objectBody.startsWith('(') && objectBody.endsWith(')')) { + objectBody = objectBody.slice(1, -1).trim(); + } + if (objectBody.startsWith('{') && objectBody.endsWith('}')) { + objectBody = objectBody.slice(1, -1); + } + + const keys = []; + for (const segment of splitTopLevel(objectBody)) { + if (!segment) continue; + + if (segment.startsWith('...')) { + const spreadName = segment.slice(3).trim(); + if (knownObjects.has(spreadName)) { + keys.push(...knownObjects.get(spreadName)); + } + continue; + } + + const colonIndex = segment.indexOf(':'); + if (colonIndex === -1) continue; + + const rawKey = segment.slice(0, colonIndex).trim(); + if (!rawKey || rawKey.startsWith('[')) continue; + + if ( + (rawKey.startsWith('"') && rawKey.endsWith('"')) || + (rawKey.startsWith("'") && rawKey.endsWith("'")) + ) { + keys.push(rawKey.slice(1, -1)); + continue; + } + + if (IDENTIFIER_RE.test(rawKey)) { + keys.push(rawKey); + } + } + + return keys; +} + +/** + * Extracts named keymap objects declared inside a callback body. + */ +function extractKnownKeymapObjects(blockBody) { + const knownObjects = new Map(); + const declarationRe = resetPattern(KEYMAP_OBJECT_DECL_RE); + let match; + + while ((match = declarationRe.exec(blockBody))) { + const objectStart = blockBody.indexOf('{', match.index); + const objectLiteral = readBalanced(blockBody, objectStart, '{', '}'); + if (!objectLiteral) continue; + + knownObjects.set( + match[1], + extractObjectLiteralKeys(`{${objectLiteral.body}}`, knownObjects), + ); + declarationRe.lastIndex = objectLiteral.endIndex + 1; + } + + const staticComputedAssignmentRe = resetPattern(STATIC_COMPUTED_ASSIGNMENT_RE); + while ((match = staticComputedAssignmentRe.exec(blockBody))) { + const keys = knownObjects.get(match[1]) || []; + keys.push(match[2] || match[3]); + knownObjects.set(match[1], keys); + } + + const staticMemberAssignmentRe = resetPattern(STATIC_MEMBER_ASSIGNMENT_RE); + while ((match = staticMemberAssignmentRe.exec(blockBody))) { + const keys = knownObjects.get(match[1]) || []; + keys.push(match[2]); + knownObjects.set(match[1], keys); + } + + return new Map([...knownObjects.entries()].map(([name, keys]) => [name, [...new Set(keys)]])); +} + +/** + * Extracts static keys from a returned keymap expression. + */ +function extractReturnedKeys(blockBody) { + const knownObjects = extractKnownKeymapObjects(blockBody); + let parenDepth = 0; + let braceDepth = 0; + let bracketDepth = 0; + let quote = null; + let inBlockComment = false; + let inLineComment = false; + + for (let index = 0; index < blockBody.length; index++) { + const char = blockBody[index]; + const next = blockBody[index + 1]; + + if (inLineComment) { + if (char === '\n') inLineComment = false; + continue; + } + + if (inBlockComment) { + if (char === '*' && next === '/') { + inBlockComment = false; + index++; + } + continue; + } + + if (quote) { + if (char === '\\') { + index++; + continue; + } + if (char === quote) quote = null; + continue; + } + + if (char === '/' && next === '/') { + inLineComment = true; + index++; + continue; + } + + if (char === '/' && next === '*') { + inBlockComment = true; + index++; + continue; + } + + if (char === '"' || char === "'" || char === '`') { + quote = char; + continue; + } + + if (char === '(') parenDepth++; + else if (char === ')') parenDepth--; + else if (char === '{') braceDepth++; + else if (char === '}') braceDepth--; + else if (char === '[') bracketDepth++; + else if (char === ']') bracketDepth--; + + if (parenDepth !== 0 || braceDepth !== 0 || bracketDepth !== 0) continue; + if (!blockBody.startsWith('return', index)) continue; + + const before = blockBody[index - 1]; + const after = blockBody[index + 6]; + if ((before && /\w/.test(before)) || (after && /\w/.test(after))) continue; + + const cursor = skipWhitespace(blockBody, index + 6); + const startChar = blockBody[cursor]; + + if (startChar === '{') { + const objectLiteral = readBalanced(blockBody, cursor, '{', '}'); + return objectLiteral + ? extractObjectLiteralKeys(`{${objectLiteral.body}}`, knownObjects) + : []; + } + + if (startChar === '(' && blockBody[skipWhitespace(blockBody, cursor + 1)] === '{') { + const expression = readBalanced(blockBody, cursor, '(', ')'); + return expression ? extractObjectLiteralKeys(`(${expression.body})`, knownObjects) : []; + } + + const expression = readExpression(blockBody, cursor, [';']); + return knownObjects.get(expression.body.trim()) || []; + } + + return []; +} + +/** + * Extracts callback bodies passed to addKeymap. + */ +function extractKeymapCallbackBodies(content) { + const bodies = []; + let index = 0; + + while ((index = content.indexOf('.addKeymap(', index)) !== -1) { + const arrowIndex = findArrowIndex(content, index + '.addKeymap('.length); + if (arrowIndex === -1) { + index += '.addKeymap('.length; + continue; + } + + const bodyStart = skipWhitespace(content, arrowIndex + 2); + const startChar = content[bodyStart]; + + if (startChar === '{') { + const blockBody = readBalanced(content, bodyStart, '{', '}'); + if (blockBody) { + bodies.push({type: 'block', body: blockBody.body}); + index = blockBody.endIndex + 1; + continue; + } + } + + if (startChar === '(' && content[skipWhitespace(content, bodyStart + 1)] === '{') { + const expressionBody = readBalanced(content, bodyStart, '(', ')'); + if (expressionBody) { + bodies.push({type: 'expression', body: `(${expressionBody.body})`}); + index = expressionBody.endIndex + 1; + continue; + } + } + + const expressionBody = readExpression(content, bodyStart, [',', ')']); + bodies.push({type: 'expression', body: expressionBody.body}); + index = expressionBody.endIndex + 1; + } + + return bodies; +} + +/** + * Extracts static key bindings from addKeymap callbacks. + */ +export function extractKeymaps(content) { + const keymaps = []; + + for (const callbackBody of extractKeymapCallbackBodies(content)) { + if (callbackBody.type === 'block') { + keymaps.push(...extractReturnedKeys(callbackBody.body)); + continue; + } + + if (callbackBody.body.startsWith('(') || callbackBody.body.startsWith('{')) { + keymaps.push(...extractObjectLiteralKeys(callbackBody.body)); + } + } + + return [...new Set(keymaps)]; +} + +/** + * Extracts input-rule syntax patterns. + */ +export function extractInputRules(content) { + const rules = []; + const re = resetPattern(INPUT_RULE_RE); + let match; + + while ((match = re.exec(content))) { + if (match[1]) { + rules.push(`/${match[1]}/`); + } else if (match[2] && match[3]) { + rules.push(`${match[2]}...${match[3]}`); + } + } + + return rules; +} + +/** + * Extracts markdown-it plugin registrations. + */ +export function extractMdPlugins(content) { + const plugins = []; + const re = resetPattern(MD_PLUGIN_RE); + let match; + while ((match = re.exec(content))) { + plugins.push(match[1]); + } + return plugins; +} + +/** + * Extracts serializer output snippets. + */ +export function extractSerializerSyntax(content) { + const snippets = []; + const writeRe = resetPattern(STATE_WRITE_RE); + let match; + while ((match = writeRe.exec(content))) { + if (match[1].trim()) snippets.push(match[1]); + } + + const textRe = resetPattern(STATE_TEXT_RE); + while ((match = textRe.exec(content))) { + if (match[1].trim()) snippets.push(match[1]); + } + + return snippets; +} diff --git a/infra/docs-gen/src/extractor/scan.mjs b/infra/docs-gen/src/extractor/scan.mjs new file mode 100644 index 000000000..fff8e408f --- /dev/null +++ b/infra/docs-gen/src/extractor/scan.mjs @@ -0,0 +1,197 @@ +import {basename, relative} from 'node:path'; + +import {readAllTsFiles} from '../utils.mjs'; + +import {extractConstants, resolveAllConstants} from './constants.mjs'; +import {extractTestExamples} from './examples.mjs'; +import {extractOptionsType} from './options.mjs'; +import { + extractActions, + extractAddMark, + extractAddNode, + extractInputRules, + extractKeymaps, + extractMarkSpecs, + extractMdPlugins, + extractNodeSpecs, + extractPlugins, + extractSerializerSyntax, +} from './regex.mjs'; +import { + findRootIndexFile, + findSpecsIndexFile, + isTestFile, + joinContents, + selectSerializerFiles, + selectSourceFiles, + selectSpecFiles, +} from './source-files.mjs'; + +/** + * Deduplicates extracted values while keeping the source order. + */ +function unique(values) { + return [...new Set(values)]; +} + +/** + * Reads extension files and separates production sources from tests. + */ +function readExtensionSources(extDir) { + const allFiles = readAllTsFiles(extDir); + const sourceFiles = selectSourceFiles(allFiles); + + return { + allFiles, + sourceFiles, + allContent: joinContents(sourceFiles), + }; +} + +/** + * Builds the source text used for schema extraction. + */ +function buildSchemaContent(sourceFiles, extDir) { + return joinContents(selectSpecFiles(sourceFiles, extDir)); +} + +/** + * Extracts schema nodes and marks. + */ +export function extractSchema(specContent, constants) { + return { + nodes: resolveAllConstants( + [...extractAddNode(specContent), ...extractNodeSpecs(specContent)], + constants, + ), + marks: resolveAllConstants( + [...extractAddMark(specContent), ...extractMarkSpecs(specContent)], + constants, + ), + }; +} + +/** + * Builds option declaration names preferred for an extension. + */ +function buildPreferredOptionNames(extensionName) { + return [`${extensionName}Options`, `${extensionName}SpecsOptions`]; +} + +/** + * Extracts extension options from local source declarations. + */ +export function extractOptions(sourceFiles, extDir, extensionName) { + const rootIndexFile = findRootIndexFile(sourceFiles, extDir); + const specsIndexFile = findSpecsIndexFile(sourceFiles); + const preferredNames = buildPreferredOptionNames(extensionName); + const allOptions = extractOptionsType(joinContents(sourceFiles), preferredNames); + + if (allOptions.length > 0) return allOptions; + if (rootIndexFile) return extractOptionsType(rootIndexFile.content, preferredNames); + if (specsIndexFile) return extractOptionsType(specsIndexFile.content, preferredNames); + + return []; +} + +/** + * Extracts unique markup examples from test files. + */ +export function extractMarkupExamples(files) { + return [ + ...new Set( + files + .filter((file) => isTestFile(file.path)) + .flatMap((file) => extractTestExamples(file.content)), + ), + ]; +} + +/** + * Extracts raw action identifiers. + */ +function extractActionNames(content, constants) { + return resolveAllConstants(extractActions(content), constants); +} + +/** + * Extracts unique ProseMirror plugin names. + */ +function extractPluginNames(content) { + return unique(extractPlugins(content)); +} + +/** + * Extracts unique markdown-it plugin names. + */ +function extractMdPluginNames(content) { + return unique(extractMdPlugins(content)); +} + +/** + * Extracts unique serializer output snippets. + */ +function extractSerializerHints(sourceFiles) { + const serializerContent = joinContents(selectSerializerFiles(sourceFiles)); + return unique(extractSerializerSyntax(serializerContent)); +} + +/** + * Builds the final extension IR record. + */ +function createExtensionRecord({ + name, + sourcePath, + category, + schema, + actions, + keymaps, + inputRules, + plugins, + mdPlugins, + serializerHints, + options, + markupExamples, +}) { + return { + name, + sourcePath, + category, + nodes: schema.nodes, + marks: schema.marks, + actions, + keymaps, + inputRules, + plugins, + mdPlugins, + serializerHints, + options, + markupExamples, + presets: [], + }; +} + +/** + * Scans one extension directory into raw metadata. + */ +export function scanExtension({extDir, category, repoRoot}) { + const name = basename(extDir); + const {allFiles, sourceFiles, allContent} = readExtensionSources(extDir); + const constants = extractConstants(allContent); + const schema = extractSchema(buildSchemaContent(sourceFiles, extDir), constants); + + return createExtensionRecord({ + name, + sourcePath: relative(repoRoot, extDir), + category, + schema, + actions: extractActionNames(allContent, constants), + keymaps: extractKeymaps(allContent), + inputRules: extractInputRules(allContent), + plugins: extractPluginNames(allContent), + mdPlugins: extractMdPluginNames(allContent), + serializerHints: extractSerializerHints(sourceFiles), + options: extractOptions(sourceFiles, extDir, name), + markupExamples: extractMarkupExamples(allFiles), + }); +} diff --git a/infra/docs-gen/src/extractor/source-files.mjs b/infra/docs-gen/src/extractor/source-files.mjs new file mode 100644 index 000000000..06f3afe82 --- /dev/null +++ b/infra/docs-gen/src/extractor/source-files.mjs @@ -0,0 +1,63 @@ +import {dirname} from 'node:path'; + +/** + * Checks whether a file path points to a TypeScript test file. + */ +export function isTestFile(path) { + return /\.test\.tsx?$/.test(path); +} + +/** + * Selects production files from all extension files. + */ +export function selectSourceFiles(files) { + return files.filter((file) => !isTestFile(file.path)); +} + +/** + * Joins file contents into one source string. + */ +export function joinContents(files) { + return files.map((file) => file.content).join('\n'); +} + +/** + * Checks whether a source file can contain schema metadata. + */ +export function isSpecSourceFile(file, extDir) { + return ( + file.path.includes('Specs') || + file.path.includes('const') || + file.path.includes('schema') || + file.path.includes('parser') || + (file.path.endsWith('/index.ts') && dirname(file.path) === extDir) + ); +} + +/** + * Selects files that can contain schema registrations. + */ +export function selectSpecFiles(files, extDir) { + return files.filter((file) => isSpecSourceFile(file, extDir)); +} + +/** + * Selects files that can contain serializer hints. + */ +export function selectSerializerFiles(files) { + return files.filter((file) => file.path.includes('serializer') || file.path.includes('Specs')); +} + +/** + * Finds an extension root index file. + */ +export function findRootIndexFile(files, extDir) { + return files.find((file) => file.path.endsWith('/index.ts') && dirname(file.path) === extDir); +} + +/** + * Finds a specs index file. + */ +export function findSpecsIndexFile(files) { + return files.find((file) => file.path.includes('Specs') && file.path.endsWith('/index.ts')); +} diff --git a/infra/docs-gen/src/generate-docs.mjs b/infra/docs-gen/src/generate-docs.mjs index ffb42097a..f3ef8ddc9 100644 --- a/infra/docs-gen/src/generate-docs.mjs +++ b/infra/docs-gen/src/generate-docs.mjs @@ -7,24 +7,17 @@ import { rmSync, writeFileSync, } from 'node:fs'; -import {dirname, join, resolve} from 'node:path'; +import {dirname, join} from 'node:path'; import process from 'node:process'; -import {fileURLToPath} from 'node:url'; -const REPO_ROOT = resolve(dirname(fileURLToPath(import.meta.url)), '../../..'); -const DOCS_DIR = join(REPO_ROOT, 'docs'); -const OUT_DIR = join(REPO_ROOT, 'tmp/docs-src'); -const GITHUB_RAW_RE = - /https:\/\/raw\.githubusercontent\.com\/gravity-ui\/markdown-editor\/(?:refs\/heads\/[^/]+|[^/]+)\/docs\//g; +import {DOCS_DIR, DOCS_SRC_DIR, GITHUB_RAW_RE, HEADER_RE} from './config.mjs'; // Source docs use ##### as a metadata header (not rendered). // Format: "##### Category / Title" or "##### Title" (no category). // This line is stripped from the output; the rest becomes the page content. -const HEADER_RE = /^#{5}\s+(.+)$/; /** - * Converts a string to a URL-friendly slug (lowercase, alphanumeric, hyphens). - * @param str + * Converts a string to a URL-friendly slug. */ function slugify(str) { return str @@ -34,8 +27,7 @@ function slugify(str) { } /** - * Extracts category and title from a `##### Category / Title` header line. - * @param firstLine + * Extracts category and title from a metadata header. */ function parseHeader(firstLine) { const match = firstLine.match(HEADER_RE); @@ -52,10 +44,10 @@ function parseHeader(firstLine) { /** Removes all generated content from the output directory. */ function cleanOutDir() { - if (existsSync(OUT_DIR)) { - rmSync(OUT_DIR, {recursive: true, force: true}); + if (existsSync(DOCS_SRC_DIR)) { + rmSync(DOCS_SRC_DIR, {recursive: true, force: true}); } - mkdirSync(OUT_DIR, {recursive: true}); + mkdirSync(DOCS_SRC_DIR, {recursive: true}); } /** Reads all markdown files from the source directory and parses their headers. */ @@ -94,8 +86,7 @@ function collectDocs() { } /** - * Splits docs into a category map and a top-level (uncategorized) list. - * @param docs + * Splits docs into categorized and top-level groups. */ function groupByCategory(docs) { const categories = new Map(); @@ -116,8 +107,7 @@ function groupByCategory(docs) { } /** - * Builds a relative output file path from the doc's category and title slugs. - * @param doc + * Builds a relative output file path. */ function computeOutputPath(doc) { if (doc.category) { @@ -127,8 +117,7 @@ function computeOutputPath(doc) { } /** - * Ensures no two docs resolve to the same output path; exits on collision. - * @param docs + * Ensures no two docs resolve to the same output path. */ function checkDuplicatePaths(docs) { const seen = new Map(); @@ -145,9 +134,7 @@ function checkDuplicatePaths(docs) { } /** - * Rewrites absolute GitHub raw URLs to relative paths based on doc nesting depth. - * @param content - * @param doc + * Rewrites absolute GitHub raw URLs to relative asset paths. */ function rewriteAssetUrls(content, doc) { const prefix = doc.category ? '../' : './'; @@ -155,21 +142,19 @@ function rewriteAssetUrls(content, doc) { } /** - * Writes stripped markdown content to categorized output paths. - * @param docs + * Writes stripped Markdown content to categorized output paths. */ function writeDocFiles(docs) { checkDuplicatePaths(docs); for (const doc of docs) { - const outPath = join(OUT_DIR, computeOutputPath(doc)); + const outPath = join(DOCS_SRC_DIR, computeOutputPath(doc)); mkdirSync(dirname(outPath), {recursive: true}); writeFileSync(outPath, rewriteAssetUrls(doc.content, doc)); } } /** - * Wraps a string in double quotes if it contains YAML special characters. - * @param str + * Wraps YAML values that contain special characters. */ function yamlQuote(str) { if (/[:#"'{}[\],&*?|>!%@`]/.test(str)) { @@ -179,9 +164,7 @@ function yamlQuote(str) { } /** - * Generates the `toc.yaml` table of contents for the YFM documentation site. - * @param categories - * @param topLevel + * Generates the table of contents for the documentation site. */ function generateTocYaml(categories, topLevel) { const lines = [ @@ -206,13 +189,11 @@ function generateTocYaml(categories, topLevel) { lines.push(` href: ${computeOutputPath(doc)}`); } - writeFileSync(join(OUT_DIR, 'toc.yaml'), lines.join('\n') + '\n'); + writeFileSync(join(DOCS_SRC_DIR, 'toc.yaml'), lines.join('\n') + '\n'); } /** - * Generates the `index.md` landing page with links to all doc pages. - * @param categories - * @param topLevel + * Generates the landing page with links to all doc pages. */ function generateIndexMd(categories, topLevel) { const lines = [ @@ -237,20 +218,20 @@ function generateIndexMd(categories, topLevel) { lines.push(''); } - writeFileSync(join(OUT_DIR, 'index.md'), lines.join('\n')); + writeFileSync(join(DOCS_SRC_DIR, 'index.md'), lines.join('\n')); } /** Copies the `assets/` directory from source docs to the output directory. */ function copyAssets() { const assetsDir = join(DOCS_DIR, 'assets'); if (existsSync(assetsDir)) { - cpSync(assetsDir, join(OUT_DIR, 'assets'), {recursive: true}); + cpSync(assetsDir, join(DOCS_SRC_DIR, 'assets'), {recursive: true}); } } /** Writes the `.yfm` Diplodoc config into the output directory. */ function writeYfmConfig() { - writeFileSync(join(OUT_DIR, '.yfm'), 'allowHTML: true\n'); + writeFileSync(join(DOCS_SRC_DIR, '.yfm'), 'allowHTML: true\n'); } /** Entry point: cleans output, collects docs, and generates the documentation site. */ diff --git a/infra/docs-gen/src/logger.mjs b/infra/docs-gen/src/logger.mjs new file mode 100644 index 000000000..cdb538975 --- /dev/null +++ b/infra/docs-gen/src/logger.mjs @@ -0,0 +1,33 @@ +/* eslint-disable no-console */ + +export class Logger { + /** + * Writes an informational message. + */ + info(...args) { + console.log(...args); + } + + /** + * Writes a warning message. + */ + warn(...args) { + console.warn('Warning:', ...args); + } + + /** + * Writes an error message. + */ + error(...args) { + console.error('Error:', ...args); + } + + /** + * Writes a successful completion message. + */ + success(...args) { + console.log('Done:', ...args); + } +} + +export const logger = new Logger(); diff --git a/infra/docs-gen/src/utils.mjs b/infra/docs-gen/src/utils.mjs new file mode 100644 index 000000000..2adeeded9 --- /dev/null +++ b/infra/docs-gen/src/utils.mjs @@ -0,0 +1,42 @@ +import {existsSync, readFileSync, readdirSync, statSync} from 'node:fs'; +import {join} from 'node:path'; + +/** + * Reads a UTF-8 text file. + */ +export function readText(filePath) { + return readFileSync(filePath, 'utf-8'); +} + +/** + * Lists uppercase-named child directories. + */ +export function listDirs(dir) { + if (!existsSync(dir)) return []; + + return readdirSync(dir) + .filter((name) => { + const full = join(dir, name); + return statSync(full).isDirectory() && /^[A-Z]/.test(name); + }) + .sort(); +} + +/** + * Finds recursive directory entries matching a pattern. + */ +export function findFiles(dir, pattern) { + if (!existsSync(dir)) return []; + + return readdirSync(dir, {recursive: true}) + .filter((entry) => pattern.test(entry)) + .map((entry) => join(dir, entry)) + .sort(); +} + +/** + * Reads recursive TypeScript source files. + */ +export function readAllTsFiles(dir) { + return findFiles(dir, /\.tsx?$/).map((path) => ({path, content: readText(path)})); +}