dirty range incremental parser instead of full document reparse on every typing event

Author: ryanncode

CHANGELOG.md

@@ -2,10 +2,12 @@
 
 All notable changes to the "pointblank" extension will be documented in this file.
 
-## [0.6.4] - 2025-06-27
+## [0.7.0] - 2025-06-29
 ### Added
 - You can now select a single-line bullet point for copying, making it easier to duplicate or move bullet items.
 - When typing or pasting from the start of a list item line, the bullet point style now matches the next or previous list item line at the same level of indent for consistency.
+### Changed
+- Replaced the document parser with a high-performance "Dirty Range" incremental parser, eliminating typing lag in large documents.
 ### Fixed
 - Pasting a bullet at the beginning of a line no longer erases the line; pasted content is merged or inserted intelligently.
 - Numbered list items will remove their default bullet point.

README.md

@@ -17,7 +17,7 @@ Point Blank's architecture is designed to be modular, performant, and maintainab
 *   `DocumentModel`: Manages the document's state and uses `DocumentParser` to create an immutable `DocumentTree`.
 *   `DecorationManager`: Applies visual decorations to the editor based on the `DocumentTree` and `DecorationCalculator`.
 *   `CommandManager`: Registers and handles all user commands, interacting with the `DocumentModel` and other components.
-*   `DocumentParser`: A stateless component solely responsible for converting text into an immutable `DocumentTree`.
+*   `DocumentParser`: A stateless component responsible for converting text into an immutable `DocumentTree`. It uses a highly efficient "dirty range" incremental parsing strategy to ensure excellent performance even in very large documents.
 
 For a detailed history of changes, see the [CHANGELOG](https://github.com/ryanncode/point-blank/blob/main/CHANGELOG.md).
 

docs/architecture.html

@@ -10,7 +10,7 @@ <h2>Architecture Overview</h2>
     <h3>Data Flow:</h3>
     <ol>
         <li>When a document is opened or modified, VS Code fires an event that is picked up by the <code>DocumentModel</code>.</li>
-        <li>The <code>DocumentModel</code> uses the <code>DocumentParser</code> to create or update the <code>DocumentTree</code>, an immutable, hierarchical representation of the document's text.</li>
+        <li>The <code>DocumentModel</code> uses the <code>DocumentParser</code> to create or update the <code>DocumentTree</code>, an immutable, hierarchical representation of the document's text. The parser uses a highly efficient "dirty range" incremental strategy to avoid performance bottlenecks in large files.</li>
         <li>The <code>DocumentModel</code> notifies the <code>DecorationManager</code> of the change.</li>
         <li>The <code>DecorationManager</code>, using debouncing and viewport-aware logic, retrieves the visible <code>BlockNode</code>s from the <code>DocumentTree</code>.</li>
         <li>It passes these nodes to the <code>DecorationCalculator</code>, which determines the appropriate decorations for each node based on its properties (e.g., <code>bulletType</code>).</li>
@@ -22,6 +22,9 @@ <h2>Key Design Decisions</h2>
     <h3>Immutable Document Model</h3>
     <p>The use of an immutable <code>DocumentTree</code> and <code>BlockNode</code>s is a cornerstone of the architecture. When the document changes, a new tree is created rather than modifying the old one. This provides predictability, simplifies state management, and makes the flow of data easy to trace.</p>
 
+    <h3>Performant Incremental Parsing</h3>
+    <p>To handle large documents without typing lag, Point Blank employs a "Dirty Range" incremental parser. Instead of performing a full re-parse on every keystroke, the parser identifies the minimal block of top-level nodes affected by a change, re-parses only that block, and splices the result back into the document tree. This ensures the UI remains responsive even in documents with thousands of lines.</p>
+
     <h3>Design Philosophy: Insertion vs. Styling</h3>
     <p>A critical distinction in Point Blank's architecture is the separation between one-time character <strong>insertion</strong> and continuous visual <strong>styling</strong>.</p>
     <ul>

package.json

@@ -7,7 +7,7 @@
     "type": "git",
     "url": "https://github.com/ryanncode/point-blank"
   },
-  "version": "0.6.4",
+  "version": "0.7.0",
   "engines": {
     "vscode": "^1.100.0"
   },
@@ -179,7 +179,7 @@
         {
           "command": "pointblank.enterKey",
           "when": "false"
-    }
+        }
       ]
     }
   },
@@ -193,7 +193,8 @@
     "pretest": "pnpm run compile-tests && pnpm run compile && pnpm run lint",
     "lint": "eslint src --ext ts",
     "test": "vscode-test",
-    "test:paste": "jest test/pasteUtils.test.ts"
+    "test:paste": "jest test/pasteUtils.test.ts",
+    "test:parse": "jest test/documentParser.test.ts"
   },
   "devDependencies": {
     "@types/glob": "^8.1.0",
@@ -207,6 +208,7 @@
     "@vscode/test-electron": "^2.5.2",
     "eslint": "^9.25.1",
     "glob": "^8.1.0",
+    "identity-obj-proxy": "^3.0.0",
     "jest": "^30.0.3",
     "ts-jest": "^29.4.0",
     "ts-loader": "^9.5.2",

pnpm-lock.yaml

@@ -28,6 +28,8 @@ export class BlockNode {
     public readonly type?: string;
     public readonly typedNodeRange?: vscode.Range;
     public readonly isCodeBlockDelimiter: boolean;
+    public parent: BlockNode | undefined;
+    public children: BlockNode[];
     public readonly isExcluded: boolean;
     public readonly bulletType: 'star' | 'plus' | 'minus' | 'numbered' | 'blockquote' | 'default' | 'none' | 'atSign';
     public readonly bulletRange?: vscode.Range;
@@ -37,9 +39,9 @@ export class BlockNode {
     public readonly headerLevel?: number;
     public readonly headerText?: string;
 
-    // Hierarchy properties
-    public readonly parent?: BlockNode;
-    public readonly children: readonly BlockNode[];
+    get lineCount(): number {
+        return 1 + this.children.reduce((acc, child) => acc + child.lineCount, 0);
+    }
 
     constructor(
         line: vscode.TextLine,
@@ -54,7 +56,7 @@ export class BlockNode {
         this.indent = line.firstNonWhitespaceCharacterIndex;
         this.isExcluded = isExcluded;
         this.parent = parent;
-        this.children = children;
+        this.children = children || [];
 
         // Perform all parsing upon construction to ensure immutability.
         const { isKeyValue, keyValue, isTypedNode, type, typedNodeRange, isCodeBlockDelimiter, isHeader, headerLevel, headerText } = this.parseLineContent();
@@ -145,48 +147,37 @@ export class BlockNode {
     }
 
     /**
-     * Creates a new `BlockNode` instance with updated children.
-     * This method is essential for maintaining the immutability of the document tree.
-     * @param newChildren The new array of child nodes.
-     * @returns A new `BlockNode` instance.
+     * Creates a new `BlockNode` with the specified parent.
+     * @param parent The new parent for the node.
+     * @returns A new `BlockNode` with the specified parent.
      */
-    public withChildren(newChildren: BlockNode[]): BlockNode {
-        return new BlockNode(this.line, this.lineNumber, this.isExcluded, this.parent, newChildren);
+    public withParent(parent: BlockNode): BlockNode {
+        return new BlockNode(this.line, this.lineNumber, this.isExcluded, parent, this.children);
     }
 
     /**
-     * Creates a new `BlockNode` instance with an updated parent.
-     * @param newParent The new parent node.
-     * @returns A new `BlockNode` instance.
+     * Creates a new `BlockNode` with the specified children.
+     * @param children The new children for the node.
+     * @returns A new `BlockNode` with the specified children.
      */
-    public withParent(newParent?: BlockNode): BlockNode {
-        return new BlockNode(this.line, this.lineNumber, this.isExcluded, newParent, [...this.children]);
+    public withChildren(children: BlockNode[]): BlockNode {
+        return new BlockNode(this.line, this.lineNumber, this.isExcluded, this.parent, children);
     }
 
     /**
-     * Performs a shallow comparison to check if two `BlockNode`s have different content
-     * relevant to decoration. This is used to optimize re-rendering.
-     * @param otherNode The other `BlockNode` to compare against.
-     * @returns `true` if the content is different, `false` otherwise.
+     * Adds a child to this node. This method mutates the node.
+     * @param child The `BlockNode` to add as a child.
      */
-    public isContentDifferent(otherNode: BlockNode): boolean {
-        if (this.text !== otherNode.text ||
-            this.bulletType !== otherNode.bulletType ||
-            this.isKeyValue !== otherNode.isKeyValue ||
-            this.isTypedNode !== otherNode.isTypedNode ||
-            this.isHeader !== otherNode.isHeader ||
-            this.headerLevel !== otherNode.headerLevel ||
-            this.headerText !== otherNode.headerText) {
-            return true;
-        }
+    public addChild(child: BlockNode) {
+        this.children.push(child);
+        child.parent = this;
+    }
 
-        // Deep compare bulletRange only if necessary.
-        const range1 = this.bulletRange;
-        const range2 = otherNode.bulletRange;
-        if ((range1 && !range2) || (!range1 && range2) || (range1 && range2 && !range1.isEqual(range2))) {
-            return true;
+    public getSelfAndDescendants(): BlockNode[] {
+        const result: BlockNode[] = [this];
+        for (const child of this.children) {
+            result.push(...child.getSelfAndDescendants());
         }
-
-        return false;
+        return result;
     }
 }

src/document/blockNode.ts

@@ -80,23 +80,18 @@ export class DocumentModel {
             return;
         }
 
-        // If a bulk update is in progress, defer incremental parsing until the bulk update completes.
         if (this._isBulkUpdating) {
             return;
         }
 
-        // Update the internal document reference to the latest version
         this._document = event.document;
 
-        this._isParsing = true; // Set parsing flag
-        // Create a new, updated tree by applying the changes to the previous tree.
-        const newDocumentTree = this._parser.parse(this._documentTree, event.contentChanges);
+        this._isParsing = true;
+        const newDocumentTree = this._parser.parse(this._documentTree, event.contentChanges, this._document);
         this._documentTree = newDocumentTree;
-        this._isParsing = false; // Clear parsing flag
-        this._onDidParseEventEmitter.fire(); // Fire event
+        this._isParsing = false;
+        this._onDidParseEventEmitter.fire();
 
-        // Notify the DecorationManager with the new tree. The manager will then
-        // handle the debounced update of the actual decorations in the editor.
         this._decorationManager.updateDecorations(this._documentTree);
     }