From 80197b0287b8cebd0a24b20558ad54b0b8f33dce Mon Sep 17 00:00:00 2001 From: Bart Veneman Date: Thu, 15 Oct 2026 22:53:56 +0200 Subject: [PATCH] perf: more tokenizer inlining --- src/char-types.ts | 22 ++++++++++++++-------- src/tokenize.ts | 21 ++++++++++++--------- 2 files changed, 26 insertions(+), 17 deletions(-) diff --git a/src/char-types.ts b/src/char-types.ts index 0c62d6b..065fd87 100644 --- a/src/char-types.ts +++ b/src/char-types.ts @@ -21,16 +21,22 @@ // - Cache-friendly (128 bytes fits in L1 cache) // - Supports overlapping categories (e.g., '5' is both digit and hex) -let CHAR_ALPHA = 1 << 0 // 1 -export let CHAR_DIGIT = 1 << 1 // 2 -let CHAR_HEX = 1 << 2 // 4 -export let CHAR_WHITESPACE = 1 << 3 // 8 -export let CHAR_NEWLINE = 1 << 4 // 16 -export let CHAR_IDENT = 1 << 5 // 32 -export let CHAR_IDENT_START = 1 << 6 // 64: ASCII letters and underscore (non-ASCII handled by callers via ch >= 0x80) +// `const` (not `let`): these never change after init, so bundlers/V8 can treat +// them as true constants and fold combined masks like CHAR_WHITESPACE | CHAR_NEWLINE +// into a literal instead of re-reading two module bindings on every hot-loop char. +const CHAR_ALPHA = 1 << 0 // 1 +export const CHAR_DIGIT = 1 << 1 // 2 +const CHAR_HEX = 1 << 2 // 4 +const CHAR_WHITESPACE = 1 << 3 // 8 +export const CHAR_NEWLINE = 1 << 4 // 16 +export const CHAR_IDENT = 1 << 5 // 32 +export const CHAR_IDENT_START = 1 << 6 // 64: ASCII letters and underscore (non-ASCII handled by callers via ch >= 0x80) +// Precomputed so the whitespace-skip hot loops do a single mask compare +// instead of OR-ing two constants on every character. +export const CHAR_WHITESPACE_OR_NEWLINE = CHAR_WHITESPACE | CHAR_NEWLINE // Lookup table for ASCII characters (0-127) -export let char_types = new Uint8Array(128) +export const char_types = new Uint8Array(128) // Initialize digit characters (0-9) for (let i = 0x30; i <= 0x39; i++) { diff --git a/src/tokenize.ts b/src/tokenize.ts index 5a58c63..030d32a 100644 --- a/src/tokenize.ts +++ b/src/tokenize.ts @@ -3,8 +3,8 @@ import { is_ident_start, char_types, CHAR_DIGIT, - CHAR_WHITESPACE, CHAR_NEWLINE, + CHAR_WHITESPACE_OR_NEWLINE, CHAR_IDENT, CHAR_IDENT_START, } from './char-types' @@ -174,7 +174,7 @@ export class Lexer { let line_offset = this._line_offset while (pos < source_length) { let ch = source.charCodeAt(pos) - if (ch >= 128 || (char_types[ch] & (CHAR_WHITESPACE | CHAR_NEWLINE)) === 0) break + if (ch >= 128 || (char_types[ch] & CHAR_WHITESPACE_OR_NEWLINE) === 0) break pos++ if ((char_types[ch] & CHAR_NEWLINE) !== 0) { if ( @@ -198,8 +198,6 @@ export class Lexer { } let ch = source.charCodeAt(this.pos) - // One table read classifies ch for the dispatch below (non-ASCII → 0, handled via ch >= 128) - let ch_type = ch < 128 ? char_types[ch] : 0 let start = this.pos let start_line = this._line let start_column = this.pos - this._line_offset + 1 @@ -235,8 +233,13 @@ export class Lexer { return this.make_token(TOKEN_RIGHT_PAREN, start, this.pos, start_line, start_column) } + // One table read classifies ch for the dispatch below (non-ASCII → 0, handled via + // ch >= 128). Deferred until after the switch so the 9 single-char-token cases + // above — among the most common bytes in CSS ({ } : ; , [ ] ( )) — skip this lookup. + let ch_type = ch < 128 ? char_types[ch] : 0 + // Whitespace - if ((ch_type & (CHAR_WHITESPACE | CHAR_NEWLINE)) !== 0) { + if ((ch_type & CHAR_WHITESPACE_OR_NEWLINE) !== 0) { return this.consume_whitespace(start_line, start_column) } @@ -547,7 +550,7 @@ export class Lexer { // Inline advance: read ch once, pos++, then track newlines with already-read value while (pos < source_length) { let ch = source.charCodeAt(pos) - if (ch >= 128 || (char_types[ch] & (CHAR_WHITESPACE | CHAR_NEWLINE)) === 0) break + if (ch >= 128 || (char_types[ch] & CHAR_WHITESPACE_OR_NEWLINE) === 0) break pos++ if ((char_types[ch] & CHAR_NEWLINE) !== 0) { if ( @@ -636,7 +639,7 @@ export class Lexer { // Optional whitespace after hex escape (may be a newline — inline advance) if (this.pos < source_length) { let ch = source.charCodeAt(this.pos) - if (ch < 128 && (char_types[ch] & (CHAR_WHITESPACE | CHAR_NEWLINE)) !== 0) { + if (ch < 128 && (char_types[ch] & CHAR_WHITESPACE_OR_NEWLINE) !== 0) { this.pos++ if ((char_types[ch] & CHAR_NEWLINE) !== 0) { if ( @@ -775,7 +778,7 @@ export class Lexer { // Consume optional whitespace after hex escape (may be newline) if (this.pos < source_length) { let ws = source.charCodeAt(this.pos) - if (ws < 128 && (char_types[ws] & (CHAR_WHITESPACE | CHAR_NEWLINE)) !== 0) { + if (ws < 128 && (char_types[ws] & CHAR_WHITESPACE_OR_NEWLINE) !== 0) { this.advance() } } @@ -1039,7 +1042,7 @@ export class Lexer { // Skip whitespace (space, tab, and newlines — is_whitespace() from string-utils // covers newlines too, but here we go straight to char_types to avoid the import) - if (ch < 128 && (char_types[ch] & (CHAR_WHITESPACE | CHAR_NEWLINE)) !== 0) { + if (ch < 128 && (char_types[ch] & CHAR_WHITESPACE_OR_NEWLINE) !== 0) { this.advance() continue }