Skip to content

perf: more tokenizer inlining (5% faster tokenizing) #309

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
bartveneman merged 1 commit into from
Oct 15, 2026
Merged

perf: more tokenizer inlining (5% faster tokenizing) #309

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
22 changes: 14 additions & 8 deletions src/char-types.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,16 +21,22 @@
// - Cache-friendly (128 bytes fits in L1 cache)
// - Supports overlapping categories (e.g., '5' is both digit and hex)

let CHAR_ALPHA = 1 << 0 // 1
export let CHAR_DIGIT = 1 << 1 // 2
let CHAR_HEX = 1 << 2 // 4
export let CHAR_WHITESPACE = 1 << 3 // 8
export let CHAR_NEWLINE = 1 << 4 // 16
export let CHAR_IDENT = 1 << 5 // 32
export let CHAR_IDENT_START = 1 << 6 // 64: ASCII letters and underscore (non-ASCII handled by callers via ch >= 0x80)
// `const` (not `let`): these never change after init, so bundlers/V8 can treat
// them as true constants and fold combined masks like CHAR_WHITESPACE | CHAR_NEWLINE
// into a literal instead of re-reading two module bindings on every hot-loop char.
const CHAR_ALPHA = 1 << 0 // 1
export const CHAR_DIGIT = 1 << 1 // 2
const CHAR_HEX = 1 << 2 // 4
const CHAR_WHITESPACE = 1 << 3 // 8
export const CHAR_NEWLINE = 1 << 4 // 16
export const CHAR_IDENT = 1 << 5 // 32
export const CHAR_IDENT_START = 1 << 6 // 64: ASCII letters and underscore (non-ASCII handled by callers via ch >= 0x80)
// Precomputed so the whitespace-skip hot loops do a single mask compare
// instead of OR-ing two constants on every character.
export const CHAR_WHITESPACE_OR_NEWLINE = CHAR_WHITESPACE | CHAR_NEWLINE

// Lookup table for ASCII characters (0-127)
export let char_types = new Uint8Array(128)
export const char_types = new Uint8Array(128)

// Initialize digit characters (0-9)
for (let i = 0x30; i <= 0x39; i++) {
Expand Down
21 changes: 12 additions & 9 deletions src/tokenize.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,8 +3,8 @@ import {
is_ident_start,
char_types,
CHAR_DIGIT,
CHAR_WHITESPACE,
CHAR_NEWLINE,
CHAR_WHITESPACE_OR_NEWLINE,
CHAR_IDENT,
CHAR_IDENT_START,
} from './char-types'
Expand Down Expand Up @@ -174,7 +174,7 @@ export class Lexer {
let line_offset = this._line_offset
while (pos < source_length) {
let ch = source.charCodeAt(pos)
if (ch >= 128 || (char_types[ch] & (CHAR_WHITESPACE | CHAR_NEWLINE)) === 0) break
if (ch >= 128 || (char_types[ch] & CHAR_WHITESPACE_OR_NEWLINE) === 0) break
pos++
if ((char_types[ch] & CHAR_NEWLINE) !== 0) {
if (
Expand All @@ -198,8 +198,6 @@ export class Lexer {
}

let ch = source.charCodeAt(this.pos)
// One table read classifies ch for the dispatch below (non-ASCII → 0, handled via ch >= 128)
let ch_type = ch < 128 ? char_types[ch] : 0
let start = this.pos
let start_line = this._line
let start_column = this.pos - this._line_offset + 1
Expand Down Expand Up @@ -235,8 +233,13 @@ export class Lexer {
return this.make_token(TOKEN_RIGHT_PAREN, start, this.pos, start_line, start_column)
}

// One table read classifies ch for the dispatch below (non-ASCII → 0, handled via
// ch >= 128). Deferred until after the switch so the 9 single-char-token cases
// above — among the most common bytes in CSS ({ } : ; , [ ] ( )) — skip this lookup.
let ch_type = ch < 128 ? char_types[ch] : 0

// Whitespace
if ((ch_type & (CHAR_WHITESPACE | CHAR_NEWLINE)) !== 0) {
if ((ch_type & CHAR_WHITESPACE_OR_NEWLINE) !== 0) {
return this.consume_whitespace(start_line, start_column)
}

Expand Down Expand Up @@ -547,7 +550,7 @@ export class Lexer {
// Inline advance: read ch once, pos++, then track newlines with already-read value
while (pos < source_length) {
let ch = source.charCodeAt(pos)
if (ch >= 128 || (char_types[ch] & (CHAR_WHITESPACE | CHAR_NEWLINE)) === 0) break
if (ch >= 128 || (char_types[ch] & CHAR_WHITESPACE_OR_NEWLINE) === 0) break
pos++
if ((char_types[ch] & CHAR_NEWLINE) !== 0) {
if (
Expand Down Expand Up @@ -636,7 +639,7 @@ export class Lexer {
// Optional whitespace after hex escape (may be a newline — inline advance)
if (this.pos < source_length) {
let ch = source.charCodeAt(this.pos)
if (ch < 128 && (char_types[ch] & (CHAR_WHITESPACE | CHAR_NEWLINE)) !== 0) {
if (ch < 128 && (char_types[ch] & CHAR_WHITESPACE_OR_NEWLINE) !== 0) {
this.pos++
if ((char_types[ch] & CHAR_NEWLINE) !== 0) {
if (
Expand Down Expand Up @@ -775,7 +778,7 @@ export class Lexer {
// Consume optional whitespace after hex escape (may be newline)
if (this.pos < source_length) {
let ws = source.charCodeAt(this.pos)
if (ws < 128 && (char_types[ws] & (CHAR_WHITESPACE | CHAR_NEWLINE)) !== 0) {
if (ws < 128 && (char_types[ws] & CHAR_WHITESPACE_OR_NEWLINE) !== 0) {
this.advance()
}
}
Expand Down Expand Up @@ -1039,7 +1042,7 @@ export class Lexer {

// Skip whitespace (space, tab, and newlines — is_whitespace() from string-utils
// covers newlines too, but here we go straight to char_types to avoid the import)
if (ch < 128 && (char_types[ch] & (CHAR_WHITESPACE | CHAR_NEWLINE)) !== 0) {
if (ch < 128 && (char_types[ch] & CHAR_WHITESPACE_OR_NEWLINE) !== 0) {
this.advance()
continue
}
Expand Down
Loading