perf(tokenizer): classify char once per token and use ident-start table bit

src/char-types.ts

@@ -27,6 +27,7 @@ let CHAR_HEX = 1 << 2 // 4
 export let CHAR_WHITESPACE = 1 << 3 // 8
 export let CHAR_NEWLINE = 1 << 4 // 16
 export let CHAR_IDENT = 1 << 5 // 32
+export let CHAR_IDENT_START = 1 << 6 // 64: ASCII letters and underscore (non-ASCII handled by callers via ch >= 0x80)
 
 // Lookup table for ASCII characters (0-127)
 export let char_types = new Uint8Array(128)
@@ -74,17 +75,19 @@ for (let i = 0; i < 128; i++) {
 char_types[0x2d] |= CHAR_IDENT // hyphen -
 char_types[0x5f] |= CHAR_IDENT // underscore _
 
-export function is_hex_digit(ch: number): boolean {
-	return ch < 128 && (char_types[ch] & CHAR_HEX) !== 0
+// Ident-start: letters and underscore. Single-bit test replaces is_alpha() + '_' compare.
+for (let i = 0; i < 128; i++) {
+	if (char_types[i] & CHAR_ALPHA) {
+		char_types[i] |= CHAR_IDENT_START
+	}
 }
+char_types[0x5f] |= CHAR_IDENT_START // underscore _
 
-function is_alpha(ch: number): boolean {
-	return ch < 128 && (char_types[ch] & CHAR_ALPHA) !== 0
+export function is_hex_digit(ch: number): boolean {
+	return ch < 128 && (char_types[ch] & CHAR_HEX) !== 0
 }
 
 // CSS ident start: letter, underscore, or non-ASCII (>= 0x80)
 export function is_ident_start(ch: number): boolean {
-	if (ch >= 0x80) return true // Non-ASCII
-	if (ch === 0x5f) return true // _
-	return is_alpha(ch)
+	return ch >= 0x80 || (char_types[ch] & CHAR_IDENT_START) !== 0
 }

src/tokenize.test.ts

@@ -1,5 +1,6 @@
 import { describe, test, expect } from 'vitest'
 import { Lexer } from './tokenize'
+import { is_ident_start } from './char-types'
 import {
 	TOKEN_IDENT,
 	TOKEN_FUNCTION,
@@ -745,4 +746,76 @@ describe('Lexer', () => {
 			expect(token?.column).toBe(1)
 		})
 	})
+
+	describe('first-character classification', () => {
+		// Independent oracle: what the dispatch in next_token_fast must decide from a lone first character
+		const is_letter = (c: number) => (c >= 0x41 && c <= 0x5a) || (c >= 0x61 && c <= 0x7a)
+		const oracle_ident_start = (c: number) => is_letter(c) || c === 0x5f || c >= 0x80
+		const oracle_whitespace = (c: number) =>
+			c === 0x20 || c === 0x09 || c === 0x0a || c === 0x0d || c === 0x0c
+		const oracle_digit = (c: number) => c >= 0x30 && c <= 0x39
+
+		test('is_ident_start matches letters, underscore and non-ASCII for every code unit', () => {
+			let mismatches: string[] = []
+			for (let c = 0; c <= 0xffff; c++) {
+				if (is_ident_start(c) !== oracle_ident_start(c)) mismatches.push(`0x${c.toString(16)}`)
+			}
+			expect(mismatches).toEqual([])
+		})
+
+		test('a lone character becomes IDENT, WHITESPACE or NUMBER exactly when the oracle says so', () => {
+			// Every ASCII code unit plus a spread of non-ASCII ones (Latin-1, BMP, lone surrogates)
+			const codes = [
+				...Array.from({ length: 0x180 }, (_, i) => i),
+				0x2028,
+				0x3042,
+				0x65e5,
+				0xd83d,
+				0xde00,
+				0xfeff,
+				0xffff,
+			]
+			let mismatches: string[] = []
+			for (let c of codes) {
+				let lexer = new Lexer(String.fromCharCode(c))
+				lexer.next_token_fast(false)
+				let type = lexer.token_type
+				let ok =
+					(type === TOKEN_IDENT) === oracle_ident_start(c) &&
+					(type === TOKEN_WHITESPACE) === oracle_whitespace(c) &&
+					(type === TOKEN_NUMBER) === oracle_digit(c) &&
+					// Whatever it is, it must cover the whole single character
+					lexer.token_start === 0 &&
+					lexer.token_end === 1
+				if (!ok) mismatches.push(`0x${c.toString(16)} -> type ${type}`)
+			}
+			expect(mismatches).toEqual([])
+		})
+
+		test('skip_whitespace=true lands on the same token as an explicit whitespace token', () => {
+			let source = ' \t\n\r\f_x'
+			let lexer = new Lexer(source)
+			lexer.next_token_fast(true)
+			expect(lexer.token_type).toBe(TOKEN_IDENT)
+			expect(source.slice(lexer.token_start, lexer.token_end)).toBe('_x')
+			expect(lexer.token_line).toBe(4) // \n, then \r, then \f each start a line
+		})
+
+		test('identifiers may start with underscore, a letter or a non-ASCII character', () => {
+			for (let source of ['_a', 'a', 'Z', 'é', '日本', '\u{1F600}x']) {
+				let lexer = new Lexer(source)
+				lexer.next_token_fast(false)
+				expect(lexer.token_type).toBe(TOKEN_IDENT)
+				expect(lexer.token_end).toBe(source.length)
+			}
+		})
+
+		test('digits start numbers and are not identifiers', () => {
+			for (let source of ['0', '9', '5px', '.5']) {
+				let lexer = new Lexer(source)
+				lexer.next_token_fast(false)
+				expect(lexer.token_type === TOKEN_IDENT).toBe(false)
+			}
+		})
+	})
 })

src/tokenize.ts

@@ -6,6 +6,7 @@ import {
 	CHAR_WHITESPACE,
 	CHAR_NEWLINE,
 	CHAR_IDENT,
+	CHAR_IDENT_START,
 } from './char-types'
 
 // Local inline version for hot paths that still need it
@@ -197,6 +198,8 @@ export class Lexer {
 			}
 
 			let ch = source.charCodeAt(this.pos)
+			// One table read classifies ch for the dispatch below (non-ASCII → 0, handled via ch >= 128)
+			let ch_type = ch < 128 ? char_types[ch] : 0
 			let start = this.pos
 			let start_line = this._line
 			let start_column = this.pos - this._line_offset + 1
@@ -233,7 +236,7 @@ export class Lexer {
 			}
 
 			// Whitespace
-			if (ch < 128 && (char_types[ch] & (CHAR_WHITESPACE | CHAR_NEWLINE)) !== 0) {
+			if ((ch_type & (CHAR_WHITESPACE | CHAR_NEWLINE)) !== 0) {
 				return this.consume_whitespace(start_line, start_column)
 			}
 
@@ -252,7 +255,7 @@ export class Lexer {
 			// Identifier or function — checked first: the single most common non-whitespace
 			// token (~15-20% of all tokens), ahead of numbers/hashes/at-keywords/strings and
 			// the now-extinct CDO/CDC tokens that used to precede it (profiled on bootstrap.css/tailwind.css).
-			if (is_ident_start(ch)) {
+			if (ch >= 128 || (ch_type & CHAR_IDENT_START) !== 0) {
 				return this.consume_ident_or_function(start_line, start_column)
 			}
 
@@ -296,7 +299,7 @@ export class Lexer {
 			}
 
 			// Numbers: digit or . followed by digit
-			if (ch < 128 && (char_types[ch] & CHAR_DIGIT) !== 0) {
+			if ((ch_type & CHAR_DIGIT) !== 0) {
 				return this.consume_number(start_line, start_column)
 			}