diff --git a/src/char-types.ts b/src/char-types.ts index 60983de..0c62d6b 100644 --- a/src/char-types.ts +++ b/src/char-types.ts @@ -27,6 +27,7 @@ let CHAR_HEX = 1 << 2 // 4 export let CHAR_WHITESPACE = 1 << 3 // 8 export let CHAR_NEWLINE = 1 << 4 // 16 export let CHAR_IDENT = 1 << 5 // 32 +export let CHAR_IDENT_START = 1 << 6 // 64: ASCII letters and underscore (non-ASCII handled by callers via ch >= 0x80) // Lookup table for ASCII characters (0-127) export let char_types = new Uint8Array(128) @@ -74,17 +75,19 @@ for (let i = 0; i < 128; i++) { char_types[0x2d] |= CHAR_IDENT // hyphen - char_types[0x5f] |= CHAR_IDENT // underscore _ -export function is_hex_digit(ch: number): boolean { - return ch < 128 && (char_types[ch] & CHAR_HEX) !== 0 +// Ident-start: letters and underscore. Single-bit test replaces is_alpha() + '_' compare. +for (let i = 0; i < 128; i++) { + if (char_types[i] & CHAR_ALPHA) { + char_types[i] |= CHAR_IDENT_START + } } +char_types[0x5f] |= CHAR_IDENT_START // underscore _ -function is_alpha(ch: number): boolean { - return ch < 128 && (char_types[ch] & CHAR_ALPHA) !== 0 +export function is_hex_digit(ch: number): boolean { + return ch < 128 && (char_types[ch] & CHAR_HEX) !== 0 } // CSS ident start: letter, underscore, or non-ASCII (>= 0x80) export function is_ident_start(ch: number): boolean { - if (ch >= 0x80) return true // Non-ASCII - if (ch === 0x5f) return true // _ - return is_alpha(ch) + return ch >= 0x80 || (char_types[ch] & CHAR_IDENT_START) !== 0 } diff --git a/src/tokenize.test.ts b/src/tokenize.test.ts index 3d31c29..3e256ee 100644 --- a/src/tokenize.test.ts +++ b/src/tokenize.test.ts @@ -1,5 +1,6 @@ import { describe, test, expect } from 'vitest' import { Lexer } from './tokenize' +import { is_ident_start } from './char-types' import { TOKEN_IDENT, TOKEN_FUNCTION, @@ -745,4 +746,76 @@ describe('Lexer', () => { expect(token?.column).toBe(1) }) }) + + describe('first-character classification', () => { + // Independent oracle: what the dispatch in next_token_fast must decide from a lone first character + const is_letter = (c: number) => (c >= 0x41 && c <= 0x5a) || (c >= 0x61 && c <= 0x7a) + const oracle_ident_start = (c: number) => is_letter(c) || c === 0x5f || c >= 0x80 + const oracle_whitespace = (c: number) => + c === 0x20 || c === 0x09 || c === 0x0a || c === 0x0d || c === 0x0c + const oracle_digit = (c: number) => c >= 0x30 && c <= 0x39 + + test('is_ident_start matches letters, underscore and non-ASCII for every code unit', () => { + let mismatches: string[] = [] + for (let c = 0; c <= 0xffff; c++) { + if (is_ident_start(c) !== oracle_ident_start(c)) mismatches.push(`0x${c.toString(16)}`) + } + expect(mismatches).toEqual([]) + }) + + test('a lone character becomes IDENT, WHITESPACE or NUMBER exactly when the oracle says so', () => { + // Every ASCII code unit plus a spread of non-ASCII ones (Latin-1, BMP, lone surrogates) + const codes = [ + ...Array.from({ length: 0x180 }, (_, i) => i), + 0x2028, + 0x3042, + 0x65e5, + 0xd83d, + 0xde00, + 0xfeff, + 0xffff, + ] + let mismatches: string[] = [] + for (let c of codes) { + let lexer = new Lexer(String.fromCharCode(c)) + lexer.next_token_fast(false) + let type = lexer.token_type + let ok = + (type === TOKEN_IDENT) === oracle_ident_start(c) && + (type === TOKEN_WHITESPACE) === oracle_whitespace(c) && + (type === TOKEN_NUMBER) === oracle_digit(c) && + // Whatever it is, it must cover the whole single character + lexer.token_start === 0 && + lexer.token_end === 1 + if (!ok) mismatches.push(`0x${c.toString(16)} -> type ${type}`) + } + expect(mismatches).toEqual([]) + }) + + test('skip_whitespace=true lands on the same token as an explicit whitespace token', () => { + let source = ' \t\n\r\f_x' + let lexer = new Lexer(source) + lexer.next_token_fast(true) + expect(lexer.token_type).toBe(TOKEN_IDENT) + expect(source.slice(lexer.token_start, lexer.token_end)).toBe('_x') + expect(lexer.token_line).toBe(4) // \n, then \r, then \f each start a line + }) + + test('identifiers may start with underscore, a letter or a non-ASCII character', () => { + for (let source of ['_a', 'a', 'Z', 'é', '日本', '\u{1F600}x']) { + let lexer = new Lexer(source) + lexer.next_token_fast(false) + expect(lexer.token_type).toBe(TOKEN_IDENT) + expect(lexer.token_end).toBe(source.length) + } + }) + + test('digits start numbers and are not identifiers', () => { + for (let source of ['0', '9', '5px', '.5']) { + let lexer = new Lexer(source) + lexer.next_token_fast(false) + expect(lexer.token_type === TOKEN_IDENT).toBe(false) + } + }) + }) }) diff --git a/src/tokenize.ts b/src/tokenize.ts index 8694816..5a58c63 100644 --- a/src/tokenize.ts +++ b/src/tokenize.ts @@ -6,6 +6,7 @@ import { CHAR_WHITESPACE, CHAR_NEWLINE, CHAR_IDENT, + CHAR_IDENT_START, } from './char-types' // Local inline version for hot paths that still need it @@ -197,6 +198,8 @@ export class Lexer { } let ch = source.charCodeAt(this.pos) + // One table read classifies ch for the dispatch below (non-ASCII → 0, handled via ch >= 128) + let ch_type = ch < 128 ? char_types[ch] : 0 let start = this.pos let start_line = this._line let start_column = this.pos - this._line_offset + 1 @@ -233,7 +236,7 @@ export class Lexer { } // Whitespace - if (ch < 128 && (char_types[ch] & (CHAR_WHITESPACE | CHAR_NEWLINE)) !== 0) { + if ((ch_type & (CHAR_WHITESPACE | CHAR_NEWLINE)) !== 0) { return this.consume_whitespace(start_line, start_column) } @@ -252,7 +255,7 @@ export class Lexer { // Identifier or function — checked first: the single most common non-whitespace // token (~15-20% of all tokens), ahead of numbers/hashes/at-keywords/strings and // the now-extinct CDO/CDC tokens that used to precede it (profiled on bootstrap.css/tailwind.css). - if (is_ident_start(ch)) { + if (ch >= 128 || (ch_type & CHAR_IDENT_START) !== 0) { return this.consume_ident_or_function(start_line, start_column) } @@ -296,7 +299,7 @@ export class Lexer { } // Numbers: digit or . followed by digit - if (ch < 128 && (char_types[ch] & CHAR_DIGIT) !== 0) { + if ((ch_type & CHAR_DIGIT) !== 0) { return this.consume_number(start_line, start_column) }