From ff0e98988fbb7691bb5e9eeb7808366d2bf8fe02 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 21 Oct 2026 17:24:36 +0000 Subject: [PATCH] feat!: rename next_token_fast to next_token and drop the Token-returning next_token Lexer.next_token() now returns the token type and exposes the position via token_start, token_end, token_line and token_column. tokenize() still yields Token objects. --- src/parse-anplusb.ts | 8 +- src/parse-atrule-prelude.ts | 2 +- src/parse-condition.ts | 2 +- src/parse-declaration.ts | 20 +-- src/parse-selector.ts | 20 +-- src/parse.ts | 2 +- src/tokenize.test.ts | 234 +++++++++++++++++++----------------- src/tokenize.ts | 32 ++--- src/value-node-parser.ts | 22 ++-- 9 files changed, 174 insertions(+), 168 deletions(-) diff --git a/src/parse-anplusb.ts b/src/parse-anplusb.ts index 619262b..03fadca 100644 --- a/src/parse-anplusb.ts +++ b/src/parse-anplusb.ts @@ -54,7 +54,7 @@ export class ANplusBParser { } // Get first token - this.lexer.next_token_fast(true) + this.lexer.next_token(true) // Handle special keywords: odd, even if (this.lexer.token_type === TOKEN_IDENT) { @@ -153,7 +153,7 @@ export class ANplusBParser { ) { // Look ahead for 'n' const saved = this.lexer.save_position() - this.lexer.next_token_fast(true) + this.lexer.next_token(true) if ((this.lexer.token_type as TokenType) === TOKEN_IDENT) { const text = this.source.substring(this.lexer.token_start, this.lexer.token_end) @@ -249,7 +249,7 @@ export class ANplusBParser { return null } - this.lexer.next_token_fast(true) + this.lexer.next_token(true) // Check for + or - delim if (this.lexer.token_type === TOKEN_DELIM) { @@ -259,7 +259,7 @@ export class ANplusBParser { const sign = ch === CHAR_MINUS_HYPHEN ? '-' : '' this.skip_whitespace() - this.lexer.next_token_fast(true) + this.lexer.next_token(true) if ((this.lexer.token_type as TokenType) === TOKEN_NUMBER) { let num_text = this.source.substring(this.lexer.token_start, this.lexer.token_end) diff --git a/src/parse-atrule-prelude.ts b/src/parse-atrule-prelude.ts index 36d1bd9..aeaf824 100644 --- a/src/parse-atrule-prelude.ts +++ b/src/parse-atrule-prelude.ts @@ -707,7 +707,7 @@ export class AtRulePreludeParser { this.lexer.token_type = TOKEN_EOF return TOKEN_EOF } - return this.lexer.next_token_fast(false) + return this.lexer.next_token(false) } // Parse @namespace prelude: [prefix] url("...") | "..." diff --git a/src/parse-condition.ts b/src/parse-condition.ts index de57e34..f0cb1dd 100644 --- a/src/parse-condition.ts +++ b/src/parse-condition.ts @@ -91,7 +91,7 @@ export class ConditionParser { this.lexer.token_type = TOKEN_EOF return TOKEN_EOF } - return this.lexer.next_token_fast(false) + return this.lexer.next_token(false) } // Scan tokens from just after an already-open '(' or function-call (depth 1) to its diff --git a/src/parse-declaration.ts b/src/parse-declaration.ts index 0e3f726..cfbe8fb 100644 --- a/src/parse-declaration.ts +++ b/src/parse-declaration.ts @@ -56,7 +56,7 @@ export class DeclarationParser { // Create a fresh lexer instance for standalone parsing const lexer = new Lexer(this.source) lexer.seek(start, line, column) - lexer.next_token_fast(true) // skip whitespace like Parser does + lexer.next_token(true) // skip whitespace like Parser does return this.parse_declaration_with_lexer(lexer, end) } @@ -116,7 +116,7 @@ export class DeclarationParser { browser_hack_line = lexer.token_line browser_hack_column = lexer.token_column - lexer.next_token_fast(true) // skip whitespace + lexer.next_token(true) // skip whitespace if ((lexer.token_type as TokenType) === TOKEN_IDENT) { has_browser_hack = true @@ -145,7 +145,7 @@ export class DeclarationParser { // Lookahead: save lexer state before consuming const saved = lexer.save_position_into(this.lookahead_position) - lexer.next_token_fast(true) // consume property name, skip whitespace + lexer.next_token(true) // consume property name, skip whitespace // Expect ':' (type assertion needed because TS doesn't know next_token mutates token_type) if ((lexer.token_type as TokenType) !== TOKEN_COLON) { @@ -207,7 +207,7 @@ export class DeclarationParser { if (stop_ch === CHAR_SEMICOLON && paren_depth === 0) { value_end = skip_whitespace_and_comments_backward(this.source, lexer.pos, value_start) // Tokenize ';' so token_type reflects it, matching what the old loop left behind. - lexer.next_token_fast(false) + lexer.next_token(false) break } @@ -224,7 +224,7 @@ export class DeclarationParser { } // Tokenize '}' (without consuming it) so peek_type() sees it - the enclosing // block's own loop needs to see '}' to know it's done. - lexer.next_token_fast(false) + lexer.next_token(false) break } @@ -239,16 +239,16 @@ export class DeclarationParser { value_end = lexer.pos lexer.pos++ // consume '!' // Check if next token is an identifier (doesn't verify it's literally "important") - const next_type = lexer.next_token_fast(true) // skip whitespace + const next_type = lexer.next_token(true) // skip whitespace if (next_type === TOKEN_IDENT) { has_important = true last_end = lexer.token_end - lexer.next_token_fast(true) // Advance to next token after "important", skip whitespace + lexer.next_token(true) // Advance to next token after "important", skip whitespace break } // '!' wasn't followed by an identifier - treat the already-peeked token as // ordinary content; lexer.pos is already past it, so the raw scan picks up - // naturally (an extra next_token_fast here would swallow e.g. a trailing '}'). + // naturally (an extra next_token here would swallow e.g. a trailing '}'). last_end = lexer.token_end value_end = last_end continue @@ -258,7 +258,7 @@ export class DeclarationParser { // Ran out of input before finding any of the above last_end = skip_whitespace_and_comments_backward(this.source, lexer.pos, value_start) value_end = last_end - lexer.next_token_fast(false) // tokenize EOF so token_type reflects it + lexer.next_token(false) // tokenize EOF so token_type reflects it break } @@ -327,7 +327,7 @@ export class DeclarationParser { // Consume ';' if present if ((lexer.token_type as TokenType) === TOKEN_SEMICOLON) { last_end = lexer.token_end - lexer.next_token_fast(true) // skip whitespace + lexer.next_token(true) // skip whitespace } // Set declaration length diff --git a/src/parse-selector.ts b/src/parse-selector.ts index 668a214..3c74939 100644 --- a/src/parse-selector.ts +++ b/src/parse-selector.ts @@ -144,7 +144,7 @@ export class SelectorParser { if (this.lexer.pos >= this.selector_end) break // Check for comma (comments are auto-skipped by lexer) - this.lexer.next_token_fast(false) + this.lexer.next_token(false) const token_type: TokenType = this.lexer.token_type if (token_type === TOKEN_COMMA) { @@ -194,7 +194,7 @@ export class SelectorParser { if (allow_relative && this.lexer.pos < this.selector_end) { const saved = this.lexer.save_position() - this.lexer.next_token_fast(false) + this.lexer.next_token(false) const token_type = this.lexer.token_type // Check if token is a combinator @@ -263,7 +263,7 @@ export class SelectorParser { while (this.lexer.pos < this.selector_end) { // Save lexer state before getting token const saved = this.lexer.save_position() - this.lexer.next_token_fast(false) + this.lexer.next_token(false) if (this.lexer.token_start >= this.selector_end) break @@ -358,7 +358,7 @@ export class SelectorParser { namespace_prefix_length: number, ): number | null { const saved = this.lexer.save_position() - this.lexer.next_token_fast(false) + this.lexer.next_token(false) let node_type: number if (this.lexer.token_type === TOKEN_IDENT) { @@ -471,7 +471,7 @@ export class SelectorParser { return null } - this.lexer.next_token_fast(false) + this.lexer.next_token(false) // A comma ends the selector: whitespace before it separates selectors, it is not a // descendant combinator (`a , b` is `a` and `b`, not `a` and `b`) @@ -516,7 +516,7 @@ export class SelectorParser { const saved = this.lexer.save_position() // Next token should be identifier - this.lexer.next_token_fast(false) + this.lexer.next_token(false) if (this.lexer.token_type !== TOKEN_IDENT) { // Restore lexer state and return null this.lexer.restore_position(saved) @@ -535,7 +535,7 @@ export class SelectorParser { // Find matching ] while (this.lexer.pos < this.selector_end && bracket_depth > 0) { - this.lexer.next_token_fast(false) + this.lexer.next_token(false) const token_type = this.lexer.token_type if (token_type === TOKEN_LEFT_BRACKET) { bracket_depth++ @@ -707,7 +707,7 @@ export class SelectorParser { } // Next token should be identifier or function - this.lexer.next_token_fast(false) + this.lexer.next_token(false) const token_type = this.lexer.token_type if (token_type === TOKEN_IDENT) { @@ -754,7 +754,7 @@ export class SelectorParser { let end = this.lexer.token_end while (this.lexer.pos < this.selector_end && paren_depth > 0) { - this.lexer.next_token_fast(false) + this.lexer.next_token(false) const token_type = this.lexer.token_type if (token_type === TOKEN_LEFT_PAREN || token_type === TOKEN_FUNCTION) { paren_depth++ @@ -850,7 +850,7 @@ export class SelectorParser { let last_child: number | null = null while (this.lexer.pos < end) { - this.lexer.next_token_fast(false) + this.lexer.next_token(false) const token_type = this.lexer.token_type const token_start = this.lexer.token_start const token_end = this.lexer.token_end diff --git a/src/parse.ts b/src/parse.ts index 1a45ac6..f901025 100644 --- a/src/parse.ts +++ b/src/parse.ts @@ -135,7 +135,7 @@ export class Parser { // Advance to the next token, skipping whitespace private next_token(): void { - this.lexer.next_token_fast(true) + this.lexer.next_token(true) } // Peek at current token type diff --git a/src/tokenize.test.ts b/src/tokenize.test.ts index aa436ba..a5bded6 100644 --- a/src/tokenize.test.ts +++ b/src/tokenize.test.ts @@ -25,55 +25,67 @@ import { TOKEN_RIGHT_BRACE, TOKEN_DELIM, TOKEN_EOF, + type Token, } from './token-types' +function next(lexer: Lexer, skip_whitespace: boolean = false): Token { + const type = lexer.next_token(skip_whitespace) + return { + type, + start: lexer.token_start, + end: lexer.token_end, + line: lexer.token_line, + column: lexer.token_column, + } +} + describe('Lexer', () => { describe('single-character tokens', () => { test('should tokenize braces', () => { let lexer = new Lexer('{') - let token = lexer.next_token() + let token = next(lexer) expect(token?.type).toBe(TOKEN_LEFT_BRACE) expect(token?.start).toBe(0) expect(token?.end).toBe(1) lexer = new Lexer('}') - token = lexer.next_token() + token = next(lexer) expect(token?.type).toBe(TOKEN_RIGHT_BRACE) }) test('should tokenize colon and semicolon', () => { let lexer = new Lexer(':') - let token = lexer.next_token() + let token = next(lexer) expect(token?.type).toBe(TOKEN_COLON) lexer = new Lexer(';') - token = lexer.next_token() + token = next(lexer) expect(token?.type).toBe(TOKEN_SEMICOLON) }) test('should tokenize comma', () => { const lexer = new Lexer(',') - const token = lexer.next_token() + const token = next(lexer) expect(token?.type).toBe(TOKEN_COMMA) }) test('should tokenize brackets', () => { let lexer = new Lexer('[') - let token = lexer.next_token() + let token = next(lexer) expect(token?.type).toBe(TOKEN_LEFT_BRACKET) lexer = new Lexer(']') - token = lexer.next_token() + token = next(lexer) expect(token?.type).toBe(TOKEN_RIGHT_BRACKET) }) test('should tokenize parentheses', () => { let lexer = new Lexer('(') - let token = lexer.next_token() + let token = next(lexer) expect(token?.type).toBe(TOKEN_LEFT_PAREN) lexer = new Lexer(')') - token = lexer.next_token() + token = next(lexer) expect(token?.type).toBe(TOKEN_RIGHT_PAREN) }) }) @@ -81,31 +93,31 @@ describe('Lexer', () => { describe('whitespace', () => { test('should tokenize spaces', () => { const lexer = new Lexer(' ') - const token = lexer.next_token() + const token = next(lexer) expect(token?.type).toBe(TOKEN_WHITESPACE) expect(token!.end - token!.start).toBe(3) }) test('should tokenize tabs', () => { const lexer = new Lexer('\t\t') - const token = lexer.next_token() + const token = next(lexer) expect(token?.type).toBe(TOKEN_WHITESPACE) }) test('should tokenize newlines', () => { const lexer = new Lexer('\n\r\n') - const token = lexer.next_token() + const token = next(lexer) expect(token?.type).toBe(TOKEN_WHITESPACE) expect(lexer.line).toBe(3) }) test('should track line', () => { const lexer = new Lexer('a\nb') - let token = lexer.next_token() // a + let token = next(lexer) // a expect(token?.line).toBe(1) - lexer.next_token() // \n - token = lexer.next_token() // b + next(lexer) // \n + token = next(lexer) // b expect(token?.line).toBe(2) }) }) @@ -114,7 +126,7 @@ describe('Lexer', () => { test('should skip comments via callback', () => { const comments: any[] = [] const lexer = new Lexer('/* comment */', (info) => comments.push(info)) - const token = lexer.next_token() + const token = next(lexer) expect(comments).toHaveLength(1) expect(comments[0].start).toBe(0) @@ -126,7 +138,7 @@ describe('Lexer', () => { test('should handle unclosed comments', () => { const comments: any[] = [] const lexer = new Lexer('/* unclosed', (info) => comments.push(info)) - lexer.next_token() + next(lexer) expect(comments).toHaveLength(1) expect(comments[0].start).toBe(0) @@ -136,7 +148,7 @@ describe('Lexer', () => { test('should track lines in comments', () => { const comments: any[] = [] const lexer = new Lexer('/* line1\nline2 */', (info) => comments.push(info)) - lexer.next_token() + next(lexer) expect(comments).toHaveLength(1) expect(comments[0].line).toBe(1) @@ -147,44 +159,44 @@ describe('Lexer', () => { describe('strings', () => { test('should tokenize double-quoted strings', () => { const lexer = new Lexer('"hello"') - const token = lexer.next_token() + const token = next(lexer) expect(token?.type).toBe(TOKEN_STRING) expect(lexer.source.slice(token!.start, token!.end)).toBe('"hello"') }) test('should tokenize single-quoted strings', () => { const lexer = new Lexer("'hello'") - const token = lexer.next_token() + const token = next(lexer) expect(token?.type).toBe(TOKEN_STRING) }) test('should handle escaped quotes', () => { const lexer = new Lexer('"hello \\"world\\""') - const token = lexer.next_token() + const token = next(lexer) expect(token?.type).toBe(TOKEN_STRING) }) test('should handle hex escapes', () => { const lexer = new Lexer('"\\20\\000020"') - const token = lexer.next_token() + const token = next(lexer) expect(token?.type).toBe(TOKEN_STRING) }) test('should handle escaped newlines', () => { const lexer = new Lexer('"line\\\ncontinued"') - const token = lexer.next_token() + const token = next(lexer) expect(token?.type).toBe(TOKEN_STRING) }) test('should mark unclosed strings as bad', () => { const lexer = new Lexer('"unclosed\n') - const token = lexer.next_token() + const token = next(lexer) expect(token?.type).toBe(TOKEN_BAD_STRING) }) test('should mark EOF unclosed strings as bad', () => { const lexer = new Lexer('"unclosed') - const token = lexer.next_token() + const token = next(lexer) expect(token?.type).toBe(TOKEN_BAD_STRING) }) }) @@ -192,42 +204,42 @@ describe('Lexer', () => { describe('numbers', () => { test('should tokenize integers', () => { const lexer = new Lexer('123') - const token = lexer.next_token() + const token = next(lexer) expect(token?.type).toBe(TOKEN_NUMBER) expect(lexer.source.slice(token!.start, token!.end)).toBe('123') }) test('should tokenize decimals', () => { const lexer = new Lexer('123.456') - const token = lexer.next_token() + const token = next(lexer) expect(token?.type).toBe(TOKEN_NUMBER) expect(lexer.source.slice(token!.start, token!.end)).toBe('123.456') }) test('should tokenize decimals starting with dot', () => { const lexer = new Lexer('.5') - const token = lexer.next_token() + const token = next(lexer) expect(token?.type).toBe(TOKEN_NUMBER) }) test('should tokenize signed numbers', () => { let lexer = new Lexer('+10') - let token = lexer.next_token() + let token = next(lexer) expect(token?.type).toBe(TOKEN_NUMBER) lexer = new Lexer('-10') - token = lexer.next_token() + token = next(lexer) expect(token?.type).toBe(TOKEN_NUMBER) }) test('should tokenize scientific notation', () => { let lexer = new Lexer('1e10') - let token = lexer.next_token() + let token = next(lexer) expect(token?.type).toBe(TOKEN_NUMBER) expect(lexer.source.slice(token!.start, token!.end)).toBe('1e10') lexer = new Lexer('1.5e-10') - token = lexer.next_token() + token = next(lexer) expect(token?.type).toBe(TOKEN_NUMBER) }) }) @@ -235,35 +247,35 @@ describe('Lexer', () => { describe('percentages', () => { test('should tokenize percentages', () => { const lexer = new Lexer('50%') - const token = lexer.next_token() + const token = next(lexer) expect(token?.type).toBe(TOKEN_PERCENTAGE) expect(lexer.source.slice(token!.start, token!.end)).toBe('50%') }) test('should tokenize negative percentages', () => { const lexer = new Lexer('-50%') - const token = lexer.next_token() + const token = next(lexer) expect(token?.type).toBe(TOKEN_PERCENTAGE) expect(lexer.source.slice(token!.start, token!.end)).toBe('-50%') }) test('should tokenize positive percentages', () => { const lexer = new Lexer('+50%') - const token = lexer.next_token() + const token = next(lexer) expect(token?.type).toBe(TOKEN_PERCENTAGE) expect(lexer.source.slice(token!.start, token!.end)).toBe('+50%') }) test('should tokenize fraction percentages', () => { const lexer = new Lexer('+5.5%') - const token = lexer.next_token() + const token = next(lexer) expect(token?.type).toBe(TOKEN_PERCENTAGE) expect(lexer.source.slice(token!.start, token!.end)).toBe('+5.5%') }) test('should tokenize scientific fraction percentages', () => { const lexer = new Lexer('+5.2e-10%') - const token = lexer.next_token() + const token = next(lexer) expect(token?.type).toBe(TOKEN_PERCENTAGE) expect(lexer.source.slice(token!.start, token!.end)).toBe('+5.2e-10%') }) @@ -272,7 +284,7 @@ describe('Lexer', () => { describe('dimensions', () => { test('should tokenize pixel dimensions', () => { const lexer = new Lexer('10px') - const token = lexer.next_token() + const token = next(lexer) expect(token?.type).toBe(TOKEN_DIMENSION) expect(lexer.source.slice(token!.start, token!.end)).toBe('10px') }) @@ -281,7 +293,7 @@ describe('Lexer', () => { const units = ['em', 'rem', 'vh', 'vw', 'ch', 'ex', 'cm', 'mm', 'in', 'pt', 'pc'] for (const unit of units) { const lexer = new Lexer(`10${unit}`) - const token = lexer.next_token() + const token = next(lexer) expect(token?.type).toBe(TOKEN_DIMENSION) } }) @@ -290,21 +302,21 @@ describe('Lexer', () => { describe('identifiers', () => { test('should tokenize identifiers', () => { const lexer = new Lexer('color') - const token = lexer.next_token() + const token = next(lexer) expect(token?.type).toBe(TOKEN_IDENT) expect(lexer.source.slice(token!.start, token!.end)).toBe('color') }) test('should tokenize vendor prefixes', () => { const lexer = new Lexer('-webkit-transform') - const token = lexer.next_token() + const token = next(lexer) expect(token?.type).toBe(TOKEN_IDENT) expect(lexer.source.slice(token!.start, token!.end)).toBe('-webkit-transform') }) test('should tokenize CSS custom properties (variables)', () => { let lexer = new Lexer('--test') - const token = lexer.next_token() + const token = next(lexer) expect(token?.type).toBe(TOKEN_IDENT) expect(lexer.source.slice(token!.start, token!.end)).toBe('--test') @@ -312,7 +324,7 @@ describe('Lexer', () => { lexer = new Lexer('a{--custom-prop:1}') const tokens = [] while (true) { - const t = lexer.next_token() + const t = next(lexer) if (!t || t.type === TOKEN_EOF) break tokens.push(t) } @@ -332,13 +344,13 @@ describe('Lexer', () => { test('should tokenize identifiers with underscores', () => { const lexer = new Lexer('_private') - const token = lexer.next_token() + const token = next(lexer) expect(token?.type).toBe(TOKEN_IDENT) }) test('should tokenize identifiers with digits', () => { const lexer = new Lexer('grid2') - const token = lexer.next_token() + const token = next(lexer) expect(token?.type).toBe(TOKEN_IDENT) }) @@ -346,7 +358,7 @@ describe('Lexer', () => { test('should handle hex escape sequences', () => { // \32 = character code 0x32 (the digit '2') const lexer = new Lexer('\\32xl') - const token = lexer.next_token() + const token = next(lexer) expect(token?.type).toBe(TOKEN_IDENT) expect(lexer.source.slice(token!.start, token!.end)).toBe('\\32xl') }) @@ -354,7 +366,7 @@ describe('Lexer', () => { test('should handle character escape sequences', () => { // \: = escaped colon const lexer = new Lexer('\\:ease-in') - const token = lexer.next_token() + const token = next(lexer) expect(token?.type).toBe(TOKEN_IDENT) expect(lexer.source.slice(token!.start, token!.end)).toBe('\\:ease-in') }) @@ -365,7 +377,7 @@ describe('Lexer', () => { const lexer = new Lexer('.\\32xl\\:ease-in{}') const tokens = [] while (true) { - const t = lexer.next_token() + const t = next(lexer) if (!t || t.type === TOKEN_EOF) break tokens.push(t) } @@ -382,7 +394,7 @@ describe('Lexer', () => { // Per CSS spec, hex escapes can be followed by optional whitespace // \32 xl = character 0x32 followed by 'xl' const lexer = new Lexer('\\32 xl') - const token = lexer.next_token() + const token = next(lexer) expect(token?.type).toBe(TOKEN_IDENT) expect(lexer.source.slice(token!.start, token!.end)).toBe('\\32 xl') }) @@ -390,28 +402,28 @@ describe('Lexer', () => { test('should handle 6-digit hex escapes', () => { // Maximum 6 hex digits per escape const lexer = new Lexer('\\000032xl') - const token = lexer.next_token() + const token = next(lexer) expect(token?.type).toBe(TOKEN_IDENT) expect(lexer.source.slice(token!.start, token!.end)).toBe('\\000032xl') }) test('should handle multiple escapes in sequence', () => { const lexer = new Lexer('\\32\\33\\34') - const token = lexer.next_token() + const token = next(lexer) expect(token?.type).toBe(TOKEN_IDENT) expect(lexer.source.slice(token!.start, token!.end)).toBe('\\32\\33\\34') }) test('should handle escape at start of identifier', () => { const lexer = new Lexer('\\:hover') - const token = lexer.next_token() + const token = next(lexer) expect(token?.type).toBe(TOKEN_IDENT) expect(lexer.source.slice(token!.start, token!.end)).toBe('\\:hover') }) test('should handle escape at end of identifier', () => { const lexer = new Lexer('ease-in\\:') - const token = lexer.next_token() + const token = next(lexer) expect(token?.type).toBe(TOKEN_IDENT) expect(lexer.source.slice(token!.start, token!.end)).toBe('ease-in\\:') }) @@ -419,14 +431,14 @@ describe('Lexer', () => { test('should stop at newline after backslash', () => { // Backslash followed by newline is invalid in identifier const lexer = new Lexer('test\\\nmore') - const token = lexer.next_token() + const token = next(lexer) expect(token?.type).toBe(TOKEN_IDENT) expect(lexer.source.slice(token!.start, token!.end)).toBe('test') }) test('should handle escapes in functions', () => { const lexer = new Lexer('\\32xl\\:fn(') - const token = lexer.next_token() + const token = next(lexer) expect(token?.type).toBe(TOKEN_FUNCTION) expect(lexer.source.slice(token!.start, token!.end)).toBe('\\32xl\\:fn(') }) @@ -436,14 +448,14 @@ describe('Lexer', () => { describe('functions', () => { test('should tokenize functions', () => { const lexer = new Lexer('rgb(') - const token = lexer.next_token() + const token = next(lexer) expect(token?.type).toBe(TOKEN_FUNCTION) expect(lexer.source.slice(token!.start, token!.end)).toBe('rgb(') }) test('should tokenize vendor-prefixed functions', () => { const lexer = new Lexer('-webkit-calc(') - const token = lexer.next_token() + const token = next(lexer) expect(token?.type).toBe(TOKEN_FUNCTION) }) }) @@ -451,20 +463,20 @@ describe('Lexer', () => { describe('at-keywords', () => { test('should tokenize @media', () => { const lexer = new Lexer('@media') - const token = lexer.next_token() + const token = next(lexer) expect(token?.type).toBe(TOKEN_AT_KEYWORD) expect(lexer.source.slice(token!.start, token!.end)).toBe('@media') }) test('should tokenize @keyframes', () => { const lexer = new Lexer('@keyframes') - const token = lexer.next_token() + const token = next(lexer) expect(token?.type).toBe(TOKEN_AT_KEYWORD) }) test('should tokenize @-webkitkeyframes', () => { const lexer = new Lexer('@-webkitkeyframes') - const token = lexer.next_token() + const token = next(lexer) expect(token?.type).toBe(TOKEN_AT_KEYWORD) }) }) @@ -472,14 +484,14 @@ describe('Lexer', () => { describe('hash', () => { test('should tokenize hash colors', () => { const lexer = new Lexer('#fff') - const token = lexer.next_token() + const token = next(lexer) expect(token?.type).toBe(TOKEN_HASH) expect(lexer.source.slice(token!.start, token!.end)).toBe('#fff') }) test('should tokenize hash IDs', () => { const lexer = new Lexer('#main-content') - const token = lexer.next_token() + const token = next(lexer) expect(token?.type).toBe(TOKEN_HASH) }) }) @@ -487,13 +499,13 @@ describe('Lexer', () => { describe('CDO/CDC', () => { test('should tokenize CDO', () => { const lexer = new Lexer('') - const token = lexer.next_token() + const token = next(lexer) expect(token?.type).toBe(TOKEN_CDC) }) }) @@ -501,13 +513,13 @@ describe('Lexer', () => { describe('delimiters', () => { test('should tokenize unknown characters as delimiters', () => { const lexer = new Lexer('~') - const token = lexer.next_token() + const token = next(lexer) expect(token?.type).toBe(TOKEN_DELIM) }) test('should tokenize + as delimiter when not part of number', () => { const lexer = new Lexer('+ ') - const token = lexer.next_token() + const token = next(lexer) expect(token?.type).toBe(TOKEN_DELIM) }) }) @@ -515,8 +527,8 @@ describe('Lexer', () => { describe('EOF', () => { test('should return EOF token at end', () => { const lexer = new Lexer('a') - lexer.next_token() // a - const token = lexer.next_token() + next(lexer) // a + const token = next(lexer) expect(token?.type).toBe(TOKEN_EOF) }) }) @@ -526,7 +538,7 @@ describe('Lexer', () => { const lexer = new Lexer('.class { color: red; }') const tokens = [] let token - while ((token = lexer.next_token()) && token.type !== TOKEN_EOF) { + while ((token = next(lexer)) && token.type !== TOKEN_EOF) { tokens.push(token.type) } expect(tokens).toEqual([ @@ -549,7 +561,7 @@ describe('Lexer', () => { const lexer = new Lexer('width: 100px') const tokens = [] let token - while ((token = lexer.next_token()) && token.type !== TOKEN_EOF) { + while ((token = next(lexer)) && token.type !== TOKEN_EOF) { if (token.type !== TOKEN_WHITESPACE) { tokens.push(token.type) } @@ -563,23 +575,23 @@ describe('Lexer', () => { const lexer = new Lexer('body { color: red; }') let token - token = lexer.next_token() // 'body' + token = next(lexer) // 'body' expect(token?.line).toBe(1) expect(token?.column).toBe(1) - token = lexer.next_token() // whitespace + token = next(lexer) // whitespace expect(token?.line).toBe(1) expect(token?.column).toBe(5) - token = lexer.next_token() // '{' + token = next(lexer) // '{' expect(token?.line).toBe(1) expect(token?.column).toBe(6) - token = lexer.next_token() // whitespace + token = next(lexer) // whitespace expect(token?.line).toBe(1) expect(token?.column).toBe(7) - token = lexer.next_token() // 'color' + token = next(lexer) // 'color' expect(token?.line).toBe(1) expect(token?.column).toBe(8) }) @@ -588,27 +600,27 @@ describe('Lexer', () => { const lexer = new Lexer('body {\n color: red;\n}') let token - token = lexer.next_token() // 'body' + token = next(lexer) // 'body' expect(token?.line).toBe(1) expect(token?.column).toBe(1) - token = lexer.next_token() // whitespace + token = next(lexer) // whitespace expect(token?.line).toBe(1) expect(token?.column).toBe(5) - token = lexer.next_token() // '{' + token = next(lexer) // '{' expect(token?.line).toBe(1) expect(token?.column).toBe(6) - token = lexer.next_token() // '\n ' + token = next(lexer) // '\n ' expect(token?.line).toBe(1) expect(token?.column).toBe(7) - token = lexer.next_token() // 'color' + token = next(lexer) // 'color' expect(token?.line).toBe(2) expect(token?.column).toBe(3) - token = lexer.next_token() // ':' + token = next(lexer) // ':' expect(token?.line).toBe(2) expect(token?.column).toBe(8) }) @@ -617,15 +629,15 @@ describe('Lexer', () => { const lexer = new Lexer('a\nb') let token - token = lexer.next_token() // 'a' + token = next(lexer) // 'a' expect(token?.line).toBe(1) expect(token?.column).toBe(1) - token = lexer.next_token() // '\n' + token = next(lexer) // '\n' expect(token?.line).toBe(1) expect(token?.column).toBe(2) - token = lexer.next_token() // 'b' + token = next(lexer) // 'b' expect(token?.line).toBe(2) expect(token?.column).toBe(1) }) @@ -634,20 +646,20 @@ describe('Lexer', () => { const lexer = new Lexer('font-size: 16px;') let token - token = lexer.next_token() // 'font-size' + token = next(lexer) // 'font-size' expect(token?.line).toBe(1) expect(token?.column).toBe(1) expect(lexer.source.slice(token!.start, token!.end)).toBe('font-size') - token = lexer.next_token() // ':' + token = next(lexer) // ':' expect(token?.line).toBe(1) expect(token?.column).toBe(10) - token = lexer.next_token() // ' ' + token = next(lexer) // ' ' expect(token?.line).toBe(1) expect(token?.column).toBe(11) - token = lexer.next_token() // '16px' + token = next(lexer) // '16px' expect(token?.line).toBe(1) expect(token?.column).toBe(12) }) @@ -656,14 +668,14 @@ describe('Lexer', () => { const lexer = new Lexer('content: "hello world";') let token - token = lexer.next_token() // 'content' + token = next(lexer) // 'content' expect(token?.line).toBe(1) expect(token?.column).toBe(1) - token = lexer.next_token() // ':' - token = lexer.next_token() // ' ' + token = next(lexer) // ':' + token = next(lexer) // ' ' - token = lexer.next_token() // '"hello world"' + token = next(lexer) // '"hello world"' expect(token?.line).toBe(1) expect(token?.column).toBe(10) expect(token?.type).toBe(TOKEN_STRING) @@ -673,12 +685,12 @@ describe('Lexer', () => { const lexer = new Lexer('/* comment */ body') let token - token = lexer.next_token() // ' ' + token = next(lexer) // ' ' expect(token?.line).toBe(1) expect(token?.column).toBe(14) expect(token?.type).toBe(TOKEN_WHITESPACE) - token = lexer.next_token() // 'body' + token = next(lexer) // 'body' expect(token?.line).toBe(1) expect(token?.column).toBe(15) expect(token?.type).toBe(TOKEN_IDENT) @@ -688,12 +700,12 @@ describe('Lexer', () => { const lexer = new Lexer('/* line1\nline2 */ a') let token - token = lexer.next_token() // ' ' + token = next(lexer) // ' ' expect(token?.line).toBe(2) expect(token?.column).toBe(9) expect(token?.type).toBe(TOKEN_WHITESPACE) - token = lexer.next_token() // 'a' + token = next(lexer) // 'a' expect(token?.line).toBe(2) expect(token?.column).toBe(10) }) @@ -702,13 +714,13 @@ describe('Lexer', () => { const lexer = new Lexer('opacity: 0.5;') let token - token = lexer.next_token() // 'opacity' + token = next(lexer) // 'opacity' expect(token?.column).toBe(1) - token = lexer.next_token() // ':' - token = lexer.next_token() // ' ' + token = next(lexer) // ':' + token = next(lexer) // ' ' - token = lexer.next_token() // '0.5' + token = next(lexer) // '0.5' expect(token?.line).toBe(1) expect(token?.column).toBe(10) expect(token?.type).toBe(TOKEN_NUMBER) @@ -718,13 +730,13 @@ describe('Lexer', () => { const lexer = new Lexer('@media screen') let token - token = lexer.next_token() // '@media' + token = next(lexer) // '@media' expect(token?.line).toBe(1) expect(token?.column).toBe(1) expect(token?.type).toBe(TOKEN_AT_KEYWORD) - token = lexer.next_token() // ' ' - token = lexer.next_token() // 'screen' + token = next(lexer) // ' ' + token = next(lexer) // 'screen' expect(token?.line).toBe(1) expect(token?.column).toBe(8) }) @@ -733,22 +745,22 @@ describe('Lexer', () => { const lexer = new Lexer('a\r\nb') let token - token = lexer.next_token() // 'a' + token = next(lexer) // 'a' expect(token?.line).toBe(1) expect(token?.column).toBe(1) - token = lexer.next_token() // '\r\n' + token = next(lexer) // '\r\n' expect(token?.line).toBe(1) expect(token?.column).toBe(2) - token = lexer.next_token() // 'b' + token = next(lexer) // 'b' expect(token?.line).toBe(2) expect(token?.column).toBe(1) }) }) describe('first-character classification', () => { - // Independent oracle: what the dispatch in next_token_fast must decide from a lone first character + // Independent oracle: what the dispatch in next_token must decide from a lone first character const is_letter = (c: number) => (c >= 0x41 && c <= 0x5a) || (c >= 0x61 && c <= 0x7a) const oracle_ident_start = (c: number) => is_letter(c) || c === 0x5f || c >= 0x80 const oracle_whitespace = (c: number) => @@ -778,7 +790,7 @@ describe('Lexer', () => { const mismatches: string[] = [] for (const c of codes) { const lexer = new Lexer(String.fromCharCode(c)) - lexer.next_token_fast(false) + lexer.next_token(false) const type = lexer.token_type const ok = (type === TOKEN_IDENT) === oracle_ident_start(c) && @@ -795,7 +807,7 @@ describe('Lexer', () => { test('skip_whitespace=true lands on the same token as an explicit whitespace token', () => { const source = ' \t\n\r\f_x' const lexer = new Lexer(source) - lexer.next_token_fast(true) + lexer.next_token(true) expect(lexer.token_type).toBe(TOKEN_IDENT) expect(source.slice(lexer.token_start, lexer.token_end)).toBe('_x') expect(lexer.token_line).toBe(4) // \n, then \r, then \f each start a line @@ -804,7 +816,7 @@ describe('Lexer', () => { test('identifiers may start with underscore, a letter or a non-ASCII character', () => { for (const source of ['_a', 'a', 'Z', 'é', '日本', '\u{1F600}x']) { const lexer = new Lexer(source) - lexer.next_token_fast(false) + lexer.next_token(false) expect(lexer.token_type).toBe(TOKEN_IDENT) expect(lexer.token_end).toBe(source.length) } @@ -813,7 +825,7 @@ describe('Lexer', () => { test('digits start numbers and are not identifiers', () => { for (const source of ['0', '9', '5px', '.5']) { const lexer = new Lexer(source) - lexer.next_token_fast(false) + lexer.next_token(false) expect(lexer.token_type === TOKEN_IDENT).toBe(false) } }) diff --git a/src/tokenize.ts b/src/tokenize.ts index 4b65005..4185f4d 100644 --- a/src/tokenize.ts +++ b/src/tokenize.ts @@ -158,8 +158,8 @@ export class Lexer { this._line_offset = pos - column + 1 } - // Fast token advancing without object allocation (for internal parser use) - next_token_fast(skip_whitespace: boolean = false): TokenType { + // Advances to the next token and returns its type; position is exposed via token_start/token_end/token_line/token_column + next_token(skip_whitespace: boolean = false): TokenType { const source = this.source const source_length = source.length @@ -384,7 +384,7 @@ export class Lexer { } // Skip a comment (/* ... */), firing on_comment if set. Caller must have verified - // this.pos is at '/*'. Shared by next_token_fast and the skip_to_* scans below. + // this.pos is at '/*'. Shared by next_token and the skip_to_* scans below. private _skip_comment(): void { const source = this.source const source_length = source.length @@ -543,7 +543,7 @@ export class Lexer { const source = this.source const source_length = source.length const start = this.pos - // Hoisted to locals (synced back once), same as next_token_fast's inline skip. + // Hoisted to locals (synced back once), same as next_token's inline skip. let pos = start let line = this._line let line_offset = this._line_offset @@ -973,18 +973,6 @@ export class Lexer { return type } - // Public API: returns Token object for backwards compatibility - next_token(skip_whitespace: boolean = false): Token | null { - this.next_token_fast(skip_whitespace) - return { - type: this.token_type, - start: this.token_start, - end: this.token_end, - line: this.token_line, - column: this.token_column, - } - } - /** * Save complete lexer state for backtracking * @returns Object containing all lexer state @@ -1075,10 +1063,16 @@ export function* tokenize( const lexer = new Lexer(source, on_comment) while (true) { - const token = lexer.next_token() - if (!token || token.type === TOKEN_EOF) { + const type = lexer.next_token() + if (type === TOKEN_EOF) { break } - yield token + yield { + type, + start: lexer.token_start, + end: lexer.token_end, + line: lexer.token_line, + column: lexer.token_column, + } } } diff --git a/src/value-node-parser.ts b/src/value-node-parser.ts index 5a9c487..492c223 100644 --- a/src/value-node-parser.ts +++ b/src/value-node-parser.ts @@ -96,7 +96,7 @@ export class ValueNodeParser { while (this.lexer.pos < this.end) { // Whitespace is only a separator here, so let the lexer skip it instead of // emitting a token we'd immediately discard - this.lexer.next_token_fast(true) + this.lexer.next_token(true) // Stop if we've reached the end of the range if (this.lexer.token_start >= this.end) break @@ -211,7 +211,7 @@ export class ValueNodeParser { let close_end = this.lexer.token_end while ((!bounded || this.lexer.pos < this.end) && depth > 0) { - this.lexer.next_token_fast(false) + this.lexer.next_token(false) const token_type = this.lexer.token_type if (token_type === TOKEN_EOF) break if (bounded && this.lexer.token_start >= this.end) break @@ -267,11 +267,11 @@ export class ValueNodeParser { // Peek at the next token to see if it's a string // If it's a string, parse normally. Otherwise, skip parsing children. const save_pos = this.lexer.save_position() - this.lexer.next_token_fast(false) + this.lexer.next_token(false) // Skip whitespace while (this.lexer.token_type === TOKEN_WHITESPACE && this.lexer.pos < this.end) { - this.lexer.next_token_fast(false) + this.lexer.next_token(false) } const first_token_type = this.lexer.token_type @@ -320,7 +320,7 @@ export class ValueNodeParser { let content_end = end while (this.lexer.pos < this.end && paren_depth > 0) { - this.lexer.next_token_fast(true) + this.lexer.next_token(true) const token_type = this.lexer.token_type if (token_type === TOKEN_EOF) break @@ -390,7 +390,7 @@ export class ValueNodeParser { let if_closed = false while (this.lexer.pos < this.end && !if_closed) { - this.lexer.next_token_fast(false) + this.lexer.next_token(false) const tt = this.lexer.token_type if (tt === TOKEN_EOF) break @@ -419,7 +419,7 @@ export class ValueNodeParser { // ── Find the ':' separator ───────────────────────────────────────── let colon_found = false while (this.lexer.pos < this.end) { - this.lexer.next_token_fast(false) + this.lexer.next_token(false) const t = this.lexer.token_type if (t === TOKEN_EOF) break if (this.lexer.token_start >= this.end) break @@ -447,7 +447,7 @@ export class ValueNodeParser { if (colon_found && !if_closed) { while (this.lexer.pos < this.end) { - this.lexer.next_token_fast(false) + this.lexer.next_token(false) const t = this.lexer.token_type if (t === TOKEN_EOF) break if (this.lexer.token_start >= this.end) break @@ -525,7 +525,7 @@ export class ValueNodeParser { // committing to consuming it — callers restore to a saved position when it doesn't. private next_significant_token(): TokenType { while (this.lexer.pos < this.end) { - this.lexer.next_token_fast(false) + this.lexer.next_token(false) if (this.lexer.token_start >= this.end) return TOKEN_EOF if (this.lexer.token_type === TOKEN_WHITESPACE) continue return this.lexer.token_type @@ -739,7 +739,7 @@ export class ValueNodeParser { const nodes: number[] = [] while (this.lexer.pos < this.end) { - this.lexer.next_token_fast(true) + this.lexer.next_token(true) if (this.lexer.token_start >= this.end) break const token_type = this.lexer.token_type if (token_type === TOKEN_EOF) break @@ -771,7 +771,7 @@ export class ValueNodeParser { let paren_end = end while (this.lexer.pos < this.end && paren_depth > 0) { - this.lexer.next_token_fast(true) + this.lexer.next_token(true) const token_type = this.lexer.token_type if (token_type === TOKEN_EOF) break