Skip to content

Commit fd0cba9

Browse files
aristatharistath
committed
Merge Tokens_List in Lexer
1 parent 7e1af11 commit fd0cba9

4 files changed

Lines changed: 103 additions & 149 deletions

File tree

‎wp-includes/sqlite/class-wp-sqlite-lexer.php‎

Lines changed: 101 additions & 38 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
<?php
22
/**
3-
* This file is a port of the Lexer class from the PHPMyAdmin/sql-parser library.
3+
* This file is a port of the Lexer & Tokens_List classes from the PHPMyAdmin/sql-parser library.
44
*
55
* @package wp-sqlite-integration
66
* @see https://github.com/phpmyadmin/sql-parser
@@ -120,13 +120,6 @@ class WP_SQLite_Lexer {
120120
*/
121121
public $last = 0;
122122

123-
/**
124-
* Tokens extracted from given strings.
125-
*
126-
* @var WP_SQLite_Tokens_List
127-
*/
128-
public $list;
129-
130123
/**
131124
* The default delimiter. This is used, by default, in all new instances.
132125
*
@@ -1410,18 +1403,25 @@ class WP_SQLite_Lexer {
14101403
public const SQL_MODE_ANSI_QUOTES = 2;
14111404

14121405
/**
1413-
* Gets the tokens list parsed by a new instance of a lexer.
1406+
* The array of tokens.
14141407
*
1415-
* @param string $str The query to be lexed.
1416-
* @param string $delimiter The delimiter to be used.
1408+
* @var stdClass[]
1409+
*/
1410+
public $tokens = array();
1411+
1412+
/**
1413+
* The count of tokens.
14171414
*
1418-
* @return WP_SQLite_Tokens_List
1415+
* @var int
14191416
*/
1420-
public static function get_tokens( $str, $delimiter = null ) {
1421-
$lexer = new self( $str, $delimiter );
1417+
public $tokens_count = 0;
14221418

1423-
return $lexer->list;
1424-
}
1419+
/**
1420+
* The index of the next token to be returned.
1421+
*
1422+
* @var int
1423+
*/
1424+
public $tokens_index = 0;
14251425

14261426
/**
14271427
* The object constructor.
@@ -1469,8 +1469,6 @@ public function lex() {
14691469
* Another example is `parse_comment`.
14701470
*/
14711471

1472-
$list = new WP_SQLite_Tokens_List();
1473-
14741472
/**
14751473
* Last processed token.
14761474
*
@@ -1529,7 +1527,7 @@ public function lex() {
15291527

15301528
$token->position = $last_idx;
15311529

1532-
$list->tokens[ $list->count++ ] = $token;
1530+
$this->tokens[ $this->tokens_count++ ] = $token;
15331531

15341532
// Handling delimiters.
15351533
if ( WP_SQLite_Token::TYPE_NONE === $token->type && 'DELIMITER' === $token->value ) {
@@ -1546,8 +1544,8 @@ public function lex() {
15461544
$token = $this->parse_whitespace();
15471545

15481546
if ( null !== $token ) {
1549-
$token->position = $pos;
1550-
$list->tokens[ $list->count++ ] = $token;
1547+
$token->position = $pos;
1548+
$this->tokens[ $this->tokens_count++ ] = $token;
15511549
}
15521550

15531551
// Preparing the token that holds the new delimiter.
@@ -1578,20 +1576,17 @@ public function lex() {
15781576
--$this->last;
15791577

15801578
// Saving the delimiter and its token.
1581-
$this->delimiter_length = strlen( $this->delimiter );
1582-
$token = new WP_SQLite_Token( $this->delimiter, WP_SQLite_Token::TYPE_DELIMITER );
1583-
$token->position = $pos;
1584-
$list->tokens[ $list->count++ ] = $token;
1579+
$this->delimiter_length = strlen( $this->delimiter );
1580+
$token = new WP_SQLite_Token( $this->delimiter, WP_SQLite_Token::TYPE_DELIMITER );
1581+
$token->position = $pos;
1582+
$this->tokens[ $this->tokens_count++ ] = $token;
15851583
}
15861584

15871585
$last_token = $token;
15881586
}
15891587

15901588
// Adding a final delimiter to mark the ending.
1591-
$list->tokens[ $list->count++ ] = new WP_SQLite_Token( null, WP_SQLite_Token::TYPE_DELIMITER );
1592-
1593-
// Saving the tokens list.
1594-
$this->list = $list;
1589+
$this->tokens[ $this->tokens_count++ ] = new WP_SQLite_Token( null, WP_SQLite_Token::TYPE_DELIMITER );
15951590

15961591
$this->solve_ambiguity_on_star_operator();
15971592
$this->solve_ambiguity_on_function_keywords();
@@ -1613,14 +1608,14 @@ public function lex() {
16131608
* @return void
16141609
*/
16151610
private function solve_ambiguity_on_star_operator() {
1616-
$i_bak = $this->list->index;
1611+
$i_bak = $this->tokens_index;
16171612
while ( true ) {
1618-
$star_token = $this->list->get_next_of_type_and_value( WP_SQLite_Token::TYPE_OPERATOR, '*' );
1613+
$star_token = $this->tokens_get_next_of_type_and_value( WP_SQLite_Token::TYPE_OPERATOR, '*' );
16191614
if ( null === $star_token ) {
16201615
break;
16211616
}
1622-
// get_next() already gets rid of whitespaces and comments.
1623-
$next = $this->list->get_next();
1617+
// tokens_get_next() already gets rid of whitespaces and comments.
1618+
$next = $this->tokens_get_next();
16241619

16251620
if ( null === $next ) {
16261621
continue;
@@ -1636,7 +1631,7 @@ private function solve_ambiguity_on_star_operator() {
16361631
$star_token->flags = WP_SQLite_Token::FLAG_OPERATOR_SQL;
16371632
}
16381633

1639-
$this->list->index = $i_bak;
1634+
$this->tokens_index = $i_bak;
16401635
}
16411636

16421637
/**
@@ -1661,14 +1656,14 @@ private function solve_ambiguity_on_star_operator() {
16611656
* @return void
16621657
*/
16631658
private function solve_ambiguity_on_function_keywords() {
1664-
$i_bak = $this->list->index;
1659+
$i_bak = $this->tokens_index;
16651660
$keyword_function = WP_SQLite_Token::TYPE_KEYWORD | WP_SQLite_Token::FLAG_KEYWORD_FUNCTION;
16661661
while ( true ) {
1667-
$keyword_token = $this->list->get_next_of_type_and_flag( WP_SQLite_Token::TYPE_KEYWORD, $keyword_function );
1662+
$keyword_token = $this->tokens_get_next_of_type_and_flag( WP_SQLite_Token::TYPE_KEYWORD, $keyword_function );
16681663
if ( null === $keyword_token ) {
16691664
break;
16701665
}
1671-
$next = $this->list->get_next();
1666+
$next = $this->tokens_get_next();
16721667
if (
16731668
( WP_SQLite_Token::TYPE_KEYWORD !== $next->type
16741669
|| ! in_array( $next->value, $this->keyword_name_indicators, true )
@@ -1686,7 +1681,7 @@ private function solve_ambiguity_on_function_keywords() {
16861681
$keyword_token->keyword = $keyword_token->value;
16871682
}
16881683

1689-
$this->list->index = $i_bak;
1684+
$this->tokens_index = $i_bak;
16901685
}
16911686

16921687
/**
@@ -2512,4 +2507,72 @@ public static function is_separator( $str ) {
25122507
&& ( ( $str < 'a' ) || ( $str > 'z' ) )
25132508
&& ( ( $str < 'A' ) || ( $str > 'Z' ) );
25142509
}
2510+
2511+
/**
2512+
* Constructor.
2513+
*
2514+
* @param stdClass[] $tokens The initial array of tokens.
2515+
* @param int $count The count of tokens in the initial array.
2516+
*/
2517+
public function tokens( array $tokens = array(), $count = -1 ) {
2518+
if ( empty( $tokens ) ) {
2519+
return;
2520+
}
2521+
2522+
$this->tokens = $tokens;
2523+
$this->tokens_count = -1 === $count ? count( $tokens ) : $count;
2524+
}
2525+
2526+
/**
2527+
* Gets the next token.
2528+
*
2529+
* @param int $type The type of the token.
2530+
* @param int $flag The flag of the token.
2531+
*/
2532+
public function tokens_get_next_of_type_and_flag( int $type, int $flag ) {
2533+
for ( ; $this->tokens_index < $this->tokens_count; ++$this->tokens_index ) {
2534+
if ( ( $this->tokens[ $this->tokens_index ]->type === $type ) && ( $this->tokens[ $this->tokens_index ]->flags === $flag ) ) {
2535+
return $this->tokens[ $this->tokens_index++ ];
2536+
}
2537+
}
2538+
2539+
return null;
2540+
}
2541+
2542+
/**
2543+
* Gets the next token.
2544+
*
2545+
* @param int $type The type of the token.
2546+
* @param string $value The value of the token.
2547+
*
2548+
* @return stdClass|null
2549+
*/
2550+
public function tokens_get_next_of_type_and_value( $type, $value ) {
2551+
for ( ; $this->tokens_index < $this->tokens_count; ++$this->tokens_index ) {
2552+
if ( ( $this->tokens[ $this->tokens_index ]->type === $type ) && ( $this->tokens[ $this->tokens_index ]->value === $value ) ) {
2553+
return $this->tokens[ $this->tokens_index++ ];
2554+
}
2555+
}
2556+
2557+
return null;
2558+
}
2559+
2560+
/**
2561+
* Gets the next token. Skips any irrelevant token (whitespaces and
2562+
* comments).
2563+
*
2564+
* @return stdClass|null
2565+
*/
2566+
public function tokens_get_next() {
2567+
for ( ; $this->tokens_index < $this->tokens_count; ++$this->tokens_index ) {
2568+
if (
2569+
( WP_SQLite_Token::TYPE_WHITESPACE !== $this->tokens[ $this->tokens_index ]->type )
2570+
&& ( WP_SQLite_Token::TYPE_COMMENT !== $this->tokens[ $this->tokens_index ]->type )
2571+
) {
2572+
return $this->tokens[ $this->tokens_index++ ];
2573+
}
2574+
}
2575+
2576+
return null;
2577+
}
25152578
}

‎wp-includes/sqlite/class-wp-sqlite-tokens-list.php‎

Lines changed: 0 additions & 108 deletions
This file was deleted.

‎wp-includes/sqlite/class-wp-sqlite-translator.php‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -781,7 +781,7 @@ public function get_return_value() {
781781
public function translate( string $query, $last_found_rows = null ) {
782782
$this->last_found_rows = $last_found_rows;
783783

784-
$tokens = WP_SQLite_Lexer::get_tokens( $query )->tokens;
784+
$tokens = ( new WP_SQLite_Lexer( $query ) )->tokens;
785785
$this->rewriter = new WP_SQLite_Query_Rewriter( $tokens );
786786
$this->query_type = $this->rewriter->peek()->value;
787787

@@ -2289,7 +2289,7 @@ private function translate_alter() {
22892289
// 2. Adjust the column definition.
22902290

22912291
// First, tokenize the old schema.
2292-
$tokens = WP_SQLite_Lexer::get_tokens( $old_schema )->tokens;
2292+
$tokens = ( new WP_SQLite_Lexer( $old_schema ) )->tokens;
22932293
$create_table = new WP_SQLite_Query_Rewriter( $tokens );
22942294

22952295
// Now, replace every reference to the old column name with the new column name.

‎wp-includes/sqlite/db.php‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -50,7 +50,6 @@
5050
require_once __DIR__ . '/class-wp-sqlite-query-rewriter.php';
5151
require_once __DIR__ . '/class-wp-sqlite-translator.php';
5252
require_once __DIR__ . '/class-wp-sqlite-token.php';
53-
require_once __DIR__ . '/class-wp-sqlite-tokens-list.php';
5453
require_once __DIR__ . '/class-wp-sqlite-pdo-user-defined-functions.php';
5554
require_once __DIR__ . '/class-wp-sqlite-object-array.php';
5655
require_once __DIR__ . '/class-wp-sqlite-db.php';

0 commit comments

Comments
 (0)