Merge pull request #6 from kodus/feature/ignore-empty-statements

Added tests and solution for ignoring empty statements

Author: thomasnordahl-dk

src/Tokenizer.php

@@ -55,14 +55,23 @@ protected function statements()
     {
         $statements = [];
 
-        while ($result = $this->statement()) {
-            $statements[] = $result;
-        }
+        do {
+            $result = $this->statement();
+            if ($result !== null && count($result) > 0) {
+                $statements[] = $result;
+            }
+        } while ($result !== null);
 
         return $statements;
     }
 
     /**
+     * Returns an array containing the tokens in the next statement of the input.
+     *
+     * Returns empty array on empty statements (i.e. repeated delimiters, like: ;;;).
+     *
+     * Returns null if End-Of-File reached.
+     *
      * @return string[]|null
      */
     protected function statement()
@@ -76,6 +85,10 @@ protected function statement()
         $tokens = [];
 
         while ("" !== $token = $this->token()) {
+            /**
+             * DEV NOTE: This checks for DELIMITER statement, that changes delimiter from ; to something else.
+             * If detected, it will extract the new DELIMITER and reassign $this->delimiter_pattern.
+             */
             if (is_string($token) && preg_match('/^delimiter$/i', $token) === 1) {
                 $this->consume('[ ]*');
 
@@ -218,7 +231,7 @@ protected function grouped()
                 if ($this->is($closing)) {
                     $tokens[] = $closing;
 
-                    $this->offset +=1;
+                    $this->offset += 1;
 
                     return $tokens;
                 }

tests/unit/TokenizerCest.php

@@ -138,6 +138,26 @@ protected function useCases(): array
                     ["/* three\nfour */", "\n", "SELECT", " ", "2"],
                 ],
             ],
+            [
+                self::MESSAGE  => "Empty statement - in between",
+                self::INPUT    => "SELECT 1;; SELECT 2",
+                self::EXPECTED => [["SELECT", " ", "1"], ["SELECT", " ", "2"]],
+            ],
+            [
+                self::MESSAGE  => "Empty statements - first",
+                self::INPUT    => ";;SELECT 1; SELECT 2;",
+                self::EXPECTED => [["SELECT", " ", "1"], ["SELECT", " ", "2"]],
+            ],
+            [
+                self::MESSAGE  => "Empty statements - last",
+                self::INPUT    => "SELECT 1; SELECT 2;;;",
+                self::EXPECTED => [["SELECT", " ", "1"], ["SELECT", " ", "2"]],
+            ],
+            [
+                self::MESSAGE  => "Empty statements - all over",
+                self::INPUT    => ";;;;SELECT 1;;;; SELECT 2;;;",
+                self::EXPECTED => [["SELECT", " ", "1"], ["SELECT", " ", "2"]],
+            ],
         ];
     }
 }