Added tests and solution for ignoring empty statements

Author: thomasnordahl-dk

src/Tokenizer.php

@@ -55,14 +55,23 @@ protected function statements()
     {
         $statements = [];
 
-        while ($result = $this->statement()) {
-            $statements[] = $result;
-        }
+        do {
+            $result = $this->statement();
+            if ($result !== null && count($result) > 0) {
+                $statements[] = $result;
+            }
+        } while ($result !== null);
 
         return $statements;
     }
 
     /**
+     * Returns an array containing the tokens in the next statement of the input.
+     *
+     * Returns empty array on empty statements (i.e. repeated delimiters, like: ;;;).
+     *
+     * Returns null if End-Of-File reached.
+     *
      * @return string[]|null
      */
     protected function statement()
@@ -76,6 +85,10 @@ protected function statement()
         $tokens = [];
 
         while ("" !== $token = $this->token()) {
+            /**
+             * DEV NOTE: This checks for DELIMITER statement, that changes delimiter from ; to something else.
+             * If detected, it will extract the new DELIMITER and reassign $this->delimiter_pattern.
+             */
             if (is_string($token) && preg_match('/^delimiter$/i', $token) === 1) {
                 $this->consume('[ ]*');
 
@@ -218,7 +231,7 @@ protected function grouped()
                 if ($this->is($closing)) {
                     $tokens[] = $closing;
 
-                    $this->offset +=1;
+                    $this->offset += 1;
 
                     return $tokens;
                 }

test/test.php

@@ -40,6 +40,16 @@ function () {
     }
 );
 
+test(
+    'Empty statements',
+    function () {
+        eq(Tokenizer::tokenize("SELECT 1;; SELECT 2"), [["SELECT", " ", "1"], ["SELECT", " ", "2"]], "Empty statement - in between");
+        eq(Tokenizer::tokenize(";;SELECT 1; SELECT 2;"), [["SELECT", " ", "1"], ["SELECT", " ", "2"]], "Empty statements - first");
+        eq(Tokenizer::tokenize("SELECT 1; SELECT 2;;;"), [["SELECT", " ", "1"], ["SELECT", " ", "2"]], "Empty statements - last");
+        eq(Tokenizer::tokenize(";;;;SELECT 1;;;; SELECT 2;;;"), [["SELECT", " ", "1"], ["SELECT", " ", "2"]], "Empty statements - all over");
+    }
+);
+
 test(
     'comments',
     function () {