all Postgres operators

Author: mindplay-dk

src/Tokenizer.php

@@ -117,12 +117,20 @@ protected function token()
             return $token;
         }
 
-        if ($token = $this->consume('[\*,.+-\/=;<>]')) {
-            return $token;
+        if ($token = $this->consume('\@\w+')) {
+            return $token; // @var
         }
 
-        if ($token = $this->consume('\@\w+')) {
-            return $token;
+        if ($token = $this->consume(':\w+')) {
+            return $token; // :var (PDO placeholder)
+        }
+
+        if ($token = $this->consume('[+\-\*\/.,!=^|&<>:@%~#]+')) {
+            return $token; // various operators
+        }
+
+        if ($token = $this->consume(';')) {
+            return $token; // statement separator (when $delimiter_pattern has been modified)
         }
 
         if ($token = $this->quoted()) {
@@ -133,10 +141,6 @@ protected function token()
             return $tokens;
         }
 
-        if ($token = $this->placeholder()) {
-            return $token;
-        }
-
         if ($token = $this->dollarquoted()) {
             return $token;
         }
@@ -230,18 +234,6 @@ protected function grouped()
         return null;
     }
 
-    /**
-     * @return string|null
-     */
-    protected function placeholder()
-    {
-        if ($token = $this->consume(':\w+')) {
-            return $token;
-        }
-
-        return null;
-    }
-
     /**
      * @return string|null
      */

test/test.php

@@ -36,6 +36,7 @@ function () {
         eq(Tokenizer::tokenize('CREATE FUNCTION foo AS $$RETURN $1$$;'), [["CREATE", " ", "FUNCTION", " ", "foo", " ", "AS", " ", '$$RETURN $1$$']], "stored procedure");
         eq(Tokenizer::tokenize('SELECT $$FOO$$; SELECT $$BAR$$'), [["SELECT", " ", '$$FOO$$'], ["SELECT", " ", '$$BAR$$']], "dollar-quoted strings");
         eq(Tokenizer::tokenize("SELECT 'foo\\'\\\\'"), [["SELECT", " ", "'foo\\'\\\\'"]]);
+        eq(Tokenizer::tokenize("SELECT '{\"one\":\"two\"}'::jsonb"), [["SELECT", " ", "'{\"one\":\"two\"}'", "::", "jsonb"]]);
     }
 );