all tests working

Author: brunovcosta

abstra_json_sql/apply.py

@@ -1,4 +1,4 @@
-from typing import List, Dict, Optional, Union
+from typing import List, Dict, Optional
 from .tables import ITablesSnapshot
 from .field_name import field_name, expression_name
 from .ast import (

abstra_json_sql/lexer.py

@@ -40,18 +40,24 @@ def extract_name(code: str):
         raise Exception(f"Not a valid name, code: {code}")
     result = ""
     for idx, char in enumerate(code):
-        if start_with_name(char) or any(
-            k.startswith((result + char).upper()) for k in keywords
-        ):
+        if start_with_name(char):
             result = result + char
-        elif result.upper() in keywords:
-            return Token("keyword", result), code[idx:]
         else:
             return Token("name", result), code[idx:]
-    if code.upper() in keywords:
-        return Token("keyword", code), ""
-    else:
-        return Token("name", code), ""
+    return Token("name", code), ""
+
+
+def start_with_keyword(code: str):
+    for keyword in keywords:
+        if code.upper().startswith(keyword.upper()):
+            return True
+    return False
+
+
+def extract_keyword(code: str):
+    for keyword in keywords:
+        if code.upper().startswith(keyword.upper()):
+            return Token("keyword", code[: len(keyword)]), code[len(keyword) :]
 
 
 def start_with_quoted_name(code: str):
@@ -131,6 +137,9 @@ def scan(code: str) -> List[Token]:
         elif start_with_number(code):
             token, code = extract_number(code)
             result.append(token)
+        elif start_with_keyword(code):
+            token, code = extract_keyword(code)
+            result.append(token)
         elif start_with_name(code):
             token, code = extract_name(code)
             result.append(token)

abstra_json_sql/parser.py

@@ -96,10 +96,8 @@ def parse_expression(tokens: List[Token]) -> Tuple[Expression, List[Token]]:
                 stack.append(NotExpression(expression=expression))
             elif next_token.value.upper() == "TRUE":
                 stack.append(TrueExpression())
-                tokens = tokens[1:]
             elif next_token.value.upper() == "FALSE":
                 stack.append(FalseExpression())
-                tokens = tokens[1:]
             elif next_token.value.upper() == "IS":
                 left = stack.pop()
                 right, tokens = parse_expression(tokens)
@@ -112,7 +110,7 @@ def parse_expression(tokens: List[Token]) -> Tuple[Expression, List[Token]]:
                 stack.append(NullExpression())
             else:
                 tokens = [next_token] + tokens
-                break
+            break
         if next_token.type == "int":
             stack.append(IntExpression(value=int(next_token.value)))
         elif next_token.type == "float":
@@ -240,21 +238,34 @@ def parse_group_by(tokens: List[Token]) -> Tuple[Optional[GroupBy], List[Token]]
             group_fields.append(exp)
     return GroupBy(fields=group_fields), tokens
 
+
 def parse_join(tokens: List[Token]) -> Tuple[Optional[Join], List[Token]]:
     if len(tokens) == 0:
         return None, tokens
-    
+
     # JOIN
-    if tokens[0].type == "keyword" and tokens[0].value.upper() in ["INNER JOIN", "JOIN"]:
+    if tokens[0].type == "keyword" and tokens[0].value.upper() in [
+        "INNER JOIN",
+        "JOIN",
+    ]:
         join_type = "INNER"
         tokens = tokens[1:]
-    elif tokens[0].type == "keyword" and tokens[0].value.upper() in ["LEFT JOIN", "LEFT OUTER JOIN"]:
+    elif tokens[0].type == "keyword" and tokens[0].value.upper() in [
+        "LEFT JOIN",
+        "LEFT OUTER JOIN",
+    ]:
         join_type = "LEFT"
         tokens = tokens[1:]
-    elif tokens[0].type == "keyword" and tokens[0].value.upper() in ["RIGHT JOIN", "RIGHT OUTER JOIN"]:
+    elif tokens[0].type == "keyword" and tokens[0].value.upper() in [
+        "RIGHT JOIN",
+        "RIGHT OUTER JOIN",
+    ]:
         join_type = "RIGHT"
         tokens = tokens[1:]
-    elif tokens[0].type == "keyword" and tokens[0].value.upper() in ["FULL JOIN", "FULL OUTER JOIN"]:
+    elif tokens[0].type == "keyword" and tokens[0].value.upper() in [
+        "FULL JOIN",
+        "FULL OUTER JOIN",
+    ]:
         join_type = "FULL"
         tokens = tokens[1:]
     elif tokens[0].type == "keyword" and tokens[0].value.upper() in ["CROSS JOIN"]:
@@ -265,12 +276,12 @@ def parse_join(tokens: List[Token]) -> Tuple[Optional[Join], List[Token]]:
         tokens = tokens[1:]
     else:
         return None, tokens
-    
+
     # Table
     table = tokens[0]
     assert table.type == "name", f"Expected table name, got {table}"
     tokens = tokens[1:]
-        
+
     # AS
     if (
         len(tokens) > 0
@@ -283,7 +294,7 @@ def parse_join(tokens: List[Token]) -> Tuple[Optional[Join], List[Token]]:
         tokens = tokens[1:]
     else:
         alias_token = None
-    
+
     # ON
     if (
         len(tokens) > 0
@@ -334,7 +345,7 @@ def parse_from(tokens: List[Token]) -> Tuple[Optional[From], List[Token]]:
         if j is None:
             break
         join.append(j)
-    
+
     if len(join) == 0:
         return From(table=table.value), tokens
     else:

abstra_json_sql/parser_test.py

@@ -11,7 +11,9 @@
     SelectField,
     IntExpression,
     NameExpression,
+    IsExpression,
     EqualExpression,
+    NullExpression,
     StringExpression,
     Where,
     OrderBy,
@@ -201,6 +203,33 @@ def test_limit_with_offset(self):
         self.assertEqual(tokens, [])
 
 
+class IsTest(TestCase):
+    def test_is_expression(self):
+        tokens = scan("name IS NULL")
+        ast, tokens = parse_expression(tokens)
+        self.assertEqual(
+            ast,
+            IsExpression(
+                left=NameExpression(name="name"),
+                right=NullExpression(),
+            ),
+        )
+        self.assertEqual(tokens, [])
+
+    def test_is_not_expression(self):
+        tokens = scan("name IS NOT NULL")
+        ast, tokens = parse_expression(tokens)
+        self.assertEqual(
+            ast,
+            IsExpression(
+                left=NameExpression(name="name"),
+                right=NullExpression(),
+                is_not=True,
+            ),
+        )
+        self.assertEqual(tokens, [])
+
+
 class AcceptKeywordTest(TestCase):
     def test_accept_keyword(self):
         tokens = [

abstra_json_sql/tokens.py

@@ -23,7 +23,6 @@
     "AND",
     "AS",
     "ON",
-    "OR",
     "NOT",
     "IN",
     "LIKE",
@@ -40,6 +39,7 @@
     "HAVING",
     "ASC",
     "DESC",
+    "OR",
     "INNER JOIN",
     "RIGHT OUTER JOIN",
     "LEFT OUTER JOIN",
@@ -52,8 +52,8 @@
     "LIMIT",
     "TRUE",
     "FALSE",
-    "NULL"
 ]
+keywords.sort(reverse=True)
 
 
 @dataclass