extended sql parser

Author: adranwit

expr/collate.go

@@ -0,0 +1,9 @@
+package expr
+
+import "github.com/viant/sqlparser/node"
+
+// Collate represents collate expression.
+type Collate struct {
+	X         node.Node
+	Collation string
+}

expr/ident.go

@@ -2,18 +2,20 @@ package expr
 
 import "github.com/viant/sqlparser/node"
 
-//Ident represent an identifier
+// Ident represent an identifier
 type Ident struct {
 	Name string
 }
 
-//Identity returns ident or selector expr
+// Identity returns ident or selector expr
 func Identity(node node.Node) node.Node {
 	switch actual := node.(type) {
 	case *Ident:
 		return actual
 	case *Selector:
 		return actual
+	case *Collate:
+		return Identity(actual.X)
 	}
 	return nil
 }

expr/values.go

@@ -105,6 +105,8 @@ func NewValues(n node.Node) (*Values, error) {
 		}
 
 		return NewValues(actual.Y)
+	case *Collate:
+		return NewValues(actual.X)
 	case *Literal:
 		switch actual.Kind {
 		case "int":

lex.go

@@ -76,6 +76,7 @@ const (
 	typeKeyword
 	ttlKeyword
 	truncateTableKeyword
+	collateKeyword
 )
 
 var whitespaceMatcher = parsly.NewToken(whitespaceCode, "whitespace", matcher.NewWhiteSpace())
@@ -116,6 +117,7 @@ var keyMatcher = parsly.NewToken(keyTokenCode, "[RANGE|HASH|PRIMARY] KEY", match
 }, &option.Case{}))
 
 var onKeywordMatcher = parsly.NewToken(onKeyword, "ON", matcher.NewKeyword("on", &option.Case{}))
+var collateKeywordMatcher = parsly.NewToken(collateKeyword, "COLLATE", matcher.NewKeyword("collate", &option.Case{}))
 
 var whereKeywordMatcher = parsly.NewToken(whereKeyword, "WHERE", matcher.NewKeyword("where", &option.Case{}))
 var groupByMatcher = parsly.NewToken(groupByKeyword, "GROUP BY", matcher.NewSpacedFragment("group by", &option.Case{}))

operand.go

@@ -26,7 +26,10 @@ func discoverAlias(cursor *parsly.Cursor) string {
 func expectOperand(cursor *parsly.Cursor) (node.Node, error) {
 	literal, err := TryParseLiteral(cursor)
 	if literal != nil || err != nil {
-		return literal, err
+		if err != nil {
+			return literal, err
+		}
+		return applyCollate(cursor, literal)
 	}
 
 	match := cursor.MatchAfterOptional(whitespaceMatcher,
@@ -64,7 +67,7 @@ func expectOperand(cursor *parsly.Cursor) (node.Node, error) {
 			if err != nil {
 				return nil, err
 			}
-			return &expr.Call{X: selector, Raw: raw, Args: args}, nil
+			return applyCollate(cursor, &expr.Call{X: selector, Raw: raw, Args: args})
 
 		case exceptKeyword:
 			return parseStarExpr(cursor, selRaw, selector)
@@ -75,15 +78,15 @@ func expectOperand(cursor *parsly.Cursor) (node.Node, error) {
 			if match.Code == commentBlock {
 				comments = match.Text(cursor)
 			}
-			return expr.NewStar(selector, comments), nil
+			return applyCollate(cursor, expr.NewStar(selector, comments))
 		}
-		return selector, nil
+		return applyCollate(cursor, selector)
 	case exceptKeyword:
 		return nil, cursor.NewError(selectorMatcher)
 	case nullTokenCode:
-		return expr.NewNullLiteral(match.Text(cursor)), nil
+		return applyCollate(cursor, expr.NewNullLiteral(match.Text(cursor)))
 	case caseBlock:
-		return &expr.Switch{Raw: match.Text(cursor)}, nil
+		return applyCollate(cursor, &expr.Switch{Raw: match.Text(cursor)})
 	case starTokenCode:
 		selRaw := match.Text(cursor)
 		selector := expr.NewSelector(selRaw)
@@ -97,7 +100,7 @@ func expectOperand(cursor *parsly.Cursor) (node.Node, error) {
 		case exceptKeyword:
 			return parseStarExpr(cursor, selRaw, selector)
 		}
-		return expr.NewStar(selector, comments), err
+		return applyCollate(cursor, expr.NewStar(selector, comments))
 	case parenthesesCode:
 		raw := match.Text(cursor)
 		result := expr.NewParenthesis(raw)
@@ -139,13 +142,13 @@ func expectOperand(cursor *parsly.Cursor) (node.Node, error) {
 			}
 
 		}
-		return result, nil
+		return applyCollate(cursor, result)
 	case notOperator:
 		unary := expr.NewUnary(match.Text(cursor))
 		if unary.X, err = expectOperand(cursor); unary.X == nil || err != nil {
 			return nil, cursor.NewError(selectorMatcher)
 		}
-		return unary, nil
+		return applyCollate(cursor, unary)
 	case commentBlock:
 		return expectOperand(cursor)
 	case asKeyword, orderByKeyword, onKeyword, fromKeyword, whereKeyword, joinToken, groupByKeyword, havingKeyword, windowTokenCode, nextCode:
@@ -154,6 +157,23 @@ func expectOperand(cursor *parsly.Cursor) (node.Node, error) {
 	return nil, nil
 }
 
+func applyCollate(cursor *parsly.Cursor, n node.Node) (node.Node, error) {
+	if n == nil {
+		return nil, nil
+	}
+	pos := cursor.Pos
+	match := cursor.MatchAfterOptional(whitespaceMatcher, collateKeywordMatcher)
+	if match.Code != collateKeyword {
+		cursor.Pos = pos
+		return n, nil
+	}
+	match = cursor.MatchAfterOptional(whitespaceMatcher, identifierMatcher)
+	if match.Code != identifierCode {
+		return nil, cursor.NewError(identifierMatcher)
+	}
+	return &expr.Collate{X: n, Collation: match.Text(cursor)}, nil
+}
+
 func parseCallArguments(cursor *parsly.Cursor, raw string, pos int) ([]node.Node, error) {
 	var args []node.Node
 	if len(raw) > 0 {

query_test.go

@@ -253,6 +253,16 @@ func TestParseSelect(t *testing.T) {
 				SQL:         "SELECT t.* FROM x1 t join x2 z ON t.ID = z.ID",
 				expect:      "SELECT t.* FROM x1 t join x2 z ON t.ID = z.ID",
 			},
+			{
+				description: "JOIN select with collate",
+				SQL:         "SELECT * FROM a JOIN b ON a.brand COLLATE utf8mb4_bin = b.brand COLLATE utf8mb4_bin",
+				expect:      "SELECT * FROM a JOIN b ON a.brand COLLATE utf8mb4_bin = b.brand COLLATE utf8mb4_bin",
+			},
+			{
+				description: "JOIN select with collate parenthesis",
+				SQL:         "SELECT * FROM a JOIN b ON (a.brand COLLATE utf8mb4_bin) = (b.brand COLLATE utf8mb4_bin)",
+				expect:      "SELECT * FROM a JOIN b ON (a.brand COLLATE utf8mb4_bin) = (b.brand COLLATE utf8mb4_bin)",
+			},
 			{
 				description: "JOIN select",
 				SQL:         "SELECT t.* FROM x1 t join x2 z ON t.ID = z.ID LEFT JOIN x3 y ON t.ID = x3.ID",

stringify.go

@@ -157,6 +157,10 @@ func stringify(n node.Node, builder *bytes.Buffer) {
 		builder.WriteString(actual.Raw)
 		builder.WriteString(" ")
 		builder.WriteString(actual.Unparsed)
+	case *expr.Collate:
+		stringify(actual.X, builder)
+		builder.WriteString(" COLLATE ")
+		builder.WriteString(actual.Collation)
 	case *query.From:
 		if actual.X == nil {
 			return