extended sql parser

Author: adranwit

lex.go

@@ -28,6 +28,7 @@ const (
 	boolLiteral
 	nullKeyword
 	singleQuotedStringLiteral
+	rawSingleQuotedStringLiteral
 	doubleQuotedStringLiteral
 	caseBlock
 	betweenToken
@@ -145,6 +146,7 @@ var rangeOperatorMatcher = parsly.NewToken(rangeOperator, ".. AND .. ", matcher.
 var nullKeywordMatcher = parsly.NewToken(nullKeyword, "NULL", matcher.NewKeyword("null", &option.Case{}))
 var boolLiteralMatcher = parsly.NewToken(boolLiteral, "true|false", matcher.NewSet([]string{"true", "false"}, &option.Case{}))
 var singleQuotedStringLiteralMatcher = parsly.NewToken(singleQuotedStringLiteral, `'...'`, matcher.NewByteQuote('\'', '\\'))
+var rawSingleQuotedStringLiteralMatcher = parsly.NewToken(rawSingleQuotedStringLiteral, `r'...'`, smatcher.NewPrefixedQuote([]byte{'r', 'R'}, '\'', '\\'))
 var doubleQuotedStringLiteralMatcher = parsly.NewToken(doubleQuotedStringLiteral, `"..."`, matcher.NewByteQuote('\'', '\\'))
 var intLiteralMatcher = parsly.NewToken(intLiteral, `INT`, smatcher.NewIntMatcher())
 var numericLiteralMatcher = parsly.NewToken(numericLiteral, `NUMERIC`, matcher.NewNumber())

literal.go

@@ -5,12 +5,12 @@ import (
 	"github.com/viant/sqlparser/expr"
 )
 
-//ParseLiteral parses literal
+// ParseLiteral parses literal
 func ParseLiteral(cursor *parsly.Cursor) (*expr.Literal, error) {
 	return parseLiteral(cursor, true)
 }
 
-//TryParseLiteral tries to parse literal
+// TryParseLiteral tries to parse literal
 func TryParseLiteral(cursor *parsly.Cursor) (*expr.Literal, error) {
 	return parseLiteral(cursor, false)
 }
@@ -20,6 +20,7 @@ var literalTokens = []*parsly.Token{
 	nextMatcher,
 	nullKeywordMatcher,
 	boolLiteralMatcher,
+	rawSingleQuotedStringLiteralMatcher,
 	doubleQuotedStringLiteralMatcher,
 	singleQuotedStringLiteralMatcher,
 	intLiteralMatcher,
@@ -34,7 +35,7 @@ func parseLiteral(cursor *parsly.Cursor, shallRaiseInvalidToken bool) (*expr.Lit
 		return nil, nil
 	case nullKeyword:
 		return expr.NewNullLiteral(match.Text(cursor)), nil
-	case singleQuotedStringLiteral, doubleQuotedStringLiteral:
+	case singleQuotedStringLiteral, rawSingleQuotedStringLiteral, doubleQuotedStringLiteral:
 		return expr.NewStringLiteral(match.Text(cursor)), nil
 	case boolLiteral:
 		return expr.NewBoolLiteral(match.Text(cursor)), nil

matcher/prefixed_quote.go

@@ -0,0 +1,58 @@
+package matcher
+
+import "github.com/viant/parsly"
+
+type prefixedQuote struct {
+	prefixes []byte
+	quote    byte
+	escape   byte
+}
+
+// Match matches quoted strings prefixed by a single-byte marker (for example r'...').
+func (m *prefixedQuote) Match(cursor *parsly.Cursor) (matched int) {
+	input := cursor.Input
+	pos := cursor.Pos
+	inputSize := len(input)
+	if pos+1 >= inputSize {
+		return 0
+	}
+	if !hasBytePrefix(m.prefixes, input[pos]) || input[pos+1] != m.quote {
+		return 0
+	}
+
+	matched = 2
+	for i := pos + matched; i < inputSize; i++ {
+		value := input[i]
+		matched++
+		if value == m.escape {
+			if i+1 < inputSize {
+				i++
+				matched++
+			}
+			continue
+		}
+		if value == m.quote {
+			return matched
+		}
+	}
+
+	return 0
+}
+
+func hasBytePrefix(prefixes []byte, value byte) bool {
+	for _, candidate := range prefixes {
+		if value == candidate {
+			return true
+		}
+	}
+	return false
+}
+
+// NewPrefixedQuote returns a matcher for prefixed quote literals.
+func NewPrefixedQuote(prefixes []byte, quote, escape byte) parsly.Matcher {
+	return &prefixedQuote{
+		prefixes: prefixes,
+		quote:    quote,
+		escape:   escape,
+	}
+}

operand.go

@@ -125,7 +125,7 @@ func expectOperand(cursor *parsly.Cursor) (node.Node, error) {
 					list = append(list, &expr.Placeholder{Name: matched.Text(exprCursor)})
 				case nullKeyword:
 					list = append(list, expr.NewNullLiteral(matched.Text(exprCursor)))
-				case singleQuotedStringLiteral, doubleQuotedStringLiteral:
+				case singleQuotedStringLiteral, rawSingleQuotedStringLiteral, doubleQuotedStringLiteral:
 					list = append(list, expr.NewStringLiteral(matched.Text(exprCursor)))
 				case boolLiteral:
 					list = append(list, expr.NewBoolLiteral(matched.Text(exprCursor)))

query_test.go

@@ -420,3 +420,15 @@ func TestParseSelect(t *testing.T) {
 		}
 	}
 }
+
+func TestParseSelect_BigQueryRawRegexLiteral(t *testing.T) {
+	sql := `SELECT LOWER(REGEXP_REPLACE(COALESCE(NULLIF(TRIM(s.NAME), ''), NULLIF(TRIM(s.MOBILE_URL), ''), ''),r'^(?:https?://)?(?:www\.)?', '')) AS site_domain FROM CI_SITE s`
+
+	parsed, err := ParseQuery(sql)
+	if !assert.NoError(t, err) {
+		return
+	}
+
+	actual := strings.TrimSpace(Stringify(parsed))
+	assert.Equal(t, sql, actual)
+}