Enable 791 format roundtrip tests by adding missing Format support

ast/ast.go

@@ -218,17 +218,19 @@ func (s *SettingExpr) End() token.Position { return s.Position }
 
 // InsertQuery represents an INSERT statement.
 type InsertQuery struct {
-	Position    token.Position `json:"-"`
-	Database    string         `json:"database,omitempty"`
-	Table       string         `json:"table,omitempty"`
-	Function    *FunctionCall  `json:"function,omitempty"` // For INSERT INTO FUNCTION syntax
-	Columns     []*Identifier  `json:"columns,omitempty"`
-	PartitionBy Expression     `json:"partition_by,omitempty"` // For PARTITION BY clause
-	Infile      string         `json:"infile,omitempty"`       // For FROM INFILE clause
-	Compression string         `json:"compression,omitempty"`  // For COMPRESSION clause
-	Select      Statement      `json:"select,omitempty"`
-	Format      *Identifier    `json:"format,omitempty"`
-	HasSettings bool           `json:"has_settings,omitempty"` // For SETTINGS clause
+	Position    token.Position   `json:"-"`
+	Database    string           `json:"database,omitempty"`
+	Table       string           `json:"table,omitempty"`
+	Function    *FunctionCall    `json:"function,omitempty"` // For INSERT INTO FUNCTION syntax
+	Columns     []*Identifier    `json:"columns,omitempty"`
+	PartitionBy Expression       `json:"partition_by,omitempty"` // For PARTITION BY clause
+	Infile      string           `json:"infile,omitempty"`       // For FROM INFILE clause
+	Compression string           `json:"compression,omitempty"`  // For COMPRESSION clause
+	Values      [][]Expression   `json:"-"`                      // For VALUES clause (format only, not in AST JSON)
+	Select      Statement        `json:"select,omitempty"`
+	Format      *Identifier      `json:"format,omitempty"`
+	HasSettings bool             `json:"has_settings,omitempty"` // For SETTINGS clause
+	Settings    []*SettingExpr   `json:"settings,omitempty"`     // For SETTINGS clause in INSERT
 }
 
 func (i *InsertQuery) Pos() token.Position { return i.Position }
@@ -375,7 +377,8 @@ type DropQuery struct {
 	Tables       []*TableIdentifier `json:"tables,omitempty"` // For DROP TABLE t1, t2, t3
 	View         string             `json:"view,omitempty"`
 	User         string             `json:"user,omitempty"`
-	Function     string             `json:"function,omitempty"` // For DROP FUNCTION
+	Function     string             `json:"function,omitempty"`   // For DROP FUNCTION
+	Dictionary   string             `json:"-"` // For DROP DICTIONARY (format only, not in AST JSON)
 	Temporary    bool               `json:"temporary,omitempty"`
 	OnCluster    string             `json:"on_cluster,omitempty"`
 	DropDatabase bool               `json:"drop_database,omitempty"`
@@ -487,6 +490,28 @@ func (u *UseQuery) Pos() token.Position { return u.Position }
 func (u *UseQuery) End() token.Position { return u.Position }
 func (u *UseQuery) statementNode()      {}
 
+// DetachQuery represents a DETACH statement.
+type DetachQuery struct {
+	Position token.Position `json:"-"`
+	Database string         `json:"database,omitempty"`
+	Table    string         `json:"table,omitempty"`
+}
+
+func (d *DetachQuery) Pos() token.Position { return d.Position }
+func (d *DetachQuery) End() token.Position { return d.Position }
+func (d *DetachQuery) statementNode()      {}
+
+// AttachQuery represents an ATTACH statement.
+type AttachQuery struct {
+	Position token.Position `json:"-"`
+	Database string         `json:"database,omitempty"`
+	Table    string         `json:"table,omitempty"`
+}
+
+func (a *AttachQuery) Pos() token.Position { return a.Position }
+func (a *AttachQuery) End() token.Position { return a.Position }
+func (a *AttachQuery) statementNode()      {}
+
 // DescribeQuery represents a DESCRIBE statement.
 type DescribeQuery struct {
 	Position      token.Position `json:"-"`

internal/explain/explain.go

@@ -133,6 +133,10 @@ func Node(sb *strings.Builder, node interface{}, depth int) {
 		explainDescribeQuery(sb, n, indent)
 	case *ast.ExistsQuery:
 		explainExistsTableQuery(sb, n, indent)
+	case *ast.DetachQuery:
+		explainDetachQuery(sb, n, indent)
+	case *ast.AttachQuery:
+		explainAttachQuery(sb, n, indent)
 
 	// Types
 	case *ast.DataType:

internal/explain/statements.go

@@ -320,6 +320,9 @@ func explainDropQuery(sb *strings.Builder, n *ast.DropQuery, indent string, dept
 	if n.View != "" {
 		name = n.View
 	}
+	if n.Dictionary != "" {
+		name = n.Dictionary
+	}
 	if n.DropDatabase {
 		name = n.Database
 	}
@@ -517,3 +520,11 @@ func explainParameter(sb *strings.Builder, n *ast.Parameter, indent string) {
 		fmt.Fprintf(sb, "%sQueryParameter\n", indent)
 	}
 }
+
+func explainDetachQuery(sb *strings.Builder, n *ast.DetachQuery, indent string) {
+	fmt.Fprintf(sb, "%sDetachQuery\n", indent)
+}
+
+func explainAttachQuery(sb *strings.Builder, n *ast.AttachQuery, indent string) {
+	fmt.Fprintf(sb, "%sAttachQuery\n", indent)
+}

internal/format/format.go

@@ -63,6 +63,10 @@ func Statement(sb *strings.Builder, stmt ast.Statement) {
 		formatExchangeQuery(sb, s)
 	case *ast.ExistsQuery:
 		formatExistsQueryStmt(sb, s)
+	case *ast.DetachQuery:
+		formatDetachQuery(sb, s)
+	case *ast.AttachQuery:
+		formatAttachQuery(sb, s)
 	default:
 		// Fallback for unhandled statements
 	}

internal/format/statements.go

@@ -13,11 +13,38 @@ func formatSelectWithUnionQuery(sb *strings.Builder, q *ast.SelectWithUnionQuery
 	}
 	for i, sel := range q.Selects {
 		if i > 0 {
-			sb.WriteString(" UNION ")
-			if len(q.UnionModes) > i-1 && q.UnionModes[i-1] == "ALL" {
-				sb.WriteString("ALL ")
-			} else if len(q.UnionModes) > i-1 && q.UnionModes[i-1] == "DISTINCT" {
-				sb.WriteString("DISTINCT ")
+			// Get the mode for this union - modes are stored as "UNION ALL", "UNION DISTINCT", etc.
+			modeIdx := i - 1
+			if modeIdx < len(q.UnionModes) {
+				mode := q.UnionModes[modeIdx]
+				// Parse the mode to extract the operator and modifier
+				// Format: "OPERATOR MODIFIER" (e.g., "UNION ALL", "EXCEPT DISTINCT")
+				sb.WriteString(" ")
+				if strings.HasPrefix(mode, "EXCEPT") {
+					sb.WriteString("EXCEPT ")
+					if strings.Contains(mode, "ALL") {
+						sb.WriteString("ALL ")
+					} else if strings.Contains(mode, "DISTINCT") {
+						sb.WriteString("DISTINCT ")
+					}
+				} else if strings.HasPrefix(mode, "INTERSECT") {
+					sb.WriteString("INTERSECT ")
+					if strings.Contains(mode, "ALL") {
+						sb.WriteString("ALL ")
+					} else if strings.Contains(mode, "DISTINCT") {
+						sb.WriteString("DISTINCT ")
+					}
+				} else {
+					// Default to UNION
+					sb.WriteString("UNION ")
+					if strings.Contains(mode, "ALL") {
+						sb.WriteString("ALL ")
+					} else if strings.Contains(mode, "DISTINCT") {
+						sb.WriteString("DISTINCT ")
+					}
+				}
+			} else {
+				sb.WriteString(" UNION ")
 			}
 		}
 		Statement(sb, sel)
@@ -376,6 +403,8 @@ func formatDropQuery(sb *strings.Builder, q *ast.DropQuery) {
 		sb.WriteString("VIEW ")
 	} else if q.Function != "" {
 		sb.WriteString("FUNCTION ")
+	} else if q.Dictionary != "" {
+		sb.WriteString("DICTIONARY ")
 	} else if q.User != "" {
 		sb.WriteString("USER ")
 	} else {
@@ -397,6 +426,12 @@ func formatDropQuery(sb *strings.Builder, q *ast.DropQuery) {
 		sb.WriteString(q.View)
 	} else if q.Function != "" {
 		sb.WriteString(q.Function)
+	} else if q.Dictionary != "" {
+		if q.Database != "" {
+			sb.WriteString(q.Database)
+			sb.WriteString(".")
+		}
+		sb.WriteString(q.Dictionary)
 	} else if q.User != "" {
 		sb.WriteString(q.User)
 	} else if len(q.Tables) > 0 {
@@ -670,6 +705,35 @@ func formatInsertQuery(sb *strings.Builder, q *ast.InsertQuery) {
 		}
 		sb.WriteString(")")
 	}
+	// Format SETTINGS before VALUES if present
+	if len(q.Settings) > 0 {
+		sb.WriteString(" SETTINGS ")
+		for i, s := range q.Settings {
+			if i > 0 {
+				sb.WriteString(", ")
+			}
+			sb.WriteString(s.Name)
+			sb.WriteString(" = ")
+			Expression(sb, s.Value)
+		}
+	}
+	// Format VALUES clause
+	if len(q.Values) > 0 {
+		sb.WriteString(" VALUES ")
+		for i, row := range q.Values {
+			if i > 0 {
+				sb.WriteString(", ")
+			}
+			sb.WriteString("(")
+			for j, expr := range row {
+				if j > 0 {
+					sb.WriteString(", ")
+				}
+				Expression(sb, expr)
+			}
+			sb.WriteString(")")
+		}
+	}
 	if q.Select != nil {
 		sb.WriteString(" ")
 		Statement(sb, q.Select)
@@ -994,3 +1058,29 @@ func formatExistsQueryStmt(sb *strings.Builder, q *ast.ExistsQuery) {
 	}
 	sb.WriteString(q.Table)
 }
+
+// formatDetachQuery formats a DETACH statement.
+func formatDetachQuery(sb *strings.Builder, q *ast.DetachQuery) {
+	if q == nil {
+		return
+	}
+	sb.WriteString("DETACH TABLE ")
+	if q.Database != "" {
+		sb.WriteString(q.Database)
+		sb.WriteString(".")
+	}
+	sb.WriteString(q.Table)
+}
+
+// formatAttachQuery formats an ATTACH statement.
+func formatAttachQuery(sb *strings.Builder, q *ast.AttachQuery) {
+	if q == nil {
+		return
+	}
+	sb.WriteString("ATTACH TABLE ")
+	if q.Database != "" {
+		sb.WriteString(q.Database)
+		sb.WriteString(".")
+	}
+	sb.WriteString(q.Table)
+}

parser/parser.go

@@ -149,6 +149,10 @@ func (p *Parser) parseStatement() ast.Statement {
 	case token.EXISTS:
 		// EXISTS table_name syntax (check if table exists)
 		return p.parseExistsStatement()
+	case token.DETACH:
+		return p.parseDetach()
+	case token.ATTACH:
+		return p.parseAttach()
 	default:
 		p.errors = append(p.errors, fmt.Errorf("unexpected token %s at line %d, column %d",
 			p.current.Token, p.current.Pos.Line, p.current.Pos.Column))
@@ -1160,9 +1164,34 @@ func (p *Parser) parseInsert() *ast.InsertQuery {
 	// Parse VALUES or SELECT
 	if p.currentIs(token.VALUES) {
 		p.nextToken()
-		// Skip VALUES data - consume until end of statement
-		for !p.currentIs(token.EOF) && !p.currentIs(token.SEMICOLON) && !p.currentIs(token.FORMAT) && !p.currentIs(token.SETTINGS) {
-			p.nextToken()
+		// Parse VALUES rows: (expr, expr, ...), (expr, expr, ...), ...
+		for {
+			if !p.currentIs(token.LPAREN) {
+				break
+			}
+			p.nextToken() // skip (
+			var row []ast.Expression
+			for !p.currentIs(token.RPAREN) && !p.currentIs(token.EOF) {
+				expr := p.parseExpression(LOWEST)
+				if expr != nil {
+					row = append(row, expr)
+				}
+				if p.currentIs(token.COMMA) {
+					p.nextToken()
+				} else {
+					break
+				}
+			}
+			if p.currentIs(token.RPAREN) {
+				p.nextToken() // skip )
+			}
+			ins.Values = append(ins.Values, row)
+			// Check for more rows
+			if p.currentIs(token.COMMA) {
+				p.nextToken()
+			} else {
+				break
+			}
 		}
 	} else if p.currentIs(token.SELECT) || p.currentIs(token.WITH) {
 		ins.Select = p.parseSelectWithUnion()
@@ -2047,6 +2076,7 @@ func (p *Parser) parseDrop() *ast.DropQuery {
 	// What are we dropping?
 	dropUser := false
 	dropFunction := false
+	dropDictionary := false
 	switch p.current.Token {
 	case token.TABLE:
 		p.nextToken()
@@ -2071,10 +2101,13 @@ func (p *Parser) parseDrop() *ast.DropQuery {
 			p.nextToken()
 		}
 	default:
-		// Handle multi-word DROP types: ROW POLICY, NAMED COLLECTION
+		// Handle multi-word DROP types: ROW POLICY, NAMED COLLECTION, DICTIONARY
 		if p.currentIs(token.IDENT) {
 			upper := strings.ToUpper(p.current.Value)
 			switch upper {
+			case "DICTIONARY":
+				dropDictionary = true
+				p.nextToken()
 			case "ROW", "NAMED", "POLICY", "QUOTA", "ROLE":
 				// Skip the DROP type tokens
 				for p.currentIs(token.IDENT) || p.current.Token.IsKeyword() {
@@ -2125,6 +2158,18 @@ func (p *Parser) parseDrop() *ast.DropQuery {
 			}
 		} else if dropFunction {
 			drop.Function = tableName
+		} else if dropDictionary {
+			drop.Dictionary = tableName
+			// Also set Table/Tables for backward compatibility with AST JSON
+			drop.Tables = append(drop.Tables, &ast.TableIdentifier{
+				Position: drop.Position,
+				Database: database,
+				Table:    tableName,
+			})
+			drop.Table = tableName
+			if database != "" {
+				drop.Database = database
+			}
 		} else if drop.DropDatabase {
 			drop.Database = tableName
 		} else {
@@ -3112,6 +3157,56 @@ func (p *Parser) parseExchange() *ast.ExchangeQuery {
 	return exchange
 }
 
+func (p *Parser) parseDetach() *ast.DetachQuery {
+	detach := &ast.DetachQuery{
+		Position: p.current.Pos,
+	}
+
+	p.nextToken() // skip DETACH
+
+	// Skip optional TABLE keyword
+	if p.currentIs(token.TABLE) {
+		p.nextToken()
+	}
+
+	// Parse table name (can be qualified: database.table)
+	tableName := p.parseIdentifierName()
+	if p.currentIs(token.DOT) {
+		p.nextToken()
+		detach.Database = tableName
+		detach.Table = p.parseIdentifierName()
+	} else {
+		detach.Table = tableName
+	}
+
+	return detach
+}
+
+func (p *Parser) parseAttach() *ast.AttachQuery {
+	attach := &ast.AttachQuery{
+		Position: p.current.Pos,
+	}
+
+	p.nextToken() // skip ATTACH
+
+	// Skip optional TABLE keyword
+	if p.currentIs(token.TABLE) {
+		p.nextToken()
+	}
+
+	// Parse table name (can be qualified: database.table)
+	tableName := p.parseIdentifierName()
+	if p.currentIs(token.DOT) {
+		p.nextToken()
+		attach.Database = tableName
+		attach.Table = p.parseIdentifierName()
+	} else {
+		attach.Table = tableName
+	}
+
+	return attach
+}
+
 func (p *Parser) parseArrayJoin() *ast.ArrayJoinClause {
 	aj := &ast.ArrayJoinClause{
 		Position: p.current.Pos,