From 79615cf8b074edad2b053df7cebf16848dfb3928 Mon Sep 17 00:00:00 2001 From: mysticmind Date: Fri, 9 Oct 2026 18:23:22 +0530 Subject: [PATCH 1/3] Prototype node-level deparse, walker control and parameter references - Parser.DeparseNode and node.Deparse() turn a single node back into SQL: statements, expressions, FROM items, select list and ORDER BY items, WITH clauses and type names - Walk(visitor) lets the visitor skip a node's children or stop the walk; NodeVisit now reports the parent's property and list index the node came from - Parser.ParameterRefs finds $n parameter references with their offsets in the query string --- README.md | 54 +++++++ src/PackageSmokeTest/Program.cs | 5 + src/PgSqlParser.Tests/NodeToolsTests.cs | 199 ++++++++++++++++++++++++ src/PgSqlParser/ParseTreeExtensions.cs | 84 ++++++++-- src/PgSqlParser/Parser.cs | 149 ++++++++++++++++++ 5 files changed, 476 insertions(+), 15 deletions(-) create mode 100644 src/PgSqlParser.Tests/NodeToolsTests.cs diff --git a/README.md b/README.md index 85361f9..de3f3cd 100644 --- a/README.md +++ b/README.md @@ -171,6 +171,43 @@ foreach (var stmt in tree.Stmts) // drop: DROP TABLE tmp ``` +Any node can be turned back into SQL on its own, which is how to get the text of a single clause or expression: + +```csharp +var select = tree.Stmts[0].Stmt.SelectStmt; + +Console.WriteLine(select.FromClause[0].Deparse().GetValueOrThrow()); +// Output: customers c JOIN orders o ON o.customer_id = c.id + +foreach (var table in tree.Descendants()) +{ + Console.WriteLine(table.Deparse().GetValueOrThrow()); +} +// Output: customers c, orders o +``` + +`Deparse()` on a node works for statements, expressions, items of a FROM clause, select list and ORDER BY items, WITH clauses and type names. Nodes that are not SQL on their own, such as an `Alias`, return an error. + +To control the walk, pass a function. It can skip everything below a node or stop the walk, and each visit tells you which property of its parent the node came from: + +```csharp +// Tables used by the query itself, ignoring subqueries inside expressions +var tables = new List(); +tree.Walk(visit => +{ + if (visit.Node is SubLink) + return WalkAction.SkipChildren; + + if (visit.Node is RangeVar table) + tables.Add(table.Relname); + + return WalkAction.Continue; // or WalkAction.Stop to end the walk +}); + +// visit.FieldName is the parent's property, for example nameof(SelectStmt.WhereClause); +// visit.Index is the position when that property is a list. +``` + `Walk` and `Descendants` visit parents before their children and siblings in field order, which is not always the order of the query text. `GetLocation()` returns a node's location if it has one; see [Offsets and non-ASCII text](#offsets-and-non-ascii-text) for its unit. Only statements record a length, so `GetText` is available for statements and not for other nodes. ### Scan @@ -195,6 +232,23 @@ if (result.Error is null) // { "version": 180006, "tokens": [ { "end": 6, "token": "SELECT", "keywordKind": "RESERVED_KEYWORD" }, { "start": 7, "end": 8, "token": "ICONST" } ] } ``` +### ParameterRefs + +Find the parameter references (`$1`, `$2`, ...) in a query, with where each one is in the query string. References inside string literals and comments are not parameters and are left out. + +```csharp +using PgSqlParser; + +var query = "SELECT * FROM t WHERE a = $1 AND b = $2 AND note <> '$3'"; +var parameters = Parser.ParameterRefs(query).GetValueOrThrow(); + +foreach (var parameter in parameters) +{ + Console.WriteLine($"${parameter.Number} at {parameter.Start}..{parameter.End}"); +} +// Output: $1 at 26..28, $2 at 37..39 +``` + ### ParsePlpgsql Parse PL/pgSQL function bodies and return a JSON representation diff --git a/src/PackageSmokeTest/Program.cs b/src/PackageSmokeTest/Program.cs index 32475fa..80029b2 100644 --- a/src/PackageSmokeTest/Program.cs +++ b/src/PackageSmokeTest/Program.cs @@ -45,6 +45,11 @@ void Check(string name, bool ok) Check("Descendants", walked.Descendants().Select(table => table.Relname).SequenceEqual(["données"])); Check("Unwrap", walked.Stmts[0].Stmt.Unwrap() is SelectStmt); Check("GetText", walked.Stmts[1].GetText(query) == "SELECT 2"); +Check("DeparseNode", walked.Stmts[0].Stmt.SelectStmt.WhereClause.Deparse().Value == "id = 1"); +var skipped = 0; +walked.Walk(visit => { skipped++; return visit.Node is SelectStmt ? WalkAction.SkipChildren : WalkAction.Continue; }); +Check("Walk with a visitor", skipped == 4 && walked.Walk().First().FieldName == "Stmts"); +Check("ParameterRefs", Parser.ParameterRefs("SELECT $1, '$2', $3").Value?.Select(p => p.Number).SequenceEqual([1, 3]) == true); Check("ParsePlpgsqlFunctions", Parser.ParsePlpgsqlFunctions( "CREATE FUNCTION f() RETURNS int AS $$ BEGIN RETURN 1; END; $$ LANGUAGE plpgsql").Value?[0].Queries().SequenceEqual(["1"]) == true); Check("Error", Parser.Parse("SELECT FROM WHERE").Error?.Message?.StartsWith("syntax error") == true); diff --git a/src/PgSqlParser.Tests/NodeToolsTests.cs b/src/PgSqlParser.Tests/NodeToolsTests.cs new file mode 100644 index 0000000..d526d4c --- /dev/null +++ b/src/PgSqlParser.Tests/NodeToolsTests.cs @@ -0,0 +1,199 @@ +using Google.Protobuf; +using Shouldly; +using Xunit; + +namespace PgSqlParser.Tests; + +public class NodeToolsTests +{ + private const string Query = + "WITH r AS (SELECT 1) SELECT c.name AS n, count(*) FROM customers c JOIN orders o ON o.cid = c.id, " + + "(SELECT 2) s WHERE c.a = $1 AND c.b IN (SELECT id FROM vip) AND c.t::date > now() " + + "ORDER BY n DESC NULLS LAST LIMIT 5"; + + private static ParseResult Parse(string query) => Parser.Parse(query).GetValueOrThrow(); + + private static SelectStmt Select(string query) => Parse(query).Stmts[0].Stmt.SelectStmt; + + [Fact] + public void DeparseNodeForClausesOfAQuery() + { + var select = Select(Query); + + select.WhereClause.Deparse().GetValueOrThrow() + .ShouldBe("c.a = $1 AND c.b IN (SELECT id FROM vip) AND c.t::date > now()"); + select.TargetList.Select(target => target.Deparse().GetValueOrThrow()).ShouldBe(["c.name AS n", "count(*)"]); + select.FromClause.Select(item => item.Deparse().GetValueOrThrow()) + .ShouldBe(["customers c JOIN orders o ON o.cid = c.id", "(SELECT 2) s"]); + select.SortClause[0].Deparse().GetValueOrThrow().ShouldBe("n DESC NULLS LAST"); + select.LimitCount.Deparse().GetValueOrThrow().ShouldBe("5"); + select.WithClause.Deparse().GetValueOrThrow().ShouldBe("WITH r AS (SELECT 1)"); + select.WithClause.Ctes[0].Deparse().GetValueOrThrow().ShouldBe("r AS (SELECT 1)"); + } + + [Fact] + public void DeparseNodeForNodesFoundByTheWalker() + { + var tree = Parse(Query); + + tree.Descendants().Select(table => table.Deparse().GetValueOrThrow()) + .ShouldBe(["customers c", "orders o", "vip"]); + tree.Descendants().Single().Deparse().GetValueOrThrow().ShouldBe("c.b IN (SELECT id FROM vip)"); + tree.Descendants().Single().Deparse().GetValueOrThrow().ShouldBe("c.t::date"); + tree.Descendants().Single().Deparse().GetValueOrThrow().ShouldBe("date"); + tree.Descendants().Single().Deparse().GetValueOrThrow().ShouldBe("$1"); + tree.Descendants().Select(select => select.Deparse().GetValueOrThrow()).Skip(1) + .ShouldBe(["SELECT 2", "SELECT id FROM vip", "SELECT 1"]); + } + + [Fact] + public void DeparseNodeForStatementsAndWrappers() + { + const string query = "SELECT 1; INSERT INTO t (a) VALUES (1); DROP TABLE t"; + var tree = Parse(query); + + // A whole tree, a RawStmt, a Node wrapper and the statement inside it all work. + Parser.DeparseNode(tree).GetValueOrThrow().ShouldBe(query); + tree.Stmts[1].Deparse().GetValueOrThrow().ShouldBe("INSERT INTO t (a) VALUES (1)"); + tree.Stmts[1].Stmt.Deparse().GetValueOrThrow().ShouldBe("INSERT INTO t (a) VALUES (1)"); + tree.Stmts[1].Stmt.InsertStmt.Deparse().GetValueOrThrow().ShouldBe("INSERT INTO t (a) VALUES (1)"); + tree.Stmts[2].Stmt.DropStmt.Deparse().GetValueOrThrow().ShouldBe("DROP TABLE t"); + } + + [Fact] + public void DeparseNodeKeepsNonAsciiTextAndDoesNotModifyTheTree() + { + var tree = Parse("SELECT 1 FROM \"données\" WHERE \"é\" = '😀' AND b = 2"); + var before = tree.Clone(); + + tree.Stmts[0].Stmt.SelectStmt.WhereClause.Deparse().GetValueOrThrow().ShouldBe("\"é\" = '😀' AND b = 2"); + tree.Descendants().Single().Deparse().GetValueOrThrow().ShouldBe("\"données\""); + + tree.ShouldBe(before); + } + + [Fact] + public void DeparseNodeReportsUnsupportedNodes() + { + var tree = Parse("SELECT c.name FROM customers c"); + + var alias = tree.Descendants().Single().Deparse(); + alias.IsSuccess.ShouldBeFalse(); + alias.Error.Message.ShouldBe("a Alias cannot be deparsed on its own"); + tree.Descendants().First().Deparse().IsSuccess.ShouldBeFalse(); + new Node().Deparse().Error.ShouldNotBeNull().Message.ShouldBe("an empty Node cannot be deparsed"); + Should.Throw(() => Parser.DeparseNode(null!)); + } + + [Fact] + public void WalkReportsTheFieldAndIndexOfEachNode() + { + var tree = Parse("SELECT a, b FROM t WHERE x = 1"); + + var visits = tree.Walk().ToList(); + + visits[0].FieldName.ShouldBe(nameof(ParseResult.Stmts)); + visits[0].Index.ShouldBe(0); + // A Node wrapper is transparent, so the statement reports the field that held the wrapper. + visits[1].FieldName.ShouldBe(nameof(RawStmt.Stmt)); + visits[1].Index.ShouldBeNull(); + + var targets = visits.Where(visit => visit.Node is ResTarget).ToList(); + targets.Select(visit => visit.FieldName).Distinct().ShouldBe([nameof(SelectStmt.TargetList)]); + targets.Select(visit => visit.Index).ShouldBe([0, 1]); + + visits.Single(visit => visit.FieldName == nameof(SelectStmt.WhereClause)).Node.ShouldBeOfType(); + visits.Single(visit => visit.FieldName == nameof(SelectStmt.FromClause)).Node.ShouldBeOfType(); + } + + [Fact] + public void WalkCanSkipChildren() + { + var tree = Parse("SELECT a FROM t WHERE x IN (SELECT y FROM inner_table) AND z = 1"); + var tables = new List(); + + // Collect tables, but do not look inside subqueries in expressions. + tree.Walk(visit => + { + if (visit.Node is SubLink) + return WalkAction.SkipChildren; + + if (visit.Node is RangeVar table) + tables.Add(table.Relname); + + return WalkAction.Continue; + }); + + tables.ShouldBe(["t"]); + tree.Descendants().Select(table => table.Relname).ShouldBe(["t", "inner_table"]); + } + + [Fact] + public void WalkCanStop() + { + var tree = Parse("SELECT a FROM t1, t2, t3"); + var visited = 0; + RangeVar? first = null; + + tree.Walk(visit => + { + visited++; + if (visit.Node is not RangeVar table) + return WalkAction.Continue; + + first = table; + return WalkAction.Stop; + }); + + first.ShouldNotBeNull().Relname.ShouldBe("t1"); + visited.ShouldBeLessThan(tree.Walk().Count()); + } + + [Fact] + public void WalkWithAVisitorMatchesTheEnumerableOrder() + { + var tree = Parse(Query); + var seen = new List(); + + tree.Walk(visit => + { + seen.Add(visit); + return WalkAction.Continue; + }); + + seen.ShouldBe(tree.Walk().ToList()); + Should.Throw(() => tree.Walk(null!)); + Should.Throw(() => ((IMessage)null!).Walk(_ => WalkAction.Continue)); + } + + [Fact] + public void ParameterRefsInOrderWithOffsets() + { + const string query = "SELECT * FROM t WHERE a = $1 AND b = $2 OR c = $1 AND d > $10"; + + var parameters = Parser.ParameterRefs(query).GetValueOrThrow(); + + parameters.Select(parameter => parameter.Number).ShouldBe([1, 2, 1, 10]); + parameters.Select(parameter => query[parameter.Start..parameter.End]).ShouldBe(["$1", "$2", "$1", "$10"]); + } + + [Fact] + public void ParameterRefsIgnoreLiteralsAndCommentsAndUseStringOffsets() + { + const string query = "SELECT '😀 $1', $$ $2 $$ /* $3 */ FROM \"données$4\" WHERE é = $5 -- $6"; + + var parameter = Parser.ParameterRefs(query).GetValueOrThrow().ShouldHaveSingleItem(); + + parameter.Number.ShouldBe(5); + query[parameter.Start..parameter.End].ShouldBe("$5"); + } + + [Fact] + public async Task ParameterRefsEdgeCases() + { + Parser.ParameterRefs("SELECT 1").GetValueOrThrow().ShouldBeEmpty(); + Parser.ParameterRefs("SELECT 'unterminated").IsSuccess.ShouldBeFalse(); + Should.Throw(() => Parser.ParameterRefs(null!)); + (await Parser.ParameterRefsAsync("SELECT $1")).GetValueOrThrow().ShouldBe([new ParameterRef(1, 7, 9)]); + } +} diff --git a/src/PgSqlParser/ParseTreeExtensions.cs b/src/PgSqlParser/ParseTreeExtensions.cs index b40edec..bd9b72d 100644 --- a/src/PgSqlParser/ParseTreeExtensions.cs +++ b/src/PgSqlParser/ParseTreeExtensions.cs @@ -10,7 +10,32 @@ namespace PgSqlParser; /// The node, for example a or a . /// The node that contains it, or null for a node directly below the root. /// How far below the root the node is; 1 for the root's own children. -public readonly record struct NodeVisit(IMessage Node, IMessage? Parent, int Depth); +public readonly record struct NodeVisit(IMessage Node, IMessage? Parent, int Depth) +{ + /// + /// The property of the containing node that holds this node, for example WhereClause or + /// TargetList. Compare it with nameof, as in nameof(SelectStmt.WhereClause). + /// + public string? FieldName { get; init; } + + /// The node's position if that property is a list, otherwise null. + public int? Index { get; init; } +} + +/// +/// What does after visiting a node. +/// +public enum WalkAction +{ + /// Go on to the node's children. + Continue, + + /// Do not visit anything below this node, and go on to the next one. + SkipChildren, + + /// End the walk. + Stop +} /// /// Helpers for navigating a parse tree without spelling out the path to each node. @@ -45,7 +70,32 @@ public static IEnumerable Walk(this IMessage root) } /// - /// Returns every node below , in the order of . + /// Walks the nodes below in the same order, calling + /// for each one. Its return value decides whether the walk goes into the node's children, skips + /// them, or stops, which the enumerable form cannot do. + /// + public static void Walk(this IMessage root, Func visitor) + { + ArgumentNullException.ThrowIfNull(root); + ArgumentNullException.ThrowIfNull(visitor); + + var pending = new Stack(); + PushChildren(pending, root is Node rootWrapper ? rootWrapper.Unwrap() : root, null, 1); + + while (pending.Count > 0) + { + var visit = pending.Pop(); + var action = visitor(visit); + if (action == WalkAction.Stop) + return; + + if (action == WalkAction.Continue) + PushChildren(pending, visit.Node, visit.Node, visit.Depth + 1); + } + } + + /// + /// Returns every node below , in the order of . /// public static IEnumerable Descendants(this IMessage root) { @@ -54,7 +104,7 @@ public static IEnumerable Descendants(this IMessage root) /// /// Returns every node of type below , in the order - /// of . For example parseResult.Descendants<RangeVar>() finds every + /// of . For example parseResult.Descendants<RangeVar>() finds every /// table reference. /// public static IEnumerable Descendants(this IMessage root) where T : class, IMessage @@ -107,6 +157,14 @@ public static string GetText(this RawStmt rawStmt, string query, Utf8OffsetMappe return query[start..end]; } + /// + /// Turns this node back into SQL. See for which nodes are supported. + /// + public static Result Deparse(this IMessage node) + { + return Parser.DeparseNode(node); + } + // An explicit stack instead of recursion, so a deeply nested tree cannot overflow the call stack. private static IEnumerable WalkIterator(IMessage root) { @@ -139,25 +197,21 @@ private static void PushChildren(Stack pending, IMessage? message, IM { var items = (System.Collections.IList)value; for (var j = items.Count - 1; j >= 0; j--) - Push(pending, (IMessage)items[j]!, parent, depth); + Push(pending, (IMessage)items[j]!, parent, depth, field, j); } else if (value is IMessage child) { - Push(pending, child, parent, depth); + Push(pending, child, parent, depth, field, null); } } } - private static void Push(Stack pending, IMessage child, IMessage? parent, int depth) + private static void Push(Stack pending, IMessage child, IMessage? parent, int depth, + FieldDescriptor field, int? index) { - if (child is Node wrapper) - { - if (wrapper.Unwrap() is { } inner) - pending.Push(new NodeVisit(inner, parent, depth)); - } - else - { - pending.Push(new NodeVisit(child, parent, depth)); - } + // A Node wrapper is transparent: the field and index are those of the wrapper. + var node = child is Node wrapper ? wrapper.Unwrap() : child; + if (node is not null) + pending.Push(new NodeVisit(node, parent, depth) { FieldName = field.PropertyName, Index = index }); } } diff --git a/src/PgSqlParser/Parser.cs b/src/PgSqlParser/Parser.cs index 116eb87..d30527f 100644 --- a/src/PgSqlParser/Parser.cs +++ b/src/PgSqlParser/Parser.cs @@ -179,6 +179,14 @@ public enum ParserOptions /// The comment text, including its start and end tokens. public record DeparseComment(int MatchLocation, int NewlinesBeforeComment, int NewlinesAfterComment, string Text); +/// +/// A parameter reference such as $1, found by . +/// +/// The parameter number: 1 for $1. +/// Where the reference starts in the query string, as a UTF-16 offset. +/// Where it ends (exclusive), so query[Start..End] is the reference. +public record ParameterRef(int Number, int Start, int End); + /// /// Options for . /// @@ -828,6 +836,147 @@ public static Result Deparse(ParseResult parseResult, DeparseOptions? op } } + /// + /// Deparse a single node of a parse tree back into SQL, for example only a WHERE clause, one + /// expression or one table reference. Supported are statements, expressions, items of a FROM + /// clause (, , subselects, functions), select list + /// items (), ORDER BY items (), WITH clauses and their + /// common table expressions, and type names. Other nodes, such as a bare or + /// , return an error. The node is not modified. + /// + /// A node of a parse tree, a wrapper, a or a whole . + /// + public static Result DeparseNode(IMessage node) + { + ArgumentNullException.ThrowIfNull(node); + + if (node is Node wrapper) + { + return wrapper.Unwrap() is { } inner + ? DeparseNode(inner) + : Result.Failure(NodeError("an empty Node cannot be deparsed")); + } + + switch (node) + { + case ParseResult parseResult: + return Deparse(parseResult); + case RawStmt rawStmt: + return Deparse(new ParseResult { Version = PgVersionNum, Stmts = { rawStmt } }); + } + + if (!NodeFields.Value.TryGetValue(node.Descriptor.FullName, out var nodeField)) + return Result.Failure(NodeError($"a {node.Descriptor.Name} cannot be deparsed on its own")); + + var wrapped = new Node(); + nodeField.Accessor.SetValue(wrapped, node); + + if (node.Descriptor.Name.EndsWith("Stmt", StringComparison.Ordinal)) + return Deparse(new ParseResult { Version = PgVersionNum, Stmts = { new RawStmt { Stmt = wrapped } } }); + + // Anything else is deparsed inside a minimal SELECT, and the SELECT around it is cut off again. + var select = new SelectStmt { LimitOption = LimitOption.Default, Op = SetOperation.SetopNone }; + string prefix; + var suffix = ""; + switch (node) + { + case WithClause withClause: + select.WithClause = withClause; + prefix = ""; + suffix = " SELECT"; + break; + case CommonTableExpr: + select.WithClause = new WithClause { Ctes = { wrapped } }; + prefix = "WITH "; + suffix = " SELECT"; + break; + case TypeName typeName: + select.WhereClause = new Node + { + TypeCast = new TypeCast { Arg = new Node { AConst = new A_Const { Isnull = true } }, TypeName = typeName } + }; + prefix = "SELECT WHERE NULL::"; + break; + case RangeVar or JoinExpr or RangeSubselect or RangeFunction or RangeTableSample or RangeTableFunc: + select.FromClause.Add(wrapped); + prefix = "SELECT FROM "; + break; + case ResTarget: + select.TargetList.Add(wrapped); + prefix = "SELECT "; + break; + case SortBy: + select.SortClause.Add(wrapped); + prefix = "SELECT ORDER BY "; + break; + default: + select.WhereClause = wrapped; + prefix = "SELECT WHERE "; + break; + } + + var result = Deparse(new ParseResult + { + Version = PgVersionNum, + Stmts = { new RawStmt { Stmt = new Node { SelectStmt = select } } } + }); + // A node that fits none of these positions makes the deparser fail or give something else. + if (!result.TryGetValue(out var sql) + || !sql.StartsWith(prefix, StringComparison.Ordinal) + || !sql.EndsWith(suffix, StringComparison.Ordinal)) + { + return Result.Failure(NodeError($"a {node.Descriptor.Name} cannot be deparsed on its own")); + } + + return Result.Success(sql[prefix.Length..^suffix.Length]); + } + + // The Node field that holds each node type, keyed by the node type's full name. + private static readonly Lazy> NodeFields = new(() => + Node.Descriptor.Fields.InDeclarationOrder() + .Where(field => field.FieldType == Google.Protobuf.Reflection.FieldType.Message) + .ToDictionary(field => field.MessageType.FullName)); + + private static Error NodeError(string message) => new(message, null, null, 0, 0, null); + + /// + /// Find the parameter references ($1, $2, ...) in a query, in the order they appear. + /// A reference inside a string literal or a comment is not a parameter and is not returned. + /// + /// + /// + public static Result> ParameterRefs(string query) + { + var scan = Scan(query); + if (!scan.TryGetValue(out var scanResult, out var error)) + return Result>.Failure(error); + + var parameters = new List(); + foreach (var token in scanResult.Tokens) + { + // A PARAM token is a dollar sign followed by digits. + if (token.Token == Token.Param + && int.TryParse(query.AsSpan(token.Start + 1, token.End - token.Start - 1), out var number)) + { + parameters.Add(new ParameterRef(number, token.Start, token.End)); + } + } + + return Result>.Success(parameters); + } + + /// + /// Async find the parameter references in a query + /// + /// + /// + /// + public static Task>> ParameterRefsAsync(string query, + CancellationToken cancellationToken = default) + { + return RunAsync(() => ParameterRefs(query), cancellationToken); + } + /// /// Async deparse AST back into a query string /// From 3bcdcd1fb5d3e7338556a23e3365a649617213c8 Mon Sep 17 00:00:00 2001 From: mysticmind Date: Fri, 9 Oct 2026 18:35:05 +0530 Subject: [PATCH 2/3] Add ancestors, statement context and parameter cast types to the prototype Closes the gaps against Ruby pg_query, pglast and @pgsql/traverse: - NodeVisit.Ancestors and FindAncestor() give the path from a node up to the root of the walk - NodeVisit.Statement and StatementIndex give the top-level statement a node belongs to - ParameterRef.TypeName holds the type a parameter is cast to --- README.md | 23 +++++ src/PackageSmokeTest/Program.cs | 3 + src/PgSqlParser.Tests/NodeToolsTests.cs | 84 +++++++++++++++++ src/PgSqlParser/ParseTreeExtensions.cs | 114 ++++++++++++++++++++---- src/PgSqlParser/Parser.cs | 38 +++++++- 5 files changed, 244 insertions(+), 18 deletions(-) diff --git a/README.md b/README.md index de3f3cd..de0c435 100644 --- a/README.md +++ b/README.md @@ -208,6 +208,27 @@ tree.Walk(visit => // visit.Index is the position when that property is a list. ``` +Each visit also knows where it sits in the tree, which is what most rules need: + +```csharp +foreach (var visit in tree.Walk()) +{ + if (visit.Node is not RangeVar table) + continue; + + // The nearest containing node of a type, or null + var insideCte = visit.FindAncestor() is not null; + var insideSubquery = visit.FindAncestor() is not null; + + // The top-level statement the node belongs to + var writes = visit.Statement?.Stmt.Unwrap() is InsertStmt or UpdateStmt or DeleteStmt; + + Console.WriteLine($"statement {visit.StatementIndex}: {table.Relname} (cte: {insideCte}, subquery: {insideSubquery}, writes: {writes})"); +} +``` + +`visit.Ancestors` lists every containing node, nearest first. + `Walk` and `Descendants` visit parents before their children and siblings in field order, which is not always the order of the query text. `GetLocation()` returns a node's location if it has one; see [Offsets and non-ASCII text](#offsets-and-non-ascii-text) for its unit. Only statements record a length, so `GetText` is available for statements and not for other nodes. ### Scan @@ -249,6 +270,8 @@ foreach (var parameter in parameters) // Output: $1 at 26..28, $2 at 37..39 ``` +When a parameter is cast in the query, `TypeName` holds the type: `$1::int` gives `int`, and `CAST($2 AS numeric(10,2))` gives `numeric(10, 2)`. It is null for a parameter that is not cast. + ### ParsePlpgsql Parse PL/pgSQL function bodies and return a JSON representation diff --git a/src/PackageSmokeTest/Program.cs b/src/PackageSmokeTest/Program.cs index 80029b2..3a72c70 100644 --- a/src/PackageSmokeTest/Program.cs +++ b/src/PackageSmokeTest/Program.cs @@ -49,6 +49,9 @@ void Check(string name, bool ok) var skipped = 0; walked.Walk(visit => { skipped++; return visit.Node is SelectStmt ? WalkAction.SkipChildren : WalkAction.Continue; }); Check("Walk with a visitor", skipped == 4 && walked.Walk().First().FieldName == "Stmts"); +var tableVisit = walked.Walk().First(visit => visit.Node is RangeVar); +Check("Ancestors and statement", tableVisit.FindAncestor() is not null && tableVisit.StatementIndex == 0); +Check("ParameterRefs cast type", Parser.ParameterRefs("SELECT $1::int").Value?[0].TypeName == "int"); Check("ParameterRefs", Parser.ParameterRefs("SELECT $1, '$2', $3").Value?.Select(p => p.Number).SequenceEqual([1, 3]) == true); Check("ParsePlpgsqlFunctions", Parser.ParsePlpgsqlFunctions( "CREATE FUNCTION f() RETURNS int AS $$ BEGIN RETURN 1; END; $$ LANGUAGE plpgsql").Value?[0].Queries().SequenceEqual(["1"]) == true); diff --git a/src/PgSqlParser.Tests/NodeToolsTests.cs b/src/PgSqlParser.Tests/NodeToolsTests.cs index d526d4c..54395ad 100644 --- a/src/PgSqlParser.Tests/NodeToolsTests.cs +++ b/src/PgSqlParser.Tests/NodeToolsTests.cs @@ -166,6 +166,67 @@ public void WalkWithAVisitorMatchesTheEnumerableOrder() Should.Throw(() => ((IMessage)null!).Walk(_ => WalkAction.Continue)); } + [Fact] + public void VisitsKnowTheirAncestors() + { + var tree = Parse("WITH r AS (SELECT a FROM inner_table) SELECT b FROM outer_table WHERE c IN (SELECT d FROM sub_table)"); + + var tables = tree.Walk().Where(visit => visit.Node is RangeVar) + .ToDictionary(visit => ((RangeVar)visit.Node).Relname); + + // "Is this table inside a CTE?" and "inside a subquery in an expression?" + tables["inner_table"].FindAncestor().ShouldNotBeNull().Ctename.ShouldBe("r"); + tables["outer_table"].FindAncestor().ShouldBeNull(); + tables["sub_table"].FindAncestor().ShouldNotBeNull(); + tables["outer_table"].FindAncestor().ShouldBeNull(); + + var ancestors = tables["inner_table"].Ancestors.ToList(); + ancestors[0].ShouldBeSameAs(tables["inner_table"].Parent); + ancestors.Select(node => node.Descriptor.Name) + .ShouldBe(["SelectStmt", "CommonTableExpr", "WithClause", "SelectStmt", "RawStmt"]); + // The nearest SelectStmt is the CTE's own query, not the outer one. + tables["inner_table"].FindAncestor().ShouldBeSameAs(ancestors[0]); + + tree.Walk().First().Ancestors.ShouldBeEmpty(); + tree.Walk().First().FindAncestor().ShouldBeNull(); + } + + [Fact] + public void VisitsKnowTheirStatement() + { + var tree = Parse("SELECT a FROM t1; UPDATE t2 SET x = 1 WHERE y IN (SELECT z FROM t3); DROP TABLE t4"); + + var tablesByStatement = tree.Walk() + .Where(visit => visit.Node is RangeVar) + .Select(visit => (((RangeVar)visit.Node).Relname, visit.StatementIndex, Kind: visit.Statement!.Stmt.Unwrap()!.Descriptor.Name)); + + tablesByStatement.ShouldBe([("t1", 0, "SelectStmt"), ("t2", 1, "UpdateStmt"), ("t3", 1, "UpdateStmt")]); + + var statements = tree.Walk().Where(visit => visit.Node is RawStmt).ToList(); + statements.Select(visit => visit.StatementIndex).ShouldBe([0, 1, 2]); + statements[2].Statement.ShouldBeSameAs(tree.Stmts[2]); + + // A walk that does not start from the whole tree has no statement to report. + tree.Stmts[1].Walk().ShouldAllBe(visit => visit.Statement == null && visit.StatementIndex == null); + } + + [Fact] + public void VisitorFormAlsoReportsAncestorsAndStatement() + { + var tree = Parse("SELECT 1; SELECT a FROM t WHERE b IN (SELECT c FROM u)"); + var insideSubquery = new List(); + + tree.Walk(visit => + { + if (visit.Node is RangeVar table && visit.FindAncestor() is not null && visit.StatementIndex == 1) + insideSubquery.Add(table.Relname); + + return WalkAction.Continue; + }); + + insideSubquery.ShouldBe(["u"]); + } + [Fact] public void ParameterRefsInOrderWithOffsets() { @@ -188,6 +249,29 @@ public void ParameterRefsIgnoreLiteralsAndCommentsAndUseStringOffsets() query[parameter.Start..parameter.End].ShouldBe("$5"); } + [Fact] + public void ParameterRefsReportCastTypes() + { + const string query = "SELECT $1::int, CAST($2 AS numeric(10,2)), $3, '😀', $4::text[], $1 FROM t WHERE é = $5::timestamptz"; + + var parameters = Parser.ParameterRefs(query).GetValueOrThrow(); + + parameters.Select(parameter => (parameter.Number, parameter.TypeName)).ShouldBe( + [ + (1, "int"), (2, "numeric(10, 2)"), (3, null), (4, "text[]"), (1, null), (5, "timestamptz") + ]); + parameters.Select(parameter => query[parameter.Start..parameter.End]).ShouldBe(["$1", "$2", "$3", "$4", "$1", "$5"]); + } + + [Fact] + public void ParameterRefsWithoutTypesWhenTheQueryDoesNotParse() + { + // Scans, but is not valid SQL. + var parameters = Parser.ParameterRefs("SELECT $1::int FROM WHERE $2").GetValueOrThrow(); + + parameters.Select(parameter => (parameter.Number, parameter.TypeName)).ShouldBe([(1, null), (2, null)]); + } + [Fact] public async Task ParameterRefsEdgeCases() { diff --git a/src/PgSqlParser/ParseTreeExtensions.cs b/src/PgSqlParser/ParseTreeExtensions.cs index bd9b72d..e383ddb 100644 --- a/src/PgSqlParser/ParseTreeExtensions.cs +++ b/src/PgSqlParser/ParseTreeExtensions.cs @@ -20,6 +20,71 @@ public readonly record struct NodeVisit(IMessage Node, IMessage? Parent, int Dep /// The node's position if that property is a list, otherwise null. public int? Index { get; init; } + + /// + /// The top-level statement the node belongs to, when the walk started from a . + /// For the statement's own this is that node. + /// + public RawStmt? Statement { get; init; } + + /// The position of in , or null. + public int? StatementIndex { get; init; } + + internal WalkFrame? ParentFrame { get; init; } + + /// + /// The nodes that contain this one, nearest first: , then its parent, and so on + /// up to the node directly below the root of the walk. + /// + public IEnumerable Ancestors + { + get + { + for (var frame = ParentFrame; frame is not null; frame = frame.Parent) + yield return frame.Node; + } + } + + /// + /// Returns the nearest containing node of type , or null. For example + /// visit.FindAncestor<CommonTableExpr>() tells whether a node is inside a CTE. + /// + public T? FindAncestor() where T : class, IMessage + { + for (var frame = ParentFrame; frame is not null; frame = frame.Parent) + { + if (frame.Node is T match) + return match; + } + + return null; + } + + /// + /// Two visits are equal when they are the same node in the same place. Nodes are compared by + /// reference, since a tree can hold equal-looking nodes in different places. + /// + public bool Equals(NodeVisit other) + { + return ReferenceEquals(Node, other.Node) + && ReferenceEquals(Parent, other.Parent) + && Depth == other.Depth + && FieldName == other.FieldName + && Index == other.Index + && StatementIndex == other.StatementIndex; + } + + public override int GetHashCode() + { + return HashCode.Combine(System.Runtime.CompilerServices.RuntimeHelpers.GetHashCode(Node), Depth, FieldName, Index); + } +} + +// One link of the chain from a visited node up to the root of the walk. +internal sealed class WalkFrame(IMessage node, WalkFrame? parent) +{ + public IMessage Node { get; } = node; + public WalkFrame? Parent { get; } = parent; } /// @@ -80,7 +145,7 @@ public static void Walk(this IMessage root, Func visitor) ArgumentNullException.ThrowIfNull(visitor); var pending = new Stack(); - PushChildren(pending, root is Node rootWrapper ? rootWrapper.Unwrap() : root, null, 1); + PushChildren(pending, root is Node rootWrapper ? rootWrapper.Unwrap() : root, null); while (pending.Count > 0) { @@ -90,7 +155,7 @@ public static void Walk(this IMessage root, Func visitor) return; if (action == WalkAction.Continue) - PushChildren(pending, visit.Node, visit.Node, visit.Depth + 1); + PushChildren(pending, visit.Node, visit); } } @@ -169,21 +234,27 @@ public static Result Deparse(this IMessage node) private static IEnumerable WalkIterator(IMessage root) { var pending = new Stack(); - PushChildren(pending, root is Node rootWrapper ? rootWrapper.Unwrap() : root, null, 1); + PushChildren(pending, root is Node rootWrapper ? rootWrapper.Unwrap() : root, null); while (pending.Count > 0) { var visit = pending.Pop(); yield return visit; - PushChildren(pending, visit.Node, visit.Node, visit.Depth + 1); + PushChildren(pending, visit.Node, visit); } } - private static void PushChildren(Stack pending, IMessage? message, IMessage? parent, int depth) + // Pushes the children of message. owner is the visit of message itself, or null when message is the root. + private static void PushChildren(Stack pending, IMessage? message, NodeVisit? owner) { if (message is null) return; + var frame = owner is { } visit ? new WalkFrame(visit.Node, visit.ParentFrame) : null; + var depth = (owner?.Depth ?? 0) + 1; + // Statements are the direct children of a ParseResult that the walk started from. + var startsStatements = owner is null && message is ParseResult; + // Pushed in reverse so they are popped, and so visited, in field order. var fields = message.Descriptor.Fields.InFieldNumberOrder(); for (var i = fields.Count - 1; i >= 0; i--) @@ -197,21 +268,32 @@ private static void PushChildren(Stack pending, IMessage? message, IM { var items = (System.Collections.IList)value; for (var j = items.Count - 1; j >= 0; j--) - Push(pending, (IMessage)items[j]!, parent, depth, field, j); + Push((IMessage)items[j]!, j); } else if (value is IMessage child) { - Push(pending, child, parent, depth, field, null); + Push(child, null); } - } - } - private static void Push(Stack pending, IMessage child, IMessage? parent, int depth, - FieldDescriptor field, int? index) - { - // A Node wrapper is transparent: the field and index are those of the wrapper. - var node = child is Node wrapper ? wrapper.Unwrap() : child; - if (node is not null) - pending.Push(new NodeVisit(node, parent, depth) { FieldName = field.PropertyName, Index = index }); + continue; + + void Push(IMessage child, int? index) + { + // A Node wrapper is transparent: the field and index are those of the wrapper. + var node = child is Node wrapper ? wrapper.Unwrap() : child; + if (node is null) + return; + + var statement = startsStatements ? node as RawStmt : owner?.Statement; + pending.Push(new NodeVisit(node, owner?.Node, depth) + { + FieldName = field.PropertyName, + Index = index, + Statement = statement, + StatementIndex = startsStatements && statement is not null ? index : owner?.StatementIndex, + ParentFrame = frame + }); + } + } } } diff --git a/src/PgSqlParser/Parser.cs b/src/PgSqlParser/Parser.cs index d30527f..3acd386 100644 --- a/src/PgSqlParser/Parser.cs +++ b/src/PgSqlParser/Parser.cs @@ -185,7 +185,14 @@ public record DeparseComment(int MatchLocation, int NewlinesBeforeComment, int N /// The parameter number: 1 for $1. /// Where the reference starts in the query string, as a UTF-16 offset. /// Where it ends (exclusive), so query[Start..End] is the reference. -public record ParameterRef(int Number, int Start, int End); +public record ParameterRef(int Number, int Start, int End) +{ + /// + /// The type the parameter is cast to in the query, as in $1::int or CAST($1 AS int), + /// or null if it is not cast or the query does not parse. + /// + public string? TypeName { get; init; } +} /// /// Options for . @@ -942,6 +949,7 @@ public static Result DeparseNode(IMessage node) /// /// Find the parameter references ($1, $2, ...) in a query, in the order they appear. /// A reference inside a string literal or a comment is not a parameter and is not returned. + /// When a parameter is cast, as in $1::int, its is set. /// /// /// @@ -962,7 +970,33 @@ public static Result> ParameterRefs(string query) } } - return Result>.Success(parameters); + return Result>.Success(WithCastTypes(query, parameters)); + } + + // The cast type needs the parse tree. A query that scans but does not parse keeps its parameters, + // only without types. + private static IReadOnlyList WithCastTypes(string query, List parameters) + { + if (parameters.Count == 0 || !Parse(query).TryGetValue(out var tree)) + return parameters; + + // Tree locations are UTF-8 byte offsets, the parameters' Start is a string offset. + var mapper = new Utf8OffsetMapper(query); + var typeByStart = new Dictionary(); + foreach (var cast in tree.Descendants()) + { + if (cast.Arg?.Unwrap() is ParamRef { Location: >= 0 } parameter + && cast.TypeName is not null + && mapper.TryToCharOffset(parameter.Location, out var start) + && DeparseNode(cast.TypeName).TryGetValue(out var typeName)) + { + typeByStart[start] = typeName; + } + } + + return typeByStart.Count == 0 + ? parameters + : parameters.Select(p => typeByStart.TryGetValue(p.Start, out var type) ? p with { TypeName = type } : p).ToList(); } /// From dea1134607c7f472053b9ee4114f058a2acc4808 Mon Sep 17 00:00:00 2001 From: mysticmind Date: Fri, 9 Oct 2026 18:53:01 +0530 Subject: [PATCH 3/3] Add tree rewriting and comparison, fragment parsing, identifier quoting; bump to 2.1.0 - Rewrite keeps, replaces or removes nodes in place, children first - EqualsIgnoringLocations compares trees and nodes by structure - AsNode wraps a node for a property or list that takes any node - Parser.ParseExpression and Parser.ParseTypeName parse a single expression or type name on its own - PgIdentifier.Quote and NeedsQuoting follow PostgreSQL's quote_ident, using the scanner's keyword categories - Version 2.1.0, with a What's new in 2.1 section in the README --- README.md | 48 ++++ src/PackageSmokeTest/Program.cs | 13 ++ src/PgSqlParser.Tests/TreeEditingTests.cs | 267 ++++++++++++++++++++++ src/PgSqlParser/ParseTreeExtensions.cs | 225 ++++++++++++++++++ src/PgSqlParser/Parser.cs | 71 +++++- src/PgSqlParser/PgIdentifier.cs | 71 ++++++ src/PgSqlParser/PgSqlParser.csproj | 2 +- 7 files changed, 687 insertions(+), 10 deletions(-) create mode 100644 src/PgSqlParser.Tests/TreeEditingTests.cs create mode 100644 src/PgSqlParser/PgIdentifier.cs diff --git a/README.md b/README.md index de0c435..3ef354c 100644 --- a/README.md +++ b/README.md @@ -229,6 +229,40 @@ foreach (var visit in tree.Walk()) `visit.Ancestors` lists every containing node, nearest first. +#### Changing and comparing trees + +`Rewrite` walks the tree and lets you keep, replace or remove each node. It changes the tree in place, visiting children before their parents, and the result can be deparsed: + +```csharp +var tree = Parser.Parse("SELECT a, secret FROM old_name WHERE kind = 'x'::text").GetValueOrThrow(); + +tree.Rewrite(visit => visit.Node switch +{ + // Point the query at another table + RangeVar { Relname: "old_name" } => NodeEdit.ReplaceWith(new RangeVar { Relname = "new_name", Inh = true, Relpersistence = "p" }), + // Drop a cast, keeping what it wraps + TypeCast cast => NodeEdit.ReplaceWith(cast.Arg), + // Remove an item from a list + ResTarget target when target.Deparse().Value == "secret" => NodeEdit.Remove, + _ => NodeEdit.Keep +}); + +Console.WriteLine(tree.Deparse().GetValueOrThrow()); +// Output: SELECT a FROM new_name WHERE kind = 'x' +``` + +`EqualsIgnoringLocations` compares two trees, or two nodes, by structure. Queries that differ only in whitespace, comments, keyword case or redundant parentheses are equal: + +```csharp +var a = Parser.Parse("SELECT a FROM t WHERE x <> 1").GetValueOrThrow(); +var b = Parser.Parse("select a\nfrom t -- note\nwhere (x != 1)").GetValueOrThrow(); + +Console.WriteLine(a.EqualsIgnoringLocations(b)); +// Output: True +``` + +A few helpers go with these: `Parser.ParseExpression` and `Parser.ParseTypeName` parse a single expression or type name on its own, `node.AsNode()` wraps a node for a property or list that takes any node, and `PgIdentifier.Quote` quotes a name the way PostgreSQL's `quote_ident` does. + `Walk` and `Descendants` visit parents before their children and siblings in field order, which is not always the order of the query text. `GetLocation()` returns a node's location if it has one; see [Offsets and non-ASCII text](#offsets-and-non-ascii-text) for its unit. Only statements record a length, so `GetText` is available for statements and not for other nodes. ### Scan @@ -543,6 +577,20 @@ Deeply nested queries, such as a long chain of operators without parentheses (`a Protobuf's own recursive operations on a parse tree, such as `ToString()`, `Clone()` and `Equals()`, run on your thread. On a very deeply nested tree they can still overflow a small stack. +## What's new in 2.1 + +All additions; nothing from 2.0 changes. + +- **Deparse a single node**: `node.Deparse()` turns one clause, expression or table reference back into SQL. +- **More control when walking**: skip a subtree or stop, and each visit knows its ancestors, the property it came from and its top-level statement. +- **Change trees**: `Rewrite` keeps, replaces or removes nodes in place. +- **Compare trees**: `EqualsIgnoringLocations` treats queries that differ only in formatting as equal. +- **Parse fragments**: `ParseExpression` and `ParseTypeName`. +- **Parameters**: `ParameterRefs` finds each `$n`, where it is, and the type it is cast to. +- **Identifier quoting**: `PgIdentifier.Quote`, following PostgreSQL's `quote_ident`. + +See [Navigating the parse tree](#navigating-the-parse-tree) and [ParameterRefs](#parameterrefs). + ## What's new in 2.0 - **PostgreSQL 18**: built on libpg_query 18.1.0, which uses the PostgreSQL 18.6 parser and includes its security fix for `pg_query_normalize`. diff --git a/src/PackageSmokeTest/Program.cs b/src/PackageSmokeTest/Program.cs index 3a72c70..14af966 100644 --- a/src/PackageSmokeTest/Program.cs +++ b/src/PackageSmokeTest/Program.cs @@ -52,6 +52,19 @@ void Check(string name, bool ok) var tableVisit = walked.Walk().First(visit => visit.Node is RangeVar); Check("Ancestors and statement", tableVisit.FindAncestor() is not null && tableVisit.StatementIndex == 0); Check("ParameterRefs cast type", Parser.ParameterRefs("SELECT $1::int").Value?[0].TypeName == "int"); +var edited = Parser.Parse("SELECT a, b FROM old_name WHERE c = 'x'::text").Value!; +edited.Rewrite(visit => visit.Node switch +{ + RangeVar => NodeEdit.ReplaceWith(new RangeVar { Relname = "new_name", Inh = true, Relpersistence = "p" }), + TypeCast cast => NodeEdit.ReplaceWith(cast.Arg), + ResTarget when visit.Index == 1 => NodeEdit.Remove, + _ => NodeEdit.Keep +}); +Check("Rewrite", edited.Deparse().Value == "SELECT a FROM new_name WHERE c = 'x'"); +Check("EqualsIgnoringLocations", edited.EqualsIgnoringLocations(Parser.Parse("select a\nfrom new_name where (c = 'x')").Value)); +Check("ParseExpression", Parser.ParseExpression("(a > 0)").Value?.Deparse().Value == "a > 0"); +Check("ParseTypeName", Parser.ParseTypeName("character varying(20)").Value?.Deparse().Value == "varchar(20)"); +Check("PgIdentifier", PgIdentifier.Quote("public", "order") == "public.\"order\""); Check("ParameterRefs", Parser.ParameterRefs("SELECT $1, '$2', $3").Value?.Select(p => p.Number).SequenceEqual([1, 3]) == true); Check("ParsePlpgsqlFunctions", Parser.ParsePlpgsqlFunctions( "CREATE FUNCTION f() RETURNS int AS $$ BEGIN RETURN 1; END; $$ LANGUAGE plpgsql").Value?[0].Queries().SequenceEqual(["1"]) == true); diff --git a/src/PgSqlParser.Tests/TreeEditingTests.cs b/src/PgSqlParser.Tests/TreeEditingTests.cs new file mode 100644 index 0000000..7a701ae --- /dev/null +++ b/src/PgSqlParser.Tests/TreeEditingTests.cs @@ -0,0 +1,267 @@ +using Google.Protobuf; +using Shouldly; +using Xunit; + +namespace PgSqlParser.Tests; + +public class TreeEditingTests +{ + private static ParseResult Parse(string query) => Parser.Parse(query).GetValueOrThrow(); + + private static string Deparse(IMessage node) => node.Deparse().GetValueOrThrow(); + + [Theory] + [InlineData("SELECT a FROM t WHERE x = 1", "select a\nfrom t -- comment\n where x=1")] + [InlineData("CREATE INDEX i ON t (lower(name))", "CREATE INDEX i ON t ((lower(name)))")] + [InlineData("SELECT * FROM t WHERE a <> 1", "SELECT * FROM t WHERE a != 1")] + [InlineData("CREATE TABLE t (a varchar(10), b int)", "CREATE TABLE t (a character varying(10), b integer)")] + [InlineData("SELECT 1; SELECT 2", "SELECT 1;\n\n SELECT 2;")] + public void EqualsIgnoringLocationsForTheSameSql(string left, string right) + { + var a = Parse(left); + var b = Parse(right); + + a.EqualsIgnoringLocations(b).ShouldBeTrue(); + b.EqualsIgnoringLocations(a).ShouldBeTrue(); + } + + [Theory] + [InlineData("SELECT a FROM t WHERE x = 1", "SELECT a FROM t WHERE x = 2")] + [InlineData("SELECT a FROM t", "SELECT b FROM t")] + [InlineData("SELECT a FROM t", "SELECT a FROM t WHERE true")] + [InlineData("SELECT a, b FROM t", "SELECT a FROM t")] + [InlineData("CREATE INDEX i ON t (a)", "CREATE UNIQUE INDEX i ON t (a)")] + [InlineData("CREATE INDEX i ON t (a)", "CREATE INDEX i ON t (a DESC)")] + [InlineData("SELECT 'a'", "SELECT 'A'")] + [InlineData("SELECT 1", "SELECT 1; SELECT 1")] + public void EqualsIgnoringLocationsSeesRealDifferences(string left, string right) + { + Parse(left).EqualsIgnoringLocations(Parse(right)).ShouldBeFalse(); + } + + [Fact] + public void PlainEqualityIsSensitiveToLocations() + { + var a = Parse("SELECT a FROM t"); + var b = Parse("SELECT a FROM t"); + + a.Equals(b).ShouldBeFalse(); + a.EqualsIgnoringLocations(b).ShouldBeTrue(); + } + + [Fact] + public void EqualsIgnoringLocationsOnNodesAndNulls() + { + var where1 = Parse("SELECT 1 FROM t WHERE a = 1 AND b = 'x'").Stmts[0].Stmt.SelectStmt.WhereClause; + var where2 = Parse("DELETE FROM other WHERE (a = 1) AND (b = 'x')").Stmts[0].Stmt.DeleteStmt.WhereClause; + + where1.EqualsIgnoringLocations(where2).ShouldBeTrue(); + // A wrapper and what it wraps are different things to compare. + where1.EqualsIgnoringLocations(where1.Unwrap()).ShouldBeFalse(); + where1.EqualsIgnoringLocations(null).ShouldBeFalse(); + ((IMessage?)null).EqualsIgnoringLocations(null).ShouldBeTrue(); + } + + [Fact] + public void RewriteReplacesNodes() + { + var tree = Parse("SELECT a FROM old_table WHERE b = 1 AND c IN (SELECT d FROM old_table)"); + + tree.Rewrite(visit => visit.Node is RangeVar { Relname: "old_table" } + ? NodeEdit.ReplaceWith(new RangeVar { Relname = "new_table", Inh = true, Relpersistence = "p" }) + : NodeEdit.Keep); + + Deparse(tree).ShouldBe("SELECT a FROM new_table WHERE b = 1 AND c IN (SELECT d FROM new_table)"); + } + + [Fact] + public void RewriteCanUnwrapANodeIntoItsChild() + { + // PostgreSQL adds casts like this when it stores an expression. + var tree = Parse("SELECT lower(data ->> 'name'::text) FROM t WHERE kind = 'a'::text AND n = '1'::int"); + + tree.Rewrite(visit => visit.Node is TypeCast { Arg: { } arg, TypeName: { } type } + && arg.Unwrap() is A_Const { Sval: not null } + && Deparse(type) == "text" + ? NodeEdit.ReplaceWith(arg) + : NodeEdit.Keep); + + Deparse(tree).ShouldBe("SELECT lower(data ->> 'name') FROM t WHERE kind = 'a' AND n = '1'::int"); + } + + [Fact] + public void RewriteRemovesListItemsAndSingleProperties() + { + var tree = Parse("SELECT a, secret, b FROM t WHERE x = 1 ORDER BY a LIMIT 5"); + + tree.Rewrite(visit => visit switch + { + { Node: ResTarget target } when Deparse(target) == "secret" => NodeEdit.Remove, + { FieldName: nameof(SelectStmt.WhereClause) } => NodeEdit.Remove, + { FieldName: nameof(SelectStmt.LimitCount) } => NodeEdit.Remove, + _ => NodeEdit.Keep + }); + + Deparse(tree).ShouldBe("SELECT a, b FROM t ORDER BY a"); + } + + [Fact] + public void RewriteVisitsChildrenBeforeParents() + { + var tree = Parse("SELECT f(g(1))"); + var order = new List(); + + tree.Rewrite(visit => + { + if (visit.Node is FuncCall call) + order.Add(call.Funcname[0].String.Sval); + + return NodeEdit.Keep; + }); + + order.ShouldBe(["g", "f"]); + } + + [Fact] + public void RewriteGivesTheSameContextAsWalk() + { + var tree = Parse("SELECT 1; WITH r AS (SELECT a FROM inner_table) SELECT b FROM r"); + NodeVisit? seen = null; + + tree.Rewrite(visit => + { + if (visit.Node is RangeVar { Relname: "inner_table" }) + seen = visit; + + return NodeEdit.Keep; + }); + + var expected = tree.Walk().Single(visit => visit.Node is RangeVar { Relname: "inner_table" }); + seen.ShouldNotBeNull().ShouldBe(expected); + seen.Value.StatementIndex.ShouldBe(1); + seen.Value.FindAncestor().ShouldNotBeNull(); + } + + [Fact] + public void RewriteRejectsAReplacementThatDoesNotFit() + { + var tree = Parse("INSERT INTO t (a) VALUES (1)"); + + // InsertStmt.Relation holds a RangeVar, not any node. + var exception = Should.Throw(() => tree.Rewrite(visit => + visit.Node is RangeVar ? NodeEdit.ReplaceWith(new A_Const()) : NodeEdit.Keep)); + + exception.Message.ShouldBe("InsertStmt.Relation holds a RangeVar, so it cannot be replaced with a A_Const."); + Should.Throw(() => NodeEdit.ReplaceWith(null!)); + Should.Throw(() => tree.Rewrite(null!)); + } + + [Fact] + public void RewriteThenCompare() + { + var written = Parse("CREATE INDEX i ON t (lower(data ->> 'name'))"); + var stored = Parse("CREATE INDEX i ON t USING btree (lower((data ->> 'name'::text)))"); + written.EqualsIgnoringLocations(stored).ShouldBeFalse(); + + stored.Rewrite(visit => visit.Node is TypeCast { Arg: { } arg } && arg.Unwrap() is A_Const { Sval: not null } + ? NodeEdit.ReplaceWith(arg) + : NodeEdit.Keep); + + written.EqualsIgnoringLocations(stored).ShouldBeTrue(); + } + + [Fact] + public void AsNodeWrapsAnyNodeType() + { + var table = new RangeVar { Relname = "t" }; + + var node = table.AsNode(); + + node.RangeVar.ShouldBeSameAs(table); + node.AsNode().ShouldBeSameAs(node); + new SelectStmt().AsNode().NodeCase.ShouldBe(Node.NodeOneofCase.SelectStmt); + // A ParseResult is the container of a tree, not a node in it. + Should.Throw(() => new ParseResult().AsNode()); + } + + [Theory] + [InlineData("(a > 0) AND b = 'x'::text", "a > 0 AND b = 'x'::text")] + [InlineData("nextval('seq'::regclass)", "nextval('seq'::regclass)")] + [InlineData("'😀' || \"é\" -- trailing comment", "'😀' || \"é\"")] + [InlineData("CASE WHEN a THEN 1 ELSE 2 END", "CASE WHEN a THEN 1 ELSE 2 END")] + [InlineData("(SELECT max(id) FROM t)", "(SELECT max(id) FROM t)")] + public void ParseExpression(string expression, string deparsed) + { + Deparse(Parser.ParseExpression(expression).GetValueOrThrow()).ShouldBe(deparsed); + } + + [Theory] + [InlineData("1, 2")] + [InlineData("1 AS x")] + [InlineData("a FROM t")] + [InlineData("a ORDER BY 1")] + [InlineData("a UNION SELECT 1")] + [InlineData("1) FROM t WHERE (true")] + [InlineData("1); DROP TABLE t; SELECT (1")] + [InlineData("a >")] + [InlineData("")] + public void ParseExpressionRejectsAnythingElse(string expression) + { + Parser.ParseExpression(expression).IsSuccess.ShouldBeFalse(); + } + + [Theory] + [InlineData("numeric(10,2)", "numeric(10, 2)")] + [InlineData("character varying(20)", "varchar(20)")] + [InlineData("text[]", "text[]")] + [InlineData("public.my_type", "public.my_type")] + [InlineData("timestamptz", "timestamptz")] + public void ParseTypeName(string typeName, string deparsed) + { + Deparse(Parser.ParseTypeName(typeName).GetValueOrThrow()).ShouldBe(deparsed); + } + + [Theory] + [InlineData("int, 2")] + [InlineData("int FROM t")] + [InlineData("int; DROP TABLE t")] + [InlineData("")] + public void ParseTypeNameRejectsAnythingElse(string typeName) + { + Parser.ParseTypeName(typeName).IsSuccess.ShouldBeFalse(); + } + + [Theory] + [InlineData("users", "users")] + [InlineData("_x1", "_x1")] + [InlineData("name", "name")] + [InlineData("type", "type")] + [InlineData("Users", "\"Users\"")] + [InlineData("user", "\"user\"")] + [InlineData("select", "\"select\"")] + [InlineData("order", "\"order\"")] + [InlineData("between", "\"between\"")] + [InlineData("my table", "\"my table\"")] + [InlineData("a$b", "\"a$b\"")] + [InlineData("1abc", "\"1abc\"")] + [InlineData("é", "\"é\"")] + [InlineData("say \"hi\"", "\"say \"\"hi\"\"\"")] + [InlineData("", "\"\"")] + public void QuoteIdentifier(string name, string quoted) + { + PgIdentifier.Quote(name).ShouldBe(quoted); + PgIdentifier.NeedsQuoting(name).ShouldBe(quoted != name); + + // What comes out must scan back as one identifier, unless it was empty. + if (name.Length > 0) + Parser.Parse($"SELECT 1 FROM {quoted}").GetValueOrThrow().Descendants().Single().Relname.ShouldBe(name); + } + + [Fact] + public void QuoteQualifiedIdentifier() + { + PgIdentifier.Quote("public", "users").ShouldBe("public.users"); + PgIdentifier.Quote("My Schema", "order", "col").ShouldBe("\"My Schema\".\"order\".col"); + Should.Throw(() => PgIdentifier.Quote((string)null!)); + } +} diff --git a/src/PgSqlParser/ParseTreeExtensions.cs b/src/PgSqlParser/ParseTreeExtensions.cs index e383ddb..d5d1e26 100644 --- a/src/PgSqlParser/ParseTreeExtensions.cs +++ b/src/PgSqlParser/ParseTreeExtensions.cs @@ -80,6 +80,37 @@ public override int GetHashCode() } } +/// +/// What does with a node after visiting it. +/// +public readonly struct NodeEdit +{ + private NodeEdit(bool remove, IMessage? replacement) + { + IsRemove = remove; + Replacement = replacement; + } + + internal bool IsRemove { get; } + internal IMessage? Replacement { get; } + + /// Leave the node as it is. + public static NodeEdit Keep => default; + + /// Remove the node: from its list, or by clearing the property that holds it. + public static NodeEdit Remove => new(true, null); + + /// + /// Put in the node's place. Where the parent's property takes any node, + /// any node type is accepted; where it takes one specific type, the replacement must be of that type. + /// + public static NodeEdit ReplaceWith(IMessage node) + { + ArgumentNullException.ThrowIfNull(node); + return new NodeEdit(false, node); + } +} + // One link of the chain from a visited node up to the root of the walk. internal sealed class WalkFrame(IMessage node, WalkFrame? parent) { @@ -230,6 +261,200 @@ public static Result Deparse(this IMessage node) return Parser.DeparseNode(node); } + /// + /// Wraps a node in a , the type most parse tree properties and lists hold. + /// A is returned as it is. + /// + public static Node AsNode(this IMessage node) + { + ArgumentNullException.ThrowIfNull(node); + + if (node is Node already) + return already; + + if (!NodeFields.Value.TryGetValue(node.Descriptor.FullName, out var field)) + throw new ArgumentException($"A {node.Descriptor.Name} cannot be held by a Node.", nameof(node)); + + var wrapper = new Node(); + field.Accessor.SetValue(wrapper, node); + return wrapper; + } + + // The Node field that holds each node type, keyed by the node type's full name. + internal static readonly Lazy> NodeFields = new(() => + Node.Descriptor.Fields.InDeclarationOrder() + .Where(field => field.FieldType == FieldType.Message) + .ToDictionary(field => field.MessageType.FullName)); + + /// + /// Walks the nodes below and lets keep, replace or + /// remove each one. The tree is changed in place. Children are visited before their parent, so a + /// visitor sees a node after everything below it has already been rewritten. A replacement is not + /// visited again. + /// + /// A replacement does not fit the property that holds the node. + public static void Rewrite(this IMessage root, Func visitor) + { + ArgumentNullException.ThrowIfNull(root); + ArgumentNullException.ThrowIfNull(visitor); + + RewriteChildren(root is Node rootWrapper ? rootWrapper.Unwrap() : root, null, visitor); + } + + private static void RewriteChildren(IMessage? message, NodeVisit? owner, Func visitor) + { + if (message is null) + return; + + // Recursive, so stop with an exception before a very deep tree can overflow the stack. + System.Runtime.CompilerServices.RuntimeHelpers.EnsureSufficientExecutionStack(); + + var frame = owner is { } ownerVisit ? new WalkFrame(ownerVisit.Node, ownerVisit.ParentFrame) : null; + var depth = (owner?.Depth ?? 0) + 1; + var startsStatements = owner is null && message is ParseResult; + + foreach (var field in message.Descriptor.Fields.InFieldNumberOrder()) + { + if (field.FieldType != FieldType.Message || field.IsMap) + continue; + + var value = field.Accessor.GetValue(message); + if (field.IsRepeated) + { + var items = (System.Collections.IList)value; + for (var i = 0; i < items.Count; i++) + { + var edit = Visit((IMessage)items[i]!, i); + if (edit.IsRemove) + items.RemoveAt(i--); + else if (edit.Replacement is { } replacement) + items[i] = Fit(replacement, field); + } + } + else if (value is IMessage child) + { + var edit = Visit(child, null); + if (edit.IsRemove) + field.Accessor.Clear(message); + else if (edit.Replacement is { } replacement) + field.Accessor.SetValue(message, Fit(replacement, field)); + } + + continue; + + NodeEdit Visit(IMessage child, int? index) + { + var node = child is Node wrapper ? wrapper.Unwrap() : child; + if (node is null) + return NodeEdit.Keep; + + var statement = startsStatements ? node as RawStmt : owner?.Statement; + var visit = new NodeVisit(node, owner?.Node, depth) + { + FieldName = field.PropertyName, + Index = index, + Statement = statement, + StatementIndex = startsStatements && statement is not null ? index : owner?.StatementIndex, + ParentFrame = frame + }; + + RewriteChildren(node, visit, visitor); + return visitor(visit); + } + } + } + + // Makes a replacement fit the property: wrapped in a Node where the property takes any node, + // otherwise it has to be of the property's own type. + private static IMessage Fit(IMessage replacement, FieldDescriptor field) + { + if (field.MessageType == Node.Descriptor) + return replacement.AsNode(); + + var node = replacement is Node wrapper ? wrapper.Unwrap() : replacement; + if (node is null || node.Descriptor != field.MessageType) + { + throw new InvalidOperationException( + $"{field.ContainingType.Name}.{field.PropertyName} holds a {field.MessageType.Name}, " + + $"so it cannot be replaced with a {(node ?? replacement).Descriptor.Name}."); + } + + return node; + } + + /// + /// Compares two parse trees, or two nodes, by structure and ignoring where each node was in its + /// query text. Two queries that differ only in whitespace, comments, keyword case or redundant + /// parentheses give trees that are equal by this comparison. + /// + public static bool EqualsIgnoringLocations(this IMessage? left, IMessage? right) + { + var pending = new Stack<(IMessage? Left, IMessage? Right)>(); + pending.Push((left, right)); + + while (pending.Count > 0) + { + var (a, b) = pending.Pop(); + if (ReferenceEquals(a, b)) + continue; + + if (a is null || b is null || a.Descriptor != b.Descriptor) + return false; + + // A Node holds exactly one of 271 fields. Compare what it wraps instead of reading them all. + if (a is Node nodeA && b is Node nodeB) + { + pending.Push((nodeA.Unwrap(), nodeB.Unwrap())); + continue; + } + + foreach (var field in a.Descriptor.Fields.InFieldNumberOrder()) + { + if (IsLocationField(field)) + continue; + + var valueA = field.Accessor.GetValue(a); + var valueB = field.Accessor.GetValue(b); + if (field.IsRepeated) + { + var listA = (System.Collections.IList)valueA; + var listB = (System.Collections.IList)valueB; + if (listA.Count != listB.Count) + return false; + + for (var i = 0; i < listA.Count; i++) + { + if (field.FieldType == FieldType.Message) + pending.Push(((IMessage?)listA[i], (IMessage?)listB[i])); + else if (!Equals(listA[i], listB[i])) + return false; + } + } + else if (field.FieldType == FieldType.Message) + { + pending.Push((valueA as IMessage, valueB as IMessage)); + } + else if (!Equals(valueA, valueB)) + { + return false; + } + } + } + + return true; + } + + // Positions in the query text: "location", "stmt_location", "arg_location" and so on, and the + // statement length that goes with stmt_location. + private static bool IsLocationField(FieldDescriptor field) + { + return field.FieldType == FieldType.Int32 + && !field.IsRepeated + && (field.Name == LocationFieldName + || field.Name.EndsWith("_location", StringComparison.Ordinal) + || field.Name == "stmt_len"); + } + // An explicit stack instead of recursion, so a deeply nested tree cannot overflow the call stack. private static IEnumerable WalkIterator(IMessage root) { diff --git a/src/PgSqlParser/Parser.cs b/src/PgSqlParser/Parser.cs index 3acd386..c0a28a2 100644 --- a/src/PgSqlParser/Parser.cs +++ b/src/PgSqlParser/Parser.cs @@ -415,6 +415,66 @@ public static Task> ParseAsync(string query, ParserOptions p return RunAsync(() => Parse(query, parserOptions), cancellationToken); } + /// + /// Parse a single expression on its own, such as a column default, a CHECK condition or what + /// pg_get_expr returns. Anything that is not exactly one expression is an error. Locations + /// in the returned node are not relative to . + /// + /// + /// + public static Result ParseExpression(string expression) + { + ArgumentNullException.ThrowIfNull(expression); + + // In a WHERE clause there is room for exactly one expression, with no alias and no list. + // The line break lets the expression end in a line comment. + var parsed = Parse($"SELECT WHERE {expression}\n"); + if (!parsed.TryGetValue(out var tree, out var error)) + return Result.Failure(error); + + if (tree.Stmts.Count == 1 + && tree.Stmts[0].Stmt?.SelectStmt is { WhereClause: { } value } select + && select.Equals(new SelectStmt { WhereClause = value, LimitOption = select.LimitOption, Op = SetOperation.SetopNone })) + { + return Result.Success(value); + } + + return Result.Failure(NodeError("not a single expression")); + } + + /// + /// Parse a type name on its own, such as numeric(10,2), text[] or public.my_type. + /// + /// + /// + public static Result ParseTypeName(string typeName) + { + ArgumentNullException.ThrowIfNull(typeName); + + var parsed = Parse($"SELECT NULL::{typeName}"); + if (!parsed.TryGetValue(out var tree, out var error)) + return Result.Failure(error); + + if (tree.Stmts.Count == 1 + && tree.Stmts[0].Stmt?.SelectStmt is { TargetList.Count: 1 } select + && select.TargetList[0].ResTarget is { Name: "" } target + && target.Val?.TypeCast is { TypeName: { } type, Arg.AConst.Isnull: true } + && IsBareSelect(select)) + { + return Result.Success(type); + } + + return Result.Failure(NodeError("not a single type name")); + } + + // True if the SELECT has a select list and nothing else. + private static bool IsBareSelect(SelectStmt select) + { + var bare = new SelectStmt { LimitOption = select.LimitOption, Op = select.Op }; + bare.TargetList.Add(select.TargetList); + return bare.Equals(select); + } + /// /// Parse PL/pgSQL function bodies and returns a JSON representation /// @@ -872,11 +932,10 @@ public static Result DeparseNode(IMessage node) return Deparse(new ParseResult { Version = PgVersionNum, Stmts = { rawStmt } }); } - if (!NodeFields.Value.TryGetValue(node.Descriptor.FullName, out var nodeField)) + if (!ParseTreeExtensions.NodeFields.Value.ContainsKey(node.Descriptor.FullName)) return Result.Failure(NodeError($"a {node.Descriptor.Name} cannot be deparsed on its own")); - var wrapped = new Node(); - nodeField.Accessor.SetValue(wrapped, node); + var wrapped = node.AsNode(); if (node.Descriptor.Name.EndsWith("Stmt", StringComparison.Ordinal)) return Deparse(new ParseResult { Version = PgVersionNum, Stmts = { new RawStmt { Stmt = wrapped } } }); @@ -938,12 +997,6 @@ public static Result DeparseNode(IMessage node) return Result.Success(sql[prefix.Length..^suffix.Length]); } - // The Node field that holds each node type, keyed by the node type's full name. - private static readonly Lazy> NodeFields = new(() => - Node.Descriptor.Fields.InDeclarationOrder() - .Where(field => field.FieldType == Google.Protobuf.Reflection.FieldType.Message) - .ToDictionary(field => field.MessageType.FullName)); - private static Error NodeError(string message) => new(message, null, null, 0, 0, null); /// diff --git a/src/PgSqlParser/PgIdentifier.cs b/src/PgSqlParser/PgIdentifier.cs new file mode 100644 index 0000000..6e7bd7f --- /dev/null +++ b/src/PgSqlParser/PgIdentifier.cs @@ -0,0 +1,71 @@ +using System.Collections.Concurrent; +using System.Text; + +namespace PgSqlParser; + +/// +/// Quoting of identifiers (table, column, schema names and so on) for use in SQL text, following the +/// rules of PostgreSQL's own quote_ident. +/// +public static class PgIdentifier +{ + // Whether a name that is otherwise safe is a keyword that must be quoted. Only names that are + // already safe by their characters get here, so the cache stays small. + private static readonly ConcurrentDictionary KeywordNeedsQuoting = new(); + + /// + /// Returns true if must be written in double quotes: it has characters other + /// than lower case ASCII letters, digits and underscores, starts with a digit, or is a keyword that + /// cannot be used as a name as it is. + /// + public static bool NeedsQuoting(string name) + { + ArgumentNullException.ThrowIfNull(name); + + if (name.Length == 0 || !(name[0] is >= 'a' and <= 'z' or '_')) + return true; + + foreach (var c in name) + { + if (!(c is >= 'a' and <= 'z' or >= '0' and <= '9' or '_')) + return true; + } + + return KeywordNeedsQuoting.GetOrAdd(name, static candidate => + { + // The scanner knows every keyword of this PostgreSQL version and its category. Like + // quote_ident, only unreserved keywords are left unquoted. + var scan = Parser.Scan(candidate); + return !scan.IsSuccess + || scan.Value.Tokens.Count != 1 + || scan.Value.Tokens[0].KeywordKind is not (KeywordKind.NoKeyword or KeywordKind.UnreservedKeyword); + }); + } + + /// + /// Returns ready to use in SQL: unchanged if it is safe, otherwise in double + /// quotes with any double quote inside it doubled. + /// + public static string Quote(string name) + { + return NeedsQuoting(name) ? "\"" + name.Replace("\"", "\"\"") + "\"" : name; + } + + /// + /// Returns a qualified name such as schema.table, quoting each part that needs it. + /// + public static string Quote(params string[] parts) + { + ArgumentNullException.ThrowIfNull(parts); + + var builder = new StringBuilder(); + foreach (var part in parts) + { + if (builder.Length > 0) + builder.Append('.'); + builder.Append(Quote(part)); + } + + return builder.ToString(); + } +} diff --git a/src/PgSqlParser/PgSqlParser.csproj b/src/PgSqlParser/PgSqlParser.csproj index 89c10f1..f285d3a 100644 --- a/src/PgSqlParser/PgSqlParser.csproj +++ b/src/PgSqlParser/PgSqlParser.csproj @@ -2,7 +2,7 @@ .NET wrapper on libpg_query to access PostgreSQL parser outside of the server environment - 2.0.0 + 2.1.0 Babu Annamalai net8.0;net9.0;net10.0 enable