From 097716a896b45129c1c8b0643bd4cbb349743314 Mon Sep 17 00:00:00 2001 From: mysticmind Date: Fri, 9 Oct 2026 22:14:03 +0530 Subject: [PATCH 1/2] Thin the README and move the guide into the docs site --- .github/workflows/ci.yaml | 2 + .github/workflows/docs.yaml | 1 - .gitignore | 1 - README.md | 828 ++--------------------------- docs/.vitepress/config.mts | 4 + docs/guide/analysis.md | 153 ++++++ docs/guide/deparse-format.md | 132 +++++ docs/guide/errors-offsets.md | 77 +++ docs/guide/getting-started.md | 70 +++ docs/guide/license.md | 13 + docs/guide/parse-tree.md | 200 +++++++ docs/guide/plpgsql.md | 75 +++ docs/guide/scan-split.md | 80 +++ docs/guide/whats-new.md | 63 +++ docs/package.json | 6 +- docs/scripts/copy-artwork.mjs | 12 + docs/scripts/gen.mjs | 94 ---- src/PgSqlParser/PgSqlParser.csproj | 2 +- 18 files changed, 918 insertions(+), 895 deletions(-) create mode 100644 docs/guide/analysis.md create mode 100644 docs/guide/deparse-format.md create mode 100644 docs/guide/errors-offsets.md create mode 100644 docs/guide/getting-started.md create mode 100644 docs/guide/license.md create mode 100644 docs/guide/parse-tree.md create mode 100644 docs/guide/plpgsql.md create mode 100644 docs/guide/scan-split.md create mode 100644 docs/guide/whats-new.md create mode 100644 docs/scripts/copy-artwork.mjs delete mode 100644 docs/scripts/gen.mjs diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 95f4697..ef6d4b2 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -6,12 +6,14 @@ on: - main paths-ignore: - '**.md' + - 'docs/**' - 'LICENSE*' pull_request: branches: - main paths-ignore: - '**.md' + - 'docs/**' - 'LICENSE*' workflow_dispatch: diff --git a/.github/workflows/docs.yaml b/.github/workflows/docs.yaml index a15dfc1..4b56ade 100644 --- a/.github/workflows/docs.yaml +++ b/.github/workflows/docs.yaml @@ -5,7 +5,6 @@ on: branches: [main] paths: - 'docs/**' - - 'README.md' - 'assets/**' - '.github/workflows/docs.yaml' workflow_dispatch: diff --git a/.gitignore b/.gitignore index 036bbe0..659bf77 100644 --- a/.gitignore +++ b/.gitignore @@ -21,5 +21,4 @@ Thumbs.db docs/node_modules docs/.vitepress/dist docs/.vitepress/cache -docs/guide docs/public diff --git a/README.md b/README.md index 2055bf6..c2b7960 100644 --- a/README.md +++ b/README.md @@ -15,6 +15,8 @@ With it you can: Native libraries are included for Linux (glibc and musl), macOS and Windows on x64 and ARM64, and the package is Native AOT compatible. +**Documentation: [mysticmind.github.io/pgsqlparser-dotnet](https://mysticmind.github.io/pgsqlparser-dotnet/)** + For background on why a query's parse tree is useful, see [pg_query 2.0: The easiest way to parse Postgres queries](https://pganalyze.com/blog/pg-query-2-0-postgres-query-parser). ## Installation @@ -34,822 +36,58 @@ Note that the libpg_query libs for all OS'es are already packaged with the assem This version is built on libpg_query 18.1.0, which uses the PostgreSQL 18.6 parser. It targets .NET 8, .NET 9 and .NET 10, and is compatible with trimming and Native AOT. -### Keeping deployments small - -The package contains the native library for every platform above, which is about 30 MB once unpacked. A portable publish copies all of them into your output. When deployment size matters, for example in a container image or a serverless function, publish for the platform you run on and only that platform's library is included: - -```shell -dotnet publish -r linux-x64 -``` - -| Publish command | Output size | Native libraries included | -|---|---|---| -| `dotnet publish` | about 31 MB | all 8 | -| `dotnet publish -r linux-x64` | about 5 MB | 1 | - -The sizes are for a small framework-dependent app. The runtime identifier must match where the app runs: use `linux-musl-x64` on Alpine, and the `arm64` variants on ARM64. - -## Usage - -All functions support both sync and async versions. - -### Working with results - -Every function returns a `Result` that holds either a `Value` or an `Error`. Pick the style that suits your code: - -```csharp -using PgSqlParser; - -// Check IsSuccess: the compiler then knows Value (or Error) is not null -var result = Parser.Parse(query); -if (result.IsSuccess) -{ - Console.WriteLine(result.Value.Stmts.Count); -} -else -{ - Console.WriteLine(result.Error.Message); -} - -// Throw on failure: PgSqlParserException carries the Error -var tree = Parser.Parse(query).GetValueOrThrow(); - -// Try pattern -if (Parser.Parse(query).TryGetValue(out var parsed, out var error)) -{ - Console.WriteLine(parsed.Stmts.Count); -} - -// Match and deconstruction -var text = Parser.Normalize(query).Match(value => value, e => e.Message); -var (normalized, normalizeError) = Parser.Normalize(query); -``` - -### Normalize - -Transform DML query (SELECT, INSERT, UPDATE, DELETE) into a canonical form by replacing literal values (constants) with placeholders ($1, $2) - -```csharp -using PgSqlParser; - -var query = "SELECT 1"; -var result = Parser.Normalize(query); - -if (result.Error is null) -{ - Console.WriteLine(result.Value); -} - -// Normalized value: SELECT $1 - -``` - -### NormalizeUtility - -Transform DDL and other utility commands (CREATE TABLE, ALTER TABLE, VACUUM, and ANALYZE et.al.) into a canonical form by replacing literal values (constants) with placeholders ($1, $2) - -```csharp -using PgSqlParser; - -var query = "CREATE ROLE postgres PASSWORD 'xyz'"; -var result = Parser.NormalizeUtility(query); - -if (result.Error is null) -{ - Console.WriteLine(result.Value); -} - -// Normalized Utility value: CREATE ROLE postgres PASSWORD $1 -``` - -### Parse - -Parse SQL and return a Protobuf based AST - -```csharp -using PgSqlParser; - -var query = "SELECT 1; SELECT 2"; - -var result = Parser.Parse(query); - -if (result.Error is null) -{ - Console.WriteLine(result.Value); -} - -// result.Value is a ParseResult AST object and the serialized JSON output is as below -// { "version": 180006, "stmts": [ { "stmt": { "SelectStmt": { "targetList": [ { "ResTarget": { "val": { "A_Const": { "ival": { "ival": 1 }, "location": 7 } }, "location": 7 } } ], "limitOption": "LIMIT_OPTION_DEFAULT", "op": "SETOP_NONE" } }, "stmt_len": 8 }, { "stmt": { "SelectStmt": { "targetList": [ { "ResTarget": { "val": { "A_Const": { "ival": { "ival": 2 }, "location": 17 } }, "location": 17 } } ], "limitOption": "LIMIT_OPTION_DEFAULT", "op": "SETOP_NONE" } }, "stmt_location": 10 } ] } -``` - -`Parse` can also take `ParserOptions` as a list of flags i.e. `ParserOptions.DisableBackslashQuote | ParserOptions.DisableEscapeStringWarning` - -### Navigating the parse tree - -The parse tree has more than 270 node types, and reaching a node by hand means spelling out the whole path to it. These helpers avoid that: - -```csharp -using PgSqlParser; - -var query = "SELECT c.name FROM customers c JOIN orders o ON o.customer_id = c.id; DROP TABLE tmp"; -var tree = Parser.Parse(query).GetValueOrThrow(); - -// Every node of a type, anywhere in the tree -foreach (var table in tree.Descendants()) -{ - Console.WriteLine(table.Relname); -} -// Output: customers, orders (DROP TABLE names its target as a plain name list, not a RangeVar) - -// Every node, with its parent and depth -foreach (var visit in tree.Walk()) -{ - Console.WriteLine($"{new string(' ', visit.Depth * 2)}{visit.Node.Descriptor.Name}"); -} - -// Pattern match on a statement without the .SelectStmt / .DropStmt property chain -foreach (var stmt in tree.Stmts) -{ - var kind = stmt.Stmt.Unwrap() switch - { - SelectStmt => "select", - DropStmt => "drop", - _ => "other" - }; - // The statement's text and kind - Console.WriteLine($"{kind}: {stmt.GetText(query)}"); -} -// Output: -// select: SELECT c.name FROM customers c JOIN orders o ON o.customer_id = c.id -// drop: DROP TABLE tmp -``` - -Any node can be turned back into SQL on its own, which is how to get the text of a single clause or expression: - -```csharp -var select = tree.Stmts[0].Stmt.SelectStmt; - -Console.WriteLine(select.FromClause[0].Deparse().GetValueOrThrow()); -// Output: customers c JOIN orders o ON o.customer_id = c.id - -foreach (var table in tree.Descendants()) -{ - Console.WriteLine(table.Deparse().GetValueOrThrow()); -} -// Output: customers c, orders o -``` - -`Deparse()` on a node works for statements, expressions, items of a FROM clause, select list and ORDER BY items, WITH clauses and type names. Nodes that are not SQL on their own, such as an `Alias`, return an error. - -To control the walk, pass a function. It can skip everything below a node or stop the walk, and each visit tells you which property of its parent the node came from: - -```csharp -// Tables used by the query itself, ignoring subqueries inside expressions -var tables = new List(); -tree.Walk(visit => -{ - if (visit.Node is SubLink) - return WalkAction.SkipChildren; - - if (visit.Node is RangeVar table) - tables.Add(table.Relname); - - return WalkAction.Continue; // or WalkAction.Stop to end the walk -}); - -// visit.FieldName is the parent's property, for example nameof(SelectStmt.WhereClause); -// visit.Index is the position when that property is a list. -``` - -Each visit also knows where it sits in the tree, which is what most rules need: - -```csharp -foreach (var visit in tree.Walk()) -{ - if (visit.Node is not RangeVar table) - continue; - - // The nearest containing node of a type, or null - var insideCte = visit.FindAncestor() is not null; - var insideSubquery = visit.FindAncestor() is not null; - - // The top-level statement the node belongs to - var writes = visit.Statement?.Stmt.Unwrap() is InsertStmt or UpdateStmt or DeleteStmt; - - Console.WriteLine($"statement {visit.StatementIndex}: {table.Relname} (cte: {insideCte}, subquery: {insideSubquery}, writes: {writes})"); -} -``` - -`visit.Ancestors` lists every containing node, nearest first. - -#### Changing and comparing trees - -`Rewrite` walks the tree and lets you keep, replace or remove each node. It changes the tree in place, visiting children before their parents, and the result can be deparsed: - -```csharp -var tree = Parser.Parse("SELECT a, secret FROM old_name WHERE kind = 'x'::text").GetValueOrThrow(); - -tree.Rewrite(visit => visit.Node switch -{ - // Point the query at another table - RangeVar { Relname: "old_name" } => NodeEdit.ReplaceWith(new RangeVar { Relname = "new_name", Inh = true, Relpersistence = "p" }), - // Drop a cast, keeping what it wraps - TypeCast cast => NodeEdit.ReplaceWith(cast.Arg), - // Remove an item from a list - ResTarget target when target.Deparse().Value == "secret" => NodeEdit.Remove, - _ => NodeEdit.Keep -}); - -Console.WriteLine(tree.Deparse().GetValueOrThrow()); -// Output: SELECT a FROM new_name WHERE kind = 'x' -``` - -`EqualsIgnoringLocations` compares two trees, or two nodes, by structure. Queries that differ only in whitespace, comments, keyword case or redundant parentheses are equal: - -```csharp -var a = Parser.Parse("SELECT a FROM t WHERE x <> 1").GetValueOrThrow(); -var b = Parser.Parse("select a\nfrom t -- note\nwhere (x != 1)").GetValueOrThrow(); - -Console.WriteLine(a.EqualsIgnoringLocations(b)); -// Output: True -``` - -#### Building nodes and common changes - -`Ast` builds nodes without writing out their fields. Each method gives the same node the parser would for the matching SQL, and `Ast.Expression` parses a piece of SQL when that is easier: - -```csharp -var condition = Ast.And( - Ast.Eq(Ast.Column("tenant_id"), Ast.Param(1)), - Ast.IsNull(Ast.Column("deleted_at"))); -// The same as Ast.Expression("tenant_id = $1 AND deleted_at IS NULL") -``` - -The usual changes to a statement are ready made. They work on the tree, so they hold however the SQL was written: - -```csharp -var tree = Parser.Parse("SELECT id, name FROM customers WHERE active ORDER BY name").GetValueOrThrow(); - -tree.AddWhere(condition); // combined with the existing WHERE using AND -tree.CapLimit(100); // adds LIMIT 100, or lowers a larger one -tree.QualifyTables("app"); // gives unqualified tables a schema, leaving CTE names alone - -Console.WriteLine(tree.Deparse().GetValueOrThrow()); -// Output: SELECT id, name FROM app.customers WHERE active AND tenant_id = $1 AND deleted_at IS NULL ORDER BY name LIMIT 100 - -Console.WriteLine(tree.ToCount().Deparse().GetValueOrThrow()); -// Output: SELECT count(*) FROM (SELECT id, name FROM app.customers WHERE active AND tenant_id = $1 AND deleted_at IS NULL ORDER BY name LIMIT 100) q -``` - -Also available: `SetLimit`, `SetOffset`, `RenameSchema`, `RenameTable`, and for DDL `ToDropStatement` (the DROP that undoes a CREATE), `EnsureIfNotExists` and `EnsureOrReplace`. - -#### Typed visitors - -A `NodeVisitor` holds a handler per node type. Several visitors can share one walk, and each skips or stops on its own: - -```csharp -var tables = new List(); -var functions = new List(); - -tree.Walk( - new NodeVisitor().On((table, visit) => tables.Add(table.Relname)), - new NodeVisitor().On((call, visit) => functions.Add(call.Funcname[^1].String.Sval))); -``` - -A few helpers go with these: `Parser.ParseExpression` and `Parser.ParseTypeName` parse a single expression or type name on its own, `node.AsNode()` wraps a node for a property or list that takes any node, and `PgIdentifier.Quote` quotes a name the way PostgreSQL's `quote_ident` does. - -`Walk` and `Descendants` visit parents before their children and siblings in field order, which is not always the order of the query text. `GetLocation()` returns a node's location if it has one; see [Offsets and non-ASCII text](#offsets-and-non-ascii-text) for its unit. Only statements record a length, so `GetText` is available for statements and not for other nodes. - -### Scan - -Tokenize a query. Each token has its kind, its keyword kind and its `Start` and `End` offsets in the query string. - -```csharp -using PgSqlParser; - -var query = "SELECT 1"; -var result = Parser.Scan(query); - -if (result.Error is null) -{ - foreach (var token in result.Value!.Tokens) - { - Console.WriteLine($"{query[token.Start..token.End]}: {token.Token}"); - } -} - -// result.Value is a ScanResult object and the serialized JSON output is as below -// { "version": 180006, "tokens": [ { "end": 6, "token": "SELECT", "keywordKind": "RESERVED_KEYWORD" }, { "start": 7, "end": 8, "token": "ICONST" } ] } -``` - -### Classify - -Get the facts about each statement: its kind, whether it is read-only, and anything that makes a harmless-looking statement write. This is the basis for read/write routing and for checking SQL before running it. - -```csharp -using PgSqlParser; - -foreach (var info in Parser.Classify("SELECT 1; WITH gone AS (DELETE FROM t RETURNING *) SELECT * FROM gone; EXPLAIN ANALYZE DELETE FROM t").GetValueOrThrow()) -{ - Console.WriteLine($"{info.Kind}: read-only {info.IsReadOnly}"); -} -// Output: -// Select: read-only True -// Select: read-only False (HasDataModifyingCte is true) -// Explain: read-only False (ExecutesInner is true, and Inner is a DELETE) -``` - -`StatementInfo` also has `IsUtility`, `HasSelectInto`, `HasLockingClause`, `RunsProgram` and `Inner` (the statement wrapped by EXPLAIN, PREPARE, DECLARE CURSOR, CREATE TABLE AS or COPY). `IsReadOnly` errs on the side of false: CALL, DO and EXECUTE are not read-only because what they run is not visible. A function called from a SELECT can still write, which no parser can see. These are facts, not a security boundary; what to allow is up to you. - -### References - -Find the tables, functions and columns a query refers to, as parse tree nodes with their role: - -```csharp -using PgSqlParser; - -var tree = Parser.Parse( - "WITH recent AS (SELECT * FROM orders) " + - "UPDATE customers c SET total = r.total FROM recent r WHERE r.customer_id = c.id").GetValueOrThrow(); -var references = tree.GetReferences(); - -foreach (var table in references.Tables) -{ - Console.WriteLine($"{table.Name}: {table.Role}{(table.IsCte ? " (a CTE, not a table)" : "")}"); -} -// Output: -// customers: Write -// recent: Read (a CTE, not a table) -// orders: Read - -foreach (var column in references.Columns) -{ - Console.WriteLine($"{column} -> {column.Table?.Name ?? "unknown"}"); -} -// Output: -// r.total -> unknown (recent is a CTE) -// r.customer_id -> unknown -// c.id -> customers -// * -> orders -``` - -A column is matched to its table when the query alone settles it: its qualifier names a table or alias in scope, or it has no qualifier and there is only one table in scope. Otherwise `Table` is null, because it would take the database catalog to tell. `Summary` returns similar information as plain names and is faster; use `GetReferences` when you need the nodes, aliases or column matching. - -Each column also has a `Clause` (select list, WHERE, join condition, GROUP BY, and so on), so the columns that filter rows are `references.Columns.Where(c => c.Clause is QueryClause.Where or QueryClause.JoinCondition)`. For a SELECT, `GetOutputColumns()` lists the result columns with the names PostgreSQL gives them: - -```csharp -var select = Parser.Parse("SELECT id, lower(name), total AS amount, 1 FROM t").GetValueOrThrow().Stmts[0].Stmt.SelectStmt; -Console.WriteLine(string.Join(", ", select.GetOutputColumns().Select(column => column.Name))); -// Output: id, lower, amount, ?column? -``` - -### Locks - -Find out which table locks a statement takes, and what they block. This follows PostgreSQL's documented lock rules, and for ALTER TABLE the same per-sub-command rules PostgreSQL itself uses. - -```csharp -using PgSqlParser; - -var script = "CREATE INDEX i ON orders (customer_id); ALTER TABLE orders ADD COLUMN note text; ALTER TABLE orders VALIDATE CONSTRAINT fk"; -foreach (var tableLock in Parser.Parse(script).GetValueOrThrow().GetLocks()) -{ - Console.WriteLine($"{tableLock} blocks reads: {tableLock.Mode.BlocksReads()}, blocks writes: {tableLock.Mode.BlocksWrites()}"); -} -// Output: -// orders: Share blocks reads: False, blocks writes: True -// orders: AccessExclusive blocks reads: True, blocks writes: True -// orders: ShareUpdateExclusive blocks reads: False, blocks writes: False -``` - -It reports locks on the tables a statement names. It cannot see locks on tables reached through a view, an index name, a partition or a trigger, since those need the database catalog, and it does not cover row-level locks. See the documentation of `GetLocks` for the list of statements covered. - -### Format - -Format a query across indented lines, keeping its comments. Each statement of a script is formatted on its own. - -```csharp -using PgSqlParser; - -var formatted = Parser.Format("select a, b from t -- pick\n where x = 1; delete from t where a = 1").GetValueOrThrow(); -Console.WriteLine(formatted); -// Output: -// SELECT a, b -// FROM t -// WHERE -// -- pick -// x = 1; -// -// DELETE FROM t -// WHERE a = 1; -``` - -`FormatOptions` sets the indent size, the line length, comma placement, a trailing newline, and whether comments are kept. - -### Tokenize - -Split a query into tokens classified for display, for syntax highlighting and similar uses. It works on SQL that does not parse, as long as it can be tokenized. - -```csharp -using PgSqlParser; - -var query = "SELECT name FROM users WHERE id = $1 -- by id"; -foreach (var token in Parser.Tokenize(query).GetValueOrThrow()) -{ - Console.WriteLine($"{query[token.Start..token.End]}: {token.Kind}"); -} -// Output: SELECT: Keyword, name: Keyword, FROM: Keyword, users: Identifier, WHERE: Keyword, id: Identifier, -// =: Operator, $1: Parameter, -- by id: Comment -``` - -`name` is a keyword to PostgreSQL, though an unreserved one; `token.KeywordKind` tells reserved keywords from those usable as names. - -### OperationSummary - -Describe a query in a few low-cardinality words: the operation and the tables it touches. Literals, parameters and columns are left out, so it is safe as a span name or a metric label. This is the form OpenTelemetry's `db.query.summary` attribute asks for. - -```csharp -using PgSqlParser; - -Console.WriteLine(Parser.OperationSummary("SELECT c.name FROM customers c JOIN orders o ON o.cid = c.id WHERE c.email = 'a@b.c'").GetValueOrThrow()); -// Output: SELECT customers orders - -Console.WriteLine(Parser.OperationSummary("INSERT INTO audit SELECT * FROM users").GetValueOrThrow()); -// Output: INSERT audit users -``` - -### ParameterRefs - -Find the parameter references (`$1`, `$2`, ...) in a query, with where each one is in the query string. References inside string literals and comments are not parameters and are left out. - -```csharp -using PgSqlParser; - -var query = "SELECT * FROM t WHERE a = $1 AND b = $2 AND note <> '$3'"; -var parameters = Parser.ParameterRefs(query).GetValueOrThrow(); - -foreach (var parameter in parameters) -{ - Console.WriteLine($"${parameter.Number} at {parameter.Start}..{parameter.End}"); -} -// Output: $1 at 26..28, $2 at 37..39 -``` - -When a parameter is cast in the query, `TypeName` holds the type: `$1::int` gives `int`, and `CAST($2 AS numeric(10,2))` gives `numeric(10, 2)`. It is null for a parameter that is not cast. - -### ParsePlpgsql - -Parse PL/pgSQL function bodies and return a JSON representation - -```csharp -using PgSqlParser; - -var sql = """ -CREATE OR REPLACE FUNCTION get_all_foo() RETURNS SETOF foo AS -$BODY$ -DECLARE -r foo%rowtype; -BEGIN -FOR r IN -SELECT * FROM foo WHERE fooid > 0 - LOOP - -- can do some processing here - RETURN NEXT r; -- return current row of SELECT -END LOOP; - RETURN; -END -$BODY$ -LANGUAGE plpgsql; -"""; - -var result = Parser.ParsePlpgsql(sql); - -if (result.Error is null) -{ - Console.WriteLine(result.Value); -} - -// return JSON string -// [{"PLpgSQL_function":{"datums":[{"PLpgSQL_var":{"refname":"found","datatype":{"PLpgSQL_type":{"typname":"bool"}}}},{"PLpgSQL_var":{"refname":"r","lineno":3,"datatype":{"PLpgSQL_type":{"typname":"foo%rowtype"}}}},{"PLpgSQL_row":{"refname":"(unnamed row)","lineno":5,"fields":[{"name":"r","varno":1}]}}],"action":{"PLpgSQL_stmt_block":{"lineno":4,"body":[{"PLpgSQL_stmt_fors":{"lineno":5,"var":{"PLpgSQL_row":{"refname":"(unnamed row)","lineno":5,"fields":[{"name":"r","varno":1}]}},"body":[{"PLpgSQL_stmt_return_next":{"lineno":9}}],"query":{"PLpgSQL_expr":{"query":"SELECT * FROM foo WHERE fooid \u003e 0","parseMode":0}}}},{"PLpgSQL_stmt_return":{"lineno":11}}]}}}}] -``` - -`ParsePlpgsqlFunctions` returns the same information as objects, one `PlpgsqlFunction` per `CREATE FUNCTION` or `DO` statement: - -```csharp -var function = Parser.ParsePlpgsqlFunctions(sql).GetValueOrThrow()[0]; - -// Variables and parameters -foreach (var datum in function.Datums) -{ - Console.WriteLine($"{datum.Kind}: {datum.GetString("refname")}"); -} - -// Every statement, including nested ones -foreach (var statement in function.Statements()) -{ - Console.WriteLine($"line {statement.LineNo}: {statement.Kind}"); -} - -// The SQL inside the function, ready to pass to Parser.Parse -foreach (var query in function.Queries()) -{ - Console.WriteLine(query); -} -// Output: SELECT * FROM foo WHERE fooid > 0 -``` - -`function.ParseQueries()` goes one step further and parses each of those pieces of SQL, in the mode PL/pgSQL itself uses for it, so a function body can be analysed with the same tools as any query: - -```csharp -var tables = function.ParseQueries() - .Where(query => query.Tree is not null) - .SelectMany(query => query.Tree!.GetReferences().Tables) - .Select(table => table.Name); -// foo -``` - -libpg_query has no schema for its PL/pgSQL output, so a node (`PlpgsqlNode`) exposes its `Kind`, its `Children` and its raw `Json` (a `JsonElement`) and not typed properties per statement kind. - -### Fingerprint - -Generate a normalized hash (fingerprint) of a SQL statement, ignoring literals, whitespace, and minor variations - -```csharp -using PgSqlParser; - -var query = "SELECT 1"; -var result = Parser.Fingerprint(query); - -if (result.Error is null) -{ - Console.WriteLine(result.Value); -} - -/// output: 50fde20626009aba -``` - -Fingerprints follow the PostgreSQL 18 query ID rules by default: in SELECT/DML statements a table alias replaces the relation name, and schema names are ignored. Pass `FingerprintOptions` to change this, for example to get the same fingerprints as PostgreSQL 17 and earlier: - -```csharp -var result = Parser.Fingerprint(query, ParserOptions.Default, FingerprintOptions.RangeVarPg17Compat); -``` - -### IsUtilityStmt - -Check whether each statement in a query is a utility statement (DDL and other commands that are not SELECT, INSERT, UPDATE, DELETE or MERGE) - -```csharp -using PgSqlParser; - -var query = "SELECT 1; SET fsync = off"; -var result = Parser.IsUtilityStmt(query); - -if (result.Error is null) -{ - Console.WriteLine(string.Join(", ", result.Value!)); -} - -// Output: False, True -``` - -### Summary - -Summarize a query: the tables, aliases, CTE names, functions, filter columns and statement types it references. Pass a `truncateLimit` to also get a shortened version of the query text. - -```csharp -using PgSqlParser; - -var query = "SELECT lower(x.name) FROM public.test AS x WHERE x.a = 1"; -var result = Parser.Summary(query); - -if (result.Error is null) -{ - Console.WriteLine(result.Value); -} - -// Return a `SummaryResult` and the respective JSON string is as below: -// { "tables": [ { "name": "public.test", "schemaName": "public", "tableName": "test", "context": "Select" } ], "aliases": { "x": "public.test" }, "functions": [ { "name": "lower", "functionName": "lower", "context": "Call" } ], "filterColumns": [ { "tableName": "x", "column": "a" } ], "statementTypes": [ "SelectStmt" ] } -``` +The package contains the native library for every platform above. To include only the one you run on, publish with a runtime identifier, for example `dotnet publish -r linux-x64`. See [Keeping deployments small](https://mysticmind.github.io/pgsqlparser-dotnet/guide/getting-started#keeping-deployments-small). -### SplitWithScanner +## Quick start -Split a SQL script containing multiple statements into an array of clean, standalone SQL statements using lexical (token-based) analysis. +Every function returns a `Result` that holds either a `Value` or an `Error`, and has an async version. ```csharp using PgSqlParser; -var query = "SELECT 1; SELECT 2"; -var result = Parser.SplitWithScanner(query); +// Parse SQL into the PostgreSQL parse tree +var tree = Parser.Parse("SELECT id, name FROM users WHERE id = 42").GetValueOrThrow(); +Console.WriteLine(tree.Stmts.Count); // 1 -if (result.Error is null) +// Find what a query touches +foreach (var table in tree.GetReferences().Tables) { - Console.WriteLine(JsonSerializer.Serialize(result.Value)); + Console.WriteLine($"{table.Name}: {table.Role}"); // users: Read } -// Return a `SplitResult` and the respective serialized JSON string is as below: -// {"Statements":[{"Location":0,"Length":8,"Text":"SELECT 1"},{"Location":9,"Length":9,"Text":" SELECT 2"}]} -``` - -### SplitWithParser - -Split a SQL script containing multiple statements into an array of clean, standalone SQL statements using Postgres full parser - -```csharp -using PgSqlParser; - -var query = "SELECT 1; SELECT 2"; -var result = Parser.SplitWithParser(query); - -if (result.Error is null) -{ - Console.WriteLine(JsonSerializer.Serialize(result.Value)); -} - -// Return a `SplitResult` and the respective serialized JSON string is as below: -// {"Statements":[{"Location":0,"Length":8,"Text":"SELECT 1"},{"Location":10,"Length":8,"Text":"SELECT 2"}]} -``` - -### Deparse - -Deparse AST back into a query string - -```csharp -using PgSqlParser; - -var parseJson = """ -{ "version": 180006, "stmts": [ { "stmt": { "SelectStmt": { "targetList": [ { "ResTarget": { "val": { "A_Const": { "ival": { "ival": 1 }, "location": 7 } }, "location": 7 } } ], "limitOption": "LIMIT_OPTION_DEFAULT", "op": "SETOP_NONE" } } } ] } -"""; - -var parseResult = ParseResult.Parser.ParseJson(parseJson); ; -var result = Parser.Deparse(parseResult); - -if (result.Error is null) -{ - Console.WriteLine(result.Value); -} - -// Output: SELECT 1 -``` - -Pass `DeparseOptions` to pretty print the output: - -```csharp -var parseResult = Parser.Parse("SELECT a, b FROM t WHERE x = 1 AND y = 2").Value!; -var result = Parser.Deparse(parseResult, new DeparseOptions { PrettyPrint = true }); - -// Output: +// Format a query +Console.WriteLine(Parser.Format("select a, b from t where x = 1").GetValueOrThrow()); // SELECT a, b // FROM t -// WHERE -// x = 1 -// AND y = 2 -``` +// WHERE x = 1 -Comments are not part of the parse tree. To keep them, extract them with `DeparseComments` and pass them back in: +// Fingerprint a query: the same query shape always gets the same id +Console.WriteLine(Parser.Fingerprint("SELECT 1").GetValueOrThrow()); // 50fde20626009aba -```csharp -var query = "SELECT 1 /* one */"; -var parseResult = Parser.Parse(query).Value!; -var comments = Parser.DeparseComments(query).Value!; -var result = Parser.Deparse(parseResult, new DeparseOptions { Comments = comments }); - -// Output: SELECT 1 /* one */ -``` - -### Offsets and non-ASCII text - -.NET strings are UTF-16 while libpg_query works on UTF-8 bytes, so the unit of an offset depends on where it comes from: - -- `Scan` token `Start`/`End` and `SplitStmt.Location`/`Length` are UTF-16 offsets, ready to use with `string.Substring`. -- Parse tree locations (for example `RawStmt.StmtLocation` and node `Location`) and `DeparseComment.MatchLocation` are UTF-8 byte offsets. Use `Utf8OffsetMapper` to convert them: - -```csharp -using PgSqlParser.Utils; - -var mapper = new Utf8OffsetMapper(query); -var charOffset = mapper.ToCharOffset(node.Location); -``` - -It also converts the other way, for example to compare a position in the string (such as an editor caret) against parse tree locations: - -```csharp -var byteOffset = mapper.ToByteOffset(caretIndex); -``` - -- `Error.CursorPos` is a 1-based position in Unicode code points, as PostgreSQL reports it. Use `Error.GetCursorCharOffset(query)` to get the matching string offset: - -```csharp -var result = Parser.Parse(query); -if (result.Error is { } error) -{ - var charOffset = error.GetCursorCharOffset(query); - if (charOffset >= 0) - { - Console.WriteLine($"{error.Message}: {query[charOffset..]}"); - } -} -``` - -### Parse Errors - -If the SQL has some error while parsing then the result contains an Error object which provides details of the error. - -```csharp -using PgSqlParser; - -var query = "SELEC 1"; -var result = Parser.Parse(query); -if (result.Error is not null) +// Handle a failure without throwing +var result = Parser.Parse("SELECT FROM WHERE"); +if (!result.IsSuccess) { Console.WriteLine(result.Error.Message); -} - -// Output: syntax error at or near "SELEC" -``` - -`error.Format(query)` renders the error the way psql does, with the line it points at, and `error.GetLineAndColumn(query)` gives the position: - -```csharp -var query = "SELECT a,\n b FROM WHERE x"; -var error = Parser.Parse(query).Error!; - -Console.WriteLine(error.Format(query)); -// Output: -// ERROR: syntax error at or near "WHERE" -// LINE 2: b FROM WHERE x -// ^ +} ``` -The `Error` also carries `CursorPos`, the position of the error in the query (see [Offsets and non-ASCII text](#offsets-and-non-ascii-text)), and the PostgreSQL source location that raised it in `FuncName`, `FileName` and `LineNo`. - -Two kinds of invalid input are caught before the query reaches libpg_query: - -- A `null` query throws `ArgumentNullException`. -- A query containing a NUL character (`\0`) returns an `Error` with the message `query contains a NUL character`. PostgreSQL does not allow NUL in queries. - -Deeply nested queries, such as a long chain of operators without parentheses (`a || b || c ...`) or many nested subqueries, are read on a dedicated thread with a large enough stack, so they cannot overflow the stack of the calling thread. A parse tree nested more than 4000 levels deep is rejected with an `Error`. The PostgreSQL parser has its own limit, which depends on the stack available to the calling thread and reports `stack depth limit exceeded`. - -Protobuf's own recursive operations on a parse tree, such as `ToString()`, `Clone()` and `Equals()`, run on your thread. On a very deeply nested tree they can still overflow a small stack. - -## What's new in 2.1 - -All additions; nothing from 2.0 changes. - -**Understanding a statement** - -- **Classify statements**: `Parser.Classify` gives each statement's kind, whether it is read-only, and facts such as a data-modifying CTE, `SELECT INTO`, a locking clause or `EXPLAIN ANALYZE`. -- **Resolved references**: `GetReferences` returns the tables, functions and columns of a query as nodes, with table roles (read, write, DDL), CTE detection and columns matched to their table. -- **Parameters**: `ParameterRefs` finds each `$n`, where it is, and the type it is cast to. -- **Query shape**: each column reference knows its clause (WHERE, join condition, GROUP BY, ...), and `GetOutputColumns` names a SELECT's result columns as PostgreSQL does. -- **Locks**: `GetLocks` reports the table locks a statement takes and what they block. -- **Inside PL/pgSQL**: `ParseQueries` parses the SQL in a function body. - -**Working with the tree** - -- **Deparse a single node**: `node.Deparse()` turns one clause, expression or table reference back into SQL. -- **More control when walking**: skip a subtree or stop, and each visit knows its ancestors, the property it came from and its top-level statement. -- **Typed visitors**: `NodeVisitor` with a handler per node type, and several visitors in one walk. -- **Compare trees**: `EqualsIgnoringLocations` treats queries that differ only in formatting as equal. - -**Changing a statement** - -- **Build nodes**: `Ast.Column`, `Ast.Const`, `Ast.Eq`, `Ast.And`, `Ast.Call`, `Ast.Table`, `Ast.Expression` and more, each giving the same node the parser would. -- **Rewrite**: `Rewrite` keeps, replaces or removes nodes in place. -- **Common changes ready made**: `AddWhere`, `SetLimit`, `CapLimit`, `SetOffset`, `ToCount`, `QualifyTables`, `RenameSchema`, `RenameTable`, `ToDropStatement`, `EnsureIfNotExists`, `EnsureOrReplace`. - -**Working with text** - -- **Format in one call**: `Parser.Format` pretty prints a query or a script and keeps its comments. -- **Readable errors**: `Error.Format` and `Error.GetLineAndColumn`. -- **Tokens for display**: `Parser.Tokenize` classifies each token as keyword, identifier, literal, comment and so on. -- **Telemetry summary**: `Parser.OperationSummary` gives the short form OpenTelemetry's `db.query.summary` asks for. -- **Parse fragments**: `ParseExpression` and `ParseTypeName`. -- **Identifier quoting**: `PgIdentifier.Quote`, following PostgreSQL's `quote_ident`. - -## What's new in 2.0 - -- **PostgreSQL 18**: built on libpg_query 18.1.0, which uses the PostgreSQL 18.6 parser and includes its security fix for `pg_query_normalize`. -- **More platforms**: Linux on ARM64, Alpine and other musl-based Linux, and Windows on ARM64. The Linux and macOS libraries are now built against old OS baselines, so they load on older systems too (see the table under [Installation](#installation)). -- **.NET 10**, alongside .NET 8 and .NET 9, and compatibility with trimming and Native AOT. -- **New functions**: `IsUtilityStmt`, `Summary`, `DeparseComments` and `ParsePlpgsqlFunctions`. -- **Deparse options**: pretty printing and keeping comments, through `DeparseOptions`. -- **Fingerprint options**: `FingerprintOptions`, including a PostgreSQL 17 compatible mode. -- **Easier results**: `GetValueOrThrow`, `TryGetValue`, `Match` and deconstruction on `Result` (see [Working with results](#working-with-results)). -- **Parse tree navigation**: `Descendants`, `Walk`, `Unwrap`, `GetLocation` and `GetText` (see [Navigating the parse tree](#navigating-the-parse-tree)). -- **Correct offsets for non-ASCII text**: `Scan` and split results index the query string directly, with `Utf8OffsetMapper` and `Error.GetCursorCharOffset` for the rest (see [Offsets and non-ASCII text](#offsets-and-non-ascii-text)). -- **Safer input handling**: a `null` query, a NUL character or a deeply nested query no longer crashes the process or is silently cut short. -- **Package**: IntelliSense documentation, a symbol package and Source Link are included. +## Documentation -## Upgrading from 1.x +The full guide is at [mysticmind.github.io/pgsqlparser-dotnet](https://mysticmind.github.io/pgsqlparser-dotnet/): -Version 2.0 moves from the PostgreSQL 17 parser to PostgreSQL 18, which changes some results: +| Topic | What it covers | +|---|---| +| [Getting started](https://mysticmind.github.io/pgsqlparser-dotnet/guide/getting-started) | Installation, platforms, deployment size and working with `Result` | +| [Parse trees](https://mysticmind.github.io/pgsqlparser-dotnet/guide/parse-tree) | `Parse`, walking, editing, comparing and building trees, typed visitors | +| [Deparse, format and normalize](https://mysticmind.github.io/pgsqlparser-dotnet/guide/deparse-format) | `Deparse`, `Format`, `Normalize`, `NormalizeUtility` and `Fingerprint` | +| [Query analysis](https://mysticmind.github.io/pgsqlparser-dotnet/guide/analysis) | `Classify`, references, locks, `Summary`, `OperationSummary`, `ParameterRefs` and `IsUtilityStmt` | +| [Scan, tokenize and split](https://mysticmind.github.io/pgsqlparser-dotnet/guide/scan-split) | `Scan`, `Tokenize`, `SplitWithScanner` and `SplitWithParser` | +| [PL/pgSQL](https://mysticmind.github.io/pgsqlparser-dotnet/guide/plpgsql) | Parsing function bodies and the SQL inside them | +| [Errors and offsets](https://mysticmind.github.io/pgsqlparser-dotnet/guide/errors-offsets) | Parse errors, cursor positions and non-ASCII text | +| [What's new](https://mysticmind.github.io/pgsqlparser-dotnet/guide/whats-new) | Changes in 2.1 and 2.0, and upgrading from 1.x | -- **Parse tree**: node classes and fields follow PostgreSQL 18, and `ParseResult.Version` is `180006`. -- **Fingerprints**: the default now follows PostgreSQL 18 query ID rules, so stored fingerprints for queries that use table aliases or schema names will change. Pass `FingerprintOptions.RangeVarPg17Compat` to keep the previous relation handling. -- **Statement locations**: a statement now starts at its first non-whitespace, non-comment character. This affects `SplitWithParser` and `RawStmt.StmtLocation`; for `SELECT 1; SELECT 2` the second statement is at location 10 with length 8, where it was 9 and 9. -- **Scan and Split offsets**: `Scan` token offsets and `SplitStmt` locations are UTF-16 offsets into the query string. In 1.x they were UTF-8 byte offsets, which differ as soon as the query contains non-ASCII text. Remove any conversion you did yourself. -- **Errors**: `Error.FuncName` and `Error.FileName` now hold the function and file name. In 1.x `FuncName` held the message and the file name was missing. -- **`Parse` return type**: `Parse` and `ParseAsync` return `Result`, where 1.x returned `Result`. Code that declared the nullable type explicitly needs updating. -- **Dependencies**: the minimum `Google.Protobuf` version is 3.36.2. +To work on the documentation, see the `docs` folder: `npm install` and `npm run dev` start it locally. ## License diff --git a/docs/.vitepress/config.mts b/docs/.vitepress/config.mts index 2bc5b8e..8a7c8f0 100644 --- a/docs/.vitepress/config.mts +++ b/docs/.vitepress/config.mts @@ -46,6 +46,10 @@ export default defineConfig({ ], outline: { level: [2, 3] }, search: { provider: 'local' }, + editLink: { + pattern: 'https://github.com/mysticmind/pgsqlparser-dotnet/edit/main/docs/:path', + text: 'Edit this page on GitHub', + }, socialLinks: [{ icon: 'github', link: 'https://github.com/mysticmind/pgsqlparser-dotnet' }], footer: { message: 'Released under the MIT License.', diff --git a/docs/guide/analysis.md b/docs/guide/analysis.md new file mode 100644 index 0000000..a3dfc53 --- /dev/null +++ b/docs/guide/analysis.md @@ -0,0 +1,153 @@ +# Query analysis + +## Classify + +Get the facts about each statement: its kind, whether it is read-only, and anything that makes a harmless-looking statement write. This is the basis for read/write routing and for checking SQL before running it. + +```csharp +using PgSqlParser; + +foreach (var info in Parser.Classify("SELECT 1; WITH gone AS (DELETE FROM t RETURNING *) SELECT * FROM gone; EXPLAIN ANALYZE DELETE FROM t").GetValueOrThrow()) +{ + Console.WriteLine($"{info.Kind}: read-only {info.IsReadOnly}"); +} +// Output: +// Select: read-only True +// Select: read-only False (HasDataModifyingCte is true) +// Explain: read-only False (ExecutesInner is true, and Inner is a DELETE) +``` + +`StatementInfo` also has `IsUtility`, `HasSelectInto`, `HasLockingClause`, `RunsProgram` and `Inner` (the statement wrapped by EXPLAIN, PREPARE, DECLARE CURSOR, CREATE TABLE AS or COPY). `IsReadOnly` errs on the side of false: CALL, DO and EXECUTE are not read-only because what they run is not visible. A function called from a SELECT can still write, which no parser can see. These are facts, not a security boundary; what to allow is up to you. + +## References + +Find the tables, functions and columns a query refers to, as parse tree nodes with their role: + +```csharp +using PgSqlParser; + +var tree = Parser.Parse( + "WITH recent AS (SELECT * FROM orders) " + + "UPDATE customers c SET total = r.total FROM recent r WHERE r.customer_id = c.id").GetValueOrThrow(); +var references = tree.GetReferences(); + +foreach (var table in references.Tables) +{ + Console.WriteLine($"{table.Name}: {table.Role}{(table.IsCte ? " (a CTE, not a table)" : "")}"); +} +// Output: +// customers: Write +// recent: Read (a CTE, not a table) +// orders: Read + +foreach (var column in references.Columns) +{ + Console.WriteLine($"{column} -> {column.Table?.Name ?? "unknown"}"); +} +// Output: +// r.total -> unknown (recent is a CTE) +// r.customer_id -> unknown +// c.id -> customers +// * -> orders +``` + +A column is matched to its table when the query alone settles it: its qualifier names a table or alias in scope, or it has no qualifier and there is only one table in scope. Otherwise `Table` is null, because it would take the database catalog to tell. `Summary` returns similar information as plain names and is faster; use `GetReferences` when you need the nodes, aliases or column matching. + +Each column also has a `Clause` (select list, WHERE, join condition, GROUP BY, and so on), so the columns that filter rows are `references.Columns.Where(c => c.Clause is QueryClause.Where or QueryClause.JoinCondition)`. For a SELECT, `GetOutputColumns()` lists the result columns with the names PostgreSQL gives them: + +```csharp +var select = Parser.Parse("SELECT id, lower(name), total AS amount, 1 FROM t").GetValueOrThrow().Stmts[0].Stmt.SelectStmt; +Console.WriteLine(string.Join(", ", select.GetOutputColumns().Select(column => column.Name))); +// Output: id, lower, amount, ?column? +``` + +## Locks + +Find out which table locks a statement takes, and what they block. This follows PostgreSQL's documented lock rules, and for ALTER TABLE the same per-sub-command rules PostgreSQL itself uses. + +```csharp +using PgSqlParser; + +var script = "CREATE INDEX i ON orders (customer_id); ALTER TABLE orders ADD COLUMN note text; ALTER TABLE orders VALIDATE CONSTRAINT fk"; +foreach (var tableLock in Parser.Parse(script).GetValueOrThrow().GetLocks()) +{ + Console.WriteLine($"{tableLock} blocks reads: {tableLock.Mode.BlocksReads()}, blocks writes: {tableLock.Mode.BlocksWrites()}"); +} +// Output: +// orders: Share blocks reads: False, blocks writes: True +// orders: AccessExclusive blocks reads: True, blocks writes: True +// orders: ShareUpdateExclusive blocks reads: False, blocks writes: False +``` + +It reports locks on the tables a statement names. It cannot see locks on tables reached through a view, an index name, a partition or a trigger, since those need the database catalog, and it does not cover row-level locks. See the documentation of `GetLocks` for the list of statements covered. + +## Summary + +Summarize a query: the tables, aliases, CTE names, functions, filter columns and statement types it references. Pass a `truncateLimit` to also get a shortened version of the query text. + +```csharp +using PgSqlParser; + +var query = "SELECT lower(x.name) FROM public.test AS x WHERE x.a = 1"; +var result = Parser.Summary(query); + +if (result.Error is null) +{ + Console.WriteLine(result.Value); +} + +// Return a `SummaryResult` and the respective JSON string is as below: +// { "tables": [ { "name": "public.test", "schemaName": "public", "tableName": "test", "context": "Select" } ], "aliases": { "x": "public.test" }, "functions": [ { "name": "lower", "functionName": "lower", "context": "Call" } ], "filterColumns": [ { "tableName": "x", "column": "a" } ], "statementTypes": [ "SelectStmt" ] } +``` + +## OperationSummary + +Describe a query in a few low-cardinality words: the operation and the tables it touches. Literals, parameters and columns are left out, so it is safe as a span name or a metric label. This is the form OpenTelemetry's `db.query.summary` attribute asks for. + +```csharp +using PgSqlParser; + +Console.WriteLine(Parser.OperationSummary("SELECT c.name FROM customers c JOIN orders o ON o.cid = c.id WHERE c.email = 'a@b.c'").GetValueOrThrow()); +// Output: SELECT customers orders + +Console.WriteLine(Parser.OperationSummary("INSERT INTO audit SELECT * FROM users").GetValueOrThrow()); +// Output: INSERT audit users +``` + +## ParameterRefs + +Find the parameter references (`$1`, `$2`, ...) in a query, with where each one is in the query string. References inside string literals and comments are not parameters and are left out. + +```csharp +using PgSqlParser; + +var query = "SELECT * FROM t WHERE a = $1 AND b = $2 AND note <> '$3'"; +var parameters = Parser.ParameterRefs(query).GetValueOrThrow(); + +foreach (var parameter in parameters) +{ + Console.WriteLine($"${parameter.Number} at {parameter.Start}..{parameter.End}"); +} +// Output: $1 at 26..28, $2 at 37..39 +``` + +When a parameter is cast in the query, `TypeName` holds the type: `$1::int` gives `int`, and `CAST($2 AS numeric(10,2))` gives `numeric(10, 2)`. It is null for a parameter that is not cast. + +## IsUtilityStmt + +Check whether each statement in a query is a utility statement (DDL and other commands that are not SELECT, INSERT, UPDATE, DELETE or MERGE) + +```csharp +using PgSqlParser; + +var query = "SELECT 1; SET fsync = off"; +var result = Parser.IsUtilityStmt(query); + +if (result.Error is null) +{ + Console.WriteLine(string.Join(", ", result.Value!)); +} + +// Output: False, True +``` + diff --git a/docs/guide/deparse-format.md b/docs/guide/deparse-format.md new file mode 100644 index 0000000..1c5f33d --- /dev/null +++ b/docs/guide/deparse-format.md @@ -0,0 +1,132 @@ +# Deparse, format and normalize + +## Deparse + +Deparse AST back into a query string + +```csharp +using PgSqlParser; + +var parseJson = """ +{ "version": 180006, "stmts": [ { "stmt": { "SelectStmt": { "targetList": [ { "ResTarget": { "val": { "A_Const": { "ival": { "ival": 1 }, "location": 7 } }, "location": 7 } } ], "limitOption": "LIMIT_OPTION_DEFAULT", "op": "SETOP_NONE" } } } ] } +"""; + +var parseResult = ParseResult.Parser.ParseJson(parseJson); ; +var result = Parser.Deparse(parseResult); + +if (result.Error is null) +{ + Console.WriteLine(result.Value); +} + +// Output: SELECT 1 +``` + +Pass `DeparseOptions` to pretty print the output: + +```csharp +var parseResult = Parser.Parse("SELECT a, b FROM t WHERE x = 1 AND y = 2").Value!; +var result = Parser.Deparse(parseResult, new DeparseOptions { PrettyPrint = true }); + +// Output: +// SELECT a, b +// FROM t +// WHERE +// x = 1 +// AND y = 2 +``` + +Comments are not part of the parse tree. To keep them, extract them with `DeparseComments` and pass them back in: + +```csharp +var query = "SELECT 1 /* one */"; +var parseResult = Parser.Parse(query).Value!; +var comments = Parser.DeparseComments(query).Value!; +var result = Parser.Deparse(parseResult, new DeparseOptions { Comments = comments }); + +// Output: SELECT 1 /* one */ +``` + +## Format + +Format a query across indented lines, keeping its comments. Each statement of a script is formatted on its own. + +```csharp +using PgSqlParser; + +var formatted = Parser.Format("select a, b from t -- pick\n where x = 1; delete from t where a = 1").GetValueOrThrow(); +Console.WriteLine(formatted); +// Output: +// SELECT a, b +// FROM t +// WHERE +// -- pick +// x = 1; +// +// DELETE FROM t +// WHERE a = 1; +``` + +`FormatOptions` sets the indent size, the line length, comma placement, a trailing newline, and whether comments are kept. + +## Normalize + +Transform DML query (SELECT, INSERT, UPDATE, DELETE) into a canonical form by replacing literal values (constants) with placeholders ($1, $2) + +```csharp +using PgSqlParser; + +var query = "SELECT 1"; +var result = Parser.Normalize(query); + +if (result.Error is null) +{ + Console.WriteLine(result.Value); +} + +// Normalized value: SELECT $1 + +``` + +## NormalizeUtility + +Transform DDL and other utility commands (CREATE TABLE, ALTER TABLE, VACUUM, and ANALYZE et.al.) into a canonical form by replacing literal values (constants) with placeholders ($1, $2) + +```csharp +using PgSqlParser; + +var query = "CREATE ROLE postgres PASSWORD 'xyz'"; +var result = Parser.NormalizeUtility(query); + +if (result.Error is null) +{ + Console.WriteLine(result.Value); +} + +// Normalized Utility value: CREATE ROLE postgres PASSWORD $1 +``` + +## Fingerprint + +Generate a normalized hash (fingerprint) of a SQL statement, ignoring literals, whitespace, and minor variations + +```csharp +using PgSqlParser; + +var query = "SELECT 1"; +var result = Parser.Fingerprint(query); + +if (result.Error is null) +{ + Console.WriteLine(result.Value); +} + +/// output: 50fde20626009aba +``` + +Fingerprints follow the PostgreSQL 18 query ID rules by default: in SELECT/DML statements a table alias replaces the relation name, and schema names are ignored. Pass `FingerprintOptions` to change this, for example to get the same fingerprints as PostgreSQL 17 and earlier: + +```csharp +var result = Parser.Fingerprint(query, ParserOptions.Default, FingerprintOptions.RangeVarPg17Compat); +``` + diff --git a/docs/guide/errors-offsets.md b/docs/guide/errors-offsets.md new file mode 100644 index 0000000..f9ca9e5 --- /dev/null +++ b/docs/guide/errors-offsets.md @@ -0,0 +1,77 @@ +# Errors and offsets + +## Parse Errors + +If the SQL has some error while parsing then the result contains an Error object which provides details of the error. + +```csharp +using PgSqlParser; + +var query = "SELEC 1"; +var result = Parser.Parse(query); +if (result.Error is not null) +{ + Console.WriteLine(result.Error.Message); +} + +// Output: syntax error at or near "SELEC" +``` + +`error.Format(query)` renders the error the way psql does, with the line it points at, and `error.GetLineAndColumn(query)` gives the position: + +```csharp +var query = "SELECT a,\n b FROM WHERE x"; +var error = Parser.Parse(query).Error!; + +Console.WriteLine(error.Format(query)); +// Output: +// ERROR: syntax error at or near "WHERE" +// LINE 2: b FROM WHERE x +// ^ +``` + +The `Error` also carries `CursorPos`, the position of the error in the query (see [Offsets and non-ASCII text](/guide/errors-offsets#offsets-and-non-ascii-text)), and the PostgreSQL source location that raised it in `FuncName`, `FileName` and `LineNo`. + +Two kinds of invalid input are caught before the query reaches libpg_query: + +- A `null` query throws `ArgumentNullException`. +- A query containing a NUL character (`\0`) returns an `Error` with the message `query contains a NUL character`. PostgreSQL does not allow NUL in queries. + +Deeply nested queries, such as a long chain of operators without parentheses (`a || b || c ...`) or many nested subqueries, are read on a dedicated thread with a large enough stack, so they cannot overflow the stack of the calling thread. A parse tree nested more than 4000 levels deep is rejected with an `Error`. The PostgreSQL parser has its own limit, which depends on the stack available to the calling thread and reports `stack depth limit exceeded`. + +Protobuf's own recursive operations on a parse tree, such as `ToString()`, `Clone()` and `Equals()`, run on your thread. On a very deeply nested tree they can still overflow a small stack. + +## Offsets and non-ASCII text + +.NET strings are UTF-16 while libpg_query works on UTF-8 bytes, so the unit of an offset depends on where it comes from: + +- `Scan` token `Start`/`End` and `SplitStmt.Location`/`Length` are UTF-16 offsets, ready to use with `string.Substring`. +- Parse tree locations (for example `RawStmt.StmtLocation` and node `Location`) and `DeparseComment.MatchLocation` are UTF-8 byte offsets. Use `Utf8OffsetMapper` to convert them: + +```csharp +using PgSqlParser.Utils; + +var mapper = new Utf8OffsetMapper(query); +var charOffset = mapper.ToCharOffset(node.Location); +``` + +It also converts the other way, for example to compare a position in the string (such as an editor caret) against parse tree locations: + +```csharp +var byteOffset = mapper.ToByteOffset(caretIndex); +``` + +- `Error.CursorPos` is a 1-based position in Unicode code points, as PostgreSQL reports it. Use `Error.GetCursorCharOffset(query)` to get the matching string offset: + +```csharp +var result = Parser.Parse(query); +if (result.Error is { } error) +{ + var charOffset = error.GetCursorCharOffset(query); + if (charOffset >= 0) + { + Console.WriteLine($"{error.Message}: {query[charOffset..]}"); + } +} +``` + diff --git a/docs/guide/getting-started.md b/docs/guide/getting-started.md new file mode 100644 index 0000000..651d9ed --- /dev/null +++ b/docs/guide/getting-started.md @@ -0,0 +1,70 @@ +# Getting started + +## Installation + +```shell +dotnet add package pgsqlparser +``` + +Note that the libpg_query libs for all OS'es are already packaged with the assembly. + +| Platform | Runtime identifiers | Minimum version | +|---|---|---| +| Linux (glibc) | `linux-x64`, `linux-arm64` | glibc 2.17 | +| Linux (musl, for example Alpine) | `linux-musl-x64`, `linux-musl-arm64` | Alpine 3.17 | +| macOS | `osx-x64`, `osx-arm64` | macOS 10.15 (Intel), macOS 11.0 (Apple silicon) | +| Windows | `win-x64`, `win-arm64` | | + +This version is built on libpg_query 18.1.0, which uses the PostgreSQL 18.6 parser. It targets .NET 8, .NET 9 and .NET 10, and is compatible with trimming and Native AOT. + +### Keeping deployments small + +The package contains the native library for every platform above, which is about 30 MB once unpacked. A portable publish copies all of them into your output. When deployment size matters, for example in a container image or a serverless function, publish for the platform you run on and only that platform's library is included: + +```shell +dotnet publish -r linux-x64 +``` + +| Publish command | Output size | Native libraries included | +|---|---|---| +| `dotnet publish` | about 31 MB | all 8 | +| `dotnet publish -r linux-x64` | about 5 MB | 1 | + +The sizes are for a small framework-dependent app. The runtime identifier must match where the app runs: use `linux-musl-x64` on Alpine, and the `arm64` variants on ARM64. + +## Usage + +All functions support both sync and async versions. + +## Working with results + +Every function returns a `Result` that holds either a `Value` or an `Error`. Pick the style that suits your code: + +```csharp +using PgSqlParser; + +// Check IsSuccess: the compiler then knows Value (or Error) is not null +var result = Parser.Parse(query); +if (result.IsSuccess) +{ + Console.WriteLine(result.Value.Stmts.Count); +} +else +{ + Console.WriteLine(result.Error.Message); +} + +// Throw on failure: PgSqlParserException carries the Error +var tree = Parser.Parse(query).GetValueOrThrow(); + +// Try pattern +if (Parser.Parse(query).TryGetValue(out var parsed, out var error)) +{ + Console.WriteLine(parsed.Stmts.Count); +} + +// Match and deconstruction +var text = Parser.Normalize(query).Match(value => value, e => e.Message); +var (normalized, normalizeError) = Parser.Normalize(query); +``` + diff --git a/docs/guide/license.md b/docs/guide/license.md new file mode 100644 index 0000000..e9941dd --- /dev/null +++ b/docs/guide/license.md @@ -0,0 +1,13 @@ +# License + +## License + +PgSqlParser is licensed under the [MIT License](https://github.com/mysticmind/pgsqlparser-dotnet/blob/main/LICENSE). + +Copyright (c) 2026, Babu Annamalai + +Refer to [libpg_query license](https://github.com/pganalyze/libpg_query?tab=readme-ov-file#license) for license details on libpg_query. + +This project includes code derived from the [PostgreSQL project](http://www.postgresql.org/), +see [LICENSE.POSTGRESQL](https://github.com/mysticmind/pgsqlparser-dotnet/blob/main/LICENSE.POSTGRESQL) for details. + diff --git a/docs/guide/parse-tree.md b/docs/guide/parse-tree.md new file mode 100644 index 0000000..b2979d5 --- /dev/null +++ b/docs/guide/parse-tree.md @@ -0,0 +1,200 @@ +# Parse trees + +## Parse + +Parse SQL and return a Protobuf based AST + +```csharp +using PgSqlParser; + +var query = "SELECT 1; SELECT 2"; + +var result = Parser.Parse(query); + +if (result.Error is null) +{ + Console.WriteLine(result.Value); +} + +// result.Value is a ParseResult AST object and the serialized JSON output is as below +// { "version": 180006, "stmts": [ { "stmt": { "SelectStmt": { "targetList": [ { "ResTarget": { "val": { "A_Const": { "ival": { "ival": 1 }, "location": 7 } }, "location": 7 } } ], "limitOption": "LIMIT_OPTION_DEFAULT", "op": "SETOP_NONE" } }, "stmt_len": 8 }, { "stmt": { "SelectStmt": { "targetList": [ { "ResTarget": { "val": { "A_Const": { "ival": { "ival": 2 }, "location": 17 } }, "location": 17 } } ], "limitOption": "LIMIT_OPTION_DEFAULT", "op": "SETOP_NONE" } }, "stmt_location": 10 } ] } +``` + +`Parse` can also take `ParserOptions` as a list of flags i.e. `ParserOptions.DisableBackslashQuote | ParserOptions.DisableEscapeStringWarning` + +## Navigating the parse tree + +The parse tree has more than 270 node types, and reaching a node by hand means spelling out the whole path to it. These helpers avoid that: + +```csharp +using PgSqlParser; + +var query = "SELECT c.name FROM customers c JOIN orders o ON o.customer_id = c.id; DROP TABLE tmp"; +var tree = Parser.Parse(query).GetValueOrThrow(); + +// Every node of a type, anywhere in the tree +foreach (var table in tree.Descendants()) +{ + Console.WriteLine(table.Relname); +} +// Output: customers, orders (DROP TABLE names its target as a plain name list, not a RangeVar) + +// Every node, with its parent and depth +foreach (var visit in tree.Walk()) +{ + Console.WriteLine($"{new string(' ', visit.Depth * 2)}{visit.Node.Descriptor.Name}"); +} + +// Pattern match on a statement without the .SelectStmt / .DropStmt property chain +foreach (var stmt in tree.Stmts) +{ + var kind = stmt.Stmt.Unwrap() switch + { + SelectStmt => "select", + DropStmt => "drop", + _ => "other" + }; + // The statement's text and kind + Console.WriteLine($"{kind}: {stmt.GetText(query)}"); +} +// Output: +// select: SELECT c.name FROM customers c JOIN orders o ON o.customer_id = c.id +// drop: DROP TABLE tmp +``` + +Any node can be turned back into SQL on its own, which is how to get the text of a single clause or expression: + +```csharp +var select = tree.Stmts[0].Stmt.SelectStmt; + +Console.WriteLine(select.FromClause[0].Deparse().GetValueOrThrow()); +// Output: customers c JOIN orders o ON o.customer_id = c.id + +foreach (var table in tree.Descendants()) +{ + Console.WriteLine(table.Deparse().GetValueOrThrow()); +} +// Output: customers c, orders o +``` + +`Deparse()` on a node works for statements, expressions, items of a FROM clause, select list and ORDER BY items, WITH clauses and type names. Nodes that are not SQL on their own, such as an `Alias`, return an error. + +To control the walk, pass a function. It can skip everything below a node or stop the walk, and each visit tells you which property of its parent the node came from: + +```csharp +// Tables used by the query itself, ignoring subqueries inside expressions +var tables = new List(); +tree.Walk(visit => +{ + if (visit.Node is SubLink) + return WalkAction.SkipChildren; + + if (visit.Node is RangeVar table) + tables.Add(table.Relname); + + return WalkAction.Continue; // or WalkAction.Stop to end the walk +}); + +// visit.FieldName is the parent's property, for example nameof(SelectStmt.WhereClause); +// visit.Index is the position when that property is a list. +``` + +Each visit also knows where it sits in the tree, which is what most rules need: + +```csharp +foreach (var visit in tree.Walk()) +{ + if (visit.Node is not RangeVar table) + continue; + + // The nearest containing node of a type, or null + var insideCte = visit.FindAncestor() is not null; + var insideSubquery = visit.FindAncestor() is not null; + + // The top-level statement the node belongs to + var writes = visit.Statement?.Stmt.Unwrap() is InsertStmt or UpdateStmt or DeleteStmt; + + Console.WriteLine($"statement {visit.StatementIndex}: {table.Relname} (cte: {insideCte}, subquery: {insideSubquery}, writes: {writes})"); +} +``` + +`visit.Ancestors` lists every containing node, nearest first. + +### Changing and comparing trees + +`Rewrite` walks the tree and lets you keep, replace or remove each node. It changes the tree in place, visiting children before their parents, and the result can be deparsed: + +```csharp +var tree = Parser.Parse("SELECT a, secret FROM old_name WHERE kind = 'x'::text").GetValueOrThrow(); + +tree.Rewrite(visit => visit.Node switch +{ + // Point the query at another table + RangeVar { Relname: "old_name" } => NodeEdit.ReplaceWith(new RangeVar { Relname = "new_name", Inh = true, Relpersistence = "p" }), + // Drop a cast, keeping what it wraps + TypeCast cast => NodeEdit.ReplaceWith(cast.Arg), + // Remove an item from a list + ResTarget target when target.Deparse().Value == "secret" => NodeEdit.Remove, + _ => NodeEdit.Keep +}); + +Console.WriteLine(tree.Deparse().GetValueOrThrow()); +// Output: SELECT a FROM new_name WHERE kind = 'x' +``` + +`EqualsIgnoringLocations` compares two trees, or two nodes, by structure. Queries that differ only in whitespace, comments, keyword case or redundant parentheses are equal: + +```csharp +var a = Parser.Parse("SELECT a FROM t WHERE x <> 1").GetValueOrThrow(); +var b = Parser.Parse("select a\nfrom t -- note\nwhere (x != 1)").GetValueOrThrow(); + +Console.WriteLine(a.EqualsIgnoringLocations(b)); +// Output: True +``` + +### Building nodes and common changes + +`Ast` builds nodes without writing out their fields. Each method gives the same node the parser would for the matching SQL, and `Ast.Expression` parses a piece of SQL when that is easier: + +```csharp +var condition = Ast.And( + Ast.Eq(Ast.Column("tenant_id"), Ast.Param(1)), + Ast.IsNull(Ast.Column("deleted_at"))); +// The same as Ast.Expression("tenant_id = $1 AND deleted_at IS NULL") +``` + +The usual changes to a statement are ready made. They work on the tree, so they hold however the SQL was written: + +```csharp +var tree = Parser.Parse("SELECT id, name FROM customers WHERE active ORDER BY name").GetValueOrThrow(); + +tree.AddWhere(condition); // combined with the existing WHERE using AND +tree.CapLimit(100); // adds LIMIT 100, or lowers a larger one +tree.QualifyTables("app"); // gives unqualified tables a schema, leaving CTE names alone + +Console.WriteLine(tree.Deparse().GetValueOrThrow()); +// Output: SELECT id, name FROM app.customers WHERE active AND tenant_id = $1 AND deleted_at IS NULL ORDER BY name LIMIT 100 + +Console.WriteLine(tree.ToCount().Deparse().GetValueOrThrow()); +// Output: SELECT count(*) FROM (SELECT id, name FROM app.customers WHERE active AND tenant_id = $1 AND deleted_at IS NULL ORDER BY name LIMIT 100) q +``` + +Also available: `SetLimit`, `SetOffset`, `RenameSchema`, `RenameTable`, and for DDL `ToDropStatement` (the DROP that undoes a CREATE), `EnsureIfNotExists` and `EnsureOrReplace`. + +### Typed visitors + +A `NodeVisitor` holds a handler per node type. Several visitors can share one walk, and each skips or stops on its own: + +```csharp +var tables = new List(); +var functions = new List(); + +tree.Walk( + new NodeVisitor().On((table, visit) => tables.Add(table.Relname)), + new NodeVisitor().On((call, visit) => functions.Add(call.Funcname[^1].String.Sval))); +``` + +A few helpers go with these: `Parser.ParseExpression` and `Parser.ParseTypeName` parse a single expression or type name on its own, `node.AsNode()` wraps a node for a property or list that takes any node, and `PgIdentifier.Quote` quotes a name the way PostgreSQL's `quote_ident` does. + +`Walk` and `Descendants` visit parents before their children and siblings in field order, which is not always the order of the query text. `GetLocation()` returns a node's location if it has one; see [Offsets and non-ASCII text](/guide/errors-offsets#offsets-and-non-ascii-text) for its unit. Only statements record a length, so `GetText` is available for statements and not for other nodes. + diff --git a/docs/guide/plpgsql.md b/docs/guide/plpgsql.md new file mode 100644 index 0000000..d7867bd --- /dev/null +++ b/docs/guide/plpgsql.md @@ -0,0 +1,75 @@ +# PL/pgSQL + +## ParsePlpgsql + +Parse PL/pgSQL function bodies and return a JSON representation + +```csharp +using PgSqlParser; + +var sql = """ +CREATE OR REPLACE FUNCTION get_all_foo() RETURNS SETOF foo AS +$BODY$ +DECLARE +r foo%rowtype; +BEGIN +FOR r IN +SELECT * FROM foo WHERE fooid > 0 + LOOP + -- can do some processing here + RETURN NEXT r; -- return current row of SELECT +END LOOP; + RETURN; +END +$BODY$ +LANGUAGE plpgsql; +"""; + +var result = Parser.ParsePlpgsql(sql); + +if (result.Error is null) +{ + Console.WriteLine(result.Value); +} + +// return JSON string +// [{"PLpgSQL_function":{"datums":[{"PLpgSQL_var":{"refname":"found","datatype":{"PLpgSQL_type":{"typname":"bool"}}}},{"PLpgSQL_var":{"refname":"r","lineno":3,"datatype":{"PLpgSQL_type":{"typname":"foo%rowtype"}}}},{"PLpgSQL_row":{"refname":"(unnamed row)","lineno":5,"fields":[{"name":"r","varno":1}]}}],"action":{"PLpgSQL_stmt_block":{"lineno":4,"body":[{"PLpgSQL_stmt_fors":{"lineno":5,"var":{"PLpgSQL_row":{"refname":"(unnamed row)","lineno":5,"fields":[{"name":"r","varno":1}]}},"body":[{"PLpgSQL_stmt_return_next":{"lineno":9}}],"query":{"PLpgSQL_expr":{"query":"SELECT * FROM foo WHERE fooid \u003e 0","parseMode":0}}}},{"PLpgSQL_stmt_return":{"lineno":11}}]}}}}] +``` + +`ParsePlpgsqlFunctions` returns the same information as objects, one `PlpgsqlFunction` per `CREATE FUNCTION` or `DO` statement: + +```csharp +var function = Parser.ParsePlpgsqlFunctions(sql).GetValueOrThrow()[0]; + +// Variables and parameters +foreach (var datum in function.Datums) +{ + Console.WriteLine($"{datum.Kind}: {datum.GetString("refname")}"); +} + +// Every statement, including nested ones +foreach (var statement in function.Statements()) +{ + Console.WriteLine($"line {statement.LineNo}: {statement.Kind}"); +} + +// The SQL inside the function, ready to pass to Parser.Parse +foreach (var query in function.Queries()) +{ + Console.WriteLine(query); +} +// Output: SELECT * FROM foo WHERE fooid > 0 +``` + +`function.ParseQueries()` goes one step further and parses each of those pieces of SQL, in the mode PL/pgSQL itself uses for it, so a function body can be analysed with the same tools as any query: + +```csharp +var tables = function.ParseQueries() + .Where(query => query.Tree is not null) + .SelectMany(query => query.Tree!.GetReferences().Tables) + .Select(table => table.Name); +// foo +``` + +libpg_query has no schema for its PL/pgSQL output, so a node (`PlpgsqlNode`) exposes its `Kind`, its `Children` and its raw `Json` (a `JsonElement`) and not typed properties per statement kind. + diff --git a/docs/guide/scan-split.md b/docs/guide/scan-split.md new file mode 100644 index 0000000..0eef387 --- /dev/null +++ b/docs/guide/scan-split.md @@ -0,0 +1,80 @@ +# Scan, tokenize and split + +## Scan + +Tokenize a query. Each token has its kind, its keyword kind and its `Start` and `End` offsets in the query string. + +```csharp +using PgSqlParser; + +var query = "SELECT 1"; +var result = Parser.Scan(query); + +if (result.Error is null) +{ + foreach (var token in result.Value!.Tokens) + { + Console.WriteLine($"{query[token.Start..token.End]}: {token.Token}"); + } +} + +// result.Value is a ScanResult object and the serialized JSON output is as below +// { "version": 180006, "tokens": [ { "end": 6, "token": "SELECT", "keywordKind": "RESERVED_KEYWORD" }, { "start": 7, "end": 8, "token": "ICONST" } ] } +``` + +## Tokenize + +Split a query into tokens classified for display, for syntax highlighting and similar uses. It works on SQL that does not parse, as long as it can be tokenized. + +```csharp +using PgSqlParser; + +var query = "SELECT name FROM users WHERE id = $1 -- by id"; +foreach (var token in Parser.Tokenize(query).GetValueOrThrow()) +{ + Console.WriteLine($"{query[token.Start..token.End]}: {token.Kind}"); +} +// Output: SELECT: Keyword, name: Keyword, FROM: Keyword, users: Identifier, WHERE: Keyword, id: Identifier, +// =: Operator, $1: Parameter, -- by id: Comment +``` + +`name` is a keyword to PostgreSQL, though an unreserved one; `token.KeywordKind` tells reserved keywords from those usable as names. + +## SplitWithScanner + +Split a SQL script containing multiple statements into an array of clean, standalone SQL statements using lexical (token-based) analysis. + +```csharp +using PgSqlParser; + +var query = "SELECT 1; SELECT 2"; +var result = Parser.SplitWithScanner(query); + +if (result.Error is null) +{ + Console.WriteLine(JsonSerializer.Serialize(result.Value)); +} + +// Return a `SplitResult` and the respective serialized JSON string is as below: +// {"Statements":[{"Location":0,"Length":8,"Text":"SELECT 1"},{"Location":9,"Length":9,"Text":" SELECT 2"}]} +``` + +## SplitWithParser + +Split a SQL script containing multiple statements into an array of clean, standalone SQL statements using Postgres full parser + +```csharp +using PgSqlParser; + +var query = "SELECT 1; SELECT 2"; +var result = Parser.SplitWithParser(query); + +if (result.Error is null) +{ + Console.WriteLine(JsonSerializer.Serialize(result.Value)); +} + +// Return a `SplitResult` and the respective serialized JSON string is as below: +// {"Statements":[{"Location":0,"Length":8,"Text":"SELECT 1"},{"Location":10,"Length":8,"Text":"SELECT 2"}]} +``` + diff --git a/docs/guide/whats-new.md b/docs/guide/whats-new.md new file mode 100644 index 0000000..33a5ae3 --- /dev/null +++ b/docs/guide/whats-new.md @@ -0,0 +1,63 @@ +# What's new + +## What's new in 2.1 + +All additions; nothing from 2.0 changes. + +**Understanding a statement** + +- **Classify statements**: `Parser.Classify` gives each statement's kind, whether it is read-only, and facts such as a data-modifying CTE, `SELECT INTO`, a locking clause or `EXPLAIN ANALYZE`. +- **Resolved references**: `GetReferences` returns the tables, functions and columns of a query as nodes, with table roles (read, write, DDL), CTE detection and columns matched to their table. +- **Parameters**: `ParameterRefs` finds each `$n`, where it is, and the type it is cast to. +- **Query shape**: each column reference knows its clause (WHERE, join condition, GROUP BY, ...), and `GetOutputColumns` names a SELECT's result columns as PostgreSQL does. +- **Locks**: `GetLocks` reports the table locks a statement takes and what they block. +- **Inside PL/pgSQL**: `ParseQueries` parses the SQL in a function body. + +**Working with the tree** + +- **Deparse a single node**: `node.Deparse()` turns one clause, expression or table reference back into SQL. +- **More control when walking**: skip a subtree or stop, and each visit knows its ancestors, the property it came from and its top-level statement. +- **Typed visitors**: `NodeVisitor` with a handler per node type, and several visitors in one walk. +- **Compare trees**: `EqualsIgnoringLocations` treats queries that differ only in formatting as equal. + +**Changing a statement** + +- **Build nodes**: `Ast.Column`, `Ast.Const`, `Ast.Eq`, `Ast.And`, `Ast.Call`, `Ast.Table`, `Ast.Expression` and more, each giving the same node the parser would. +- **Rewrite**: `Rewrite` keeps, replaces or removes nodes in place. +- **Common changes ready made**: `AddWhere`, `SetLimit`, `CapLimit`, `SetOffset`, `ToCount`, `QualifyTables`, `RenameSchema`, `RenameTable`, `ToDropStatement`, `EnsureIfNotExists`, `EnsureOrReplace`. + +**Working with text** + +- **Format in one call**: `Parser.Format` pretty prints a query or a script and keeps its comments. +- **Readable errors**: `Error.Format` and `Error.GetLineAndColumn`. +- **Tokens for display**: `Parser.Tokenize` classifies each token as keyword, identifier, literal, comment and so on. +- **Telemetry summary**: `Parser.OperationSummary` gives the short form OpenTelemetry's `db.query.summary` asks for. +- **Parse fragments**: `ParseExpression` and `ParseTypeName`. +- **Identifier quoting**: `PgIdentifier.Quote`, following PostgreSQL's `quote_ident`. + +## What's new in 2.0 + +- **PostgreSQL 18**: built on libpg_query 18.1.0, which uses the PostgreSQL 18.6 parser and includes its security fix for `pg_query_normalize`. +- **More platforms**: Linux on ARM64, Alpine and other musl-based Linux, and Windows on ARM64. The Linux and macOS libraries are now built against old OS baselines, so they load on older systems too (see the table under [Installation](/guide/getting-started#installation)). +- **.NET 10**, alongside .NET 8 and .NET 9, and compatibility with trimming and Native AOT. +- **New functions**: `IsUtilityStmt`, `Summary`, `DeparseComments` and `ParsePlpgsqlFunctions`. +- **Deparse options**: pretty printing and keeping comments, through `DeparseOptions`. +- **Fingerprint options**: `FingerprintOptions`, including a PostgreSQL 17 compatible mode. +- **Easier results**: `GetValueOrThrow`, `TryGetValue`, `Match` and deconstruction on `Result` (see [Working with results](/guide/getting-started#working-with-results)). +- **Parse tree navigation**: `Descendants`, `Walk`, `Unwrap`, `GetLocation` and `GetText` (see [Navigating the parse tree](/guide/parse-tree#navigating-the-parse-tree)). +- **Correct offsets for non-ASCII text**: `Scan` and split results index the query string directly, with `Utf8OffsetMapper` and `Error.GetCursorCharOffset` for the rest (see [Offsets and non-ASCII text](/guide/errors-offsets#offsets-and-non-ascii-text)). +- **Safer input handling**: a `null` query, a NUL character or a deeply nested query no longer crashes the process or is silently cut short. +- **Package**: IntelliSense documentation, a symbol package and Source Link are included. + +## Upgrading from 1.x + +Version 2.0 moves from the PostgreSQL 17 parser to PostgreSQL 18, which changes some results: + +- **Parse tree**: node classes and fields follow PostgreSQL 18, and `ParseResult.Version` is `180006`. +- **Fingerprints**: the default now follows PostgreSQL 18 query ID rules, so stored fingerprints for queries that use table aliases or schema names will change. Pass `FingerprintOptions.RangeVarPg17Compat` to keep the previous relation handling. +- **Statement locations**: a statement now starts at its first non-whitespace, non-comment character. This affects `SplitWithParser` and `RawStmt.StmtLocation`; for `SELECT 1; SELECT 2` the second statement is at location 10 with length 8, where it was 9 and 9. +- **Scan and Split offsets**: `Scan` token offsets and `SplitStmt` locations are UTF-16 offsets into the query string. In 1.x they were UTF-8 byte offsets, which differ as soon as the query contains non-ASCII text. Remove any conversion you did yourself. +- **Errors**: `Error.FuncName` and `Error.FileName` now hold the function and file name. In 1.x `FuncName` held the message and the file name was missing. +- **`Parse` return type**: `Parse` and `ParseAsync` return `Result`, where 1.x returned `Result`. Code that declared the nullable type explicitly needs updating. +- **Dependencies**: the minimum `Google.Protobuf` version is 3.36.2. + diff --git a/docs/package.json b/docs/package.json index 7d25ef8..747e236 100644 --- a/docs/package.json +++ b/docs/package.json @@ -3,9 +3,9 @@ "private": true, "type": "module", "scripts": { - "gen": "node scripts/gen.mjs", - "dev": "npm run gen && vitepress dev", - "build": "npm run gen && vitepress build", + "artwork": "node scripts/copy-artwork.mjs", + "dev": "npm run artwork && vitepress dev", + "build": "npm run artwork && vitepress build", "preview": "vitepress preview" }, "devDependencies": { diff --git a/docs/scripts/copy-artwork.mjs b/docs/scripts/copy-artwork.mjs new file mode 100644 index 0000000..6382021 --- /dev/null +++ b/docs/scripts/copy-artwork.mjs @@ -0,0 +1,12 @@ +// Copies the artwork from assets into the site, so the package icon and the site share one source. +// Run through "npm run dev" or "npm run build". +import { mkdirSync, copyFileSync } from 'node:fs' +import { dirname, join } from 'node:path' +import { fileURLToPath } from 'node:url' + +const docs = join(dirname(fileURLToPath(import.meta.url)), '..') +const root = join(docs, '..') + +const artwork = { 'icon.svg': 'mark-light.svg', 'icon-dark.svg': 'mark-dark.svg', 'icon-512.png': 'icon-512.png' } +mkdirSync(join(docs, 'public'), { recursive: true }) +for (const [from, to] of Object.entries(artwork)) copyFileSync(join(root, 'assets', from), join(docs, 'public', to)) diff --git a/docs/scripts/gen.mjs b/docs/scripts/gen.mjs deleted file mode 100644 index c3df33c..0000000 --- a/docs/scripts/gen.mjs +++ /dev/null @@ -1,94 +0,0 @@ -// Generates the guide pages from the repository README, so the README stays the single source -// of the documentation. Run through "npm run dev" or "npm run build". -import { readFileSync, writeFileSync, mkdirSync, rmSync, copyFileSync } from 'node:fs' -import { dirname, join } from 'node:path' -import { fileURLToPath } from 'node:url' - -const docs = join(dirname(fileURLToPath(import.meta.url)), '..') -const root = join(docs, '..') -const repo = 'https://github.com/mysticmind/pgsqlparser-dotnet/blob/main/' - -// Each page takes the README sections with these headings, in this order. -export const pages = [ - { file: 'getting-started', title: 'Getting started', sections: ['Installation', 'Usage', 'Working with results'] }, - { file: 'parse-tree', title: 'Parse trees', sections: ['Parse', 'Navigating the parse tree'] }, - { file: 'deparse-format', title: 'Deparse, format and normalize', sections: ['Deparse', 'Format', 'Normalize', 'NormalizeUtility', 'Fingerprint'] }, - { file: 'analysis', title: 'Query analysis', sections: ['Classify', 'References', 'Locks', 'Summary', 'OperationSummary', 'ParameterRefs', 'IsUtilityStmt'] }, - { file: 'scan-split', title: 'Scan, tokenize and split', sections: ['Scan', 'Tokenize', 'SplitWithScanner', 'SplitWithParser'] }, - { file: 'plpgsql', title: 'PL/pgSQL', sections: ['ParsePlpgsql'] }, - { file: 'errors-offsets', title: 'Errors and offsets', sections: ['Parse Errors', 'Offsets and non-ASCII text'] }, - { file: 'whats-new', title: "What's new", sections: ["What's new in 2.1", "What's new in 2.0", 'Upgrading from 1.x'] }, - { file: 'license', title: 'License', sections: ['License'] }, -] - -// The same slugs GitHub and VitePress give to headings. -const slug = text => text.trim().toLowerCase().replace(/[^\p{L}\p{N}\s-]/gu, '').replace(/\s+/g, '-') - -// Split the README into headings and the lines they own, ignoring "#" inside code fences. -const lines = readFileSync(join(root, 'README.md'), 'utf8').split('\n') -const headings = [] -let fenced = false -lines.forEach((line, index) => { - if (line.trimStart().startsWith('```')) fenced = !fenced - const match = !fenced && /^(#{1,6})\s+(.*)$/.exec(line) - if (match) headings.push({ level: match[1].length, text: match[2].trim(), index }) -}) - -// A section runs from its heading up to the next heading of the same or a higher level. -// A section named on a page stops early at the next section that is named on any page, -// so "Usage" keeps only its introduction. -const named = new Set(pages.flatMap(page => page.sections)) -function section(name) { - const at = headings.findIndex(heading => heading.text === name) - if (at < 0) throw new Error(`README has no section "${name}"; update docs/scripts/gen.mjs`) - const start = headings[at] - const next = headings.slice(at + 1).find(heading => heading.level <= start.level || named.has(heading.text)) - return { start, body: lines.slice(start.index, next ? next.index : lines.length) } -} - -// Where every README anchor ends up on the site. -const anchors = new Map() -for (const page of pages) { - for (const name of page.sections) { - const { start, body } = section(name) - let inFence = false - for (const line of body) { - if (line.trimStart().startsWith('```')) inFence = !inFence - const match = !inFence && /^#{1,6}\s+(.*)$/.exec(line) - if (match) anchors.set(slug(match[1]), `/guide/${page.file}#${slug(match[1])}`) - } - } -} - -function render(page) { - const out = [`# ${page.title}`, ''] - for (const name of page.sections) { - const { start, body } = section(name) - const shift = 2 - start.level // each section's own heading becomes a second-level heading - let inFence = false - for (let line of body) { - if (line.trimStart().startsWith('```')) inFence = !inFence - if (!inFence) { - const match = /^(#{1,6})(\s+.*)$/.exec(line) - if (match) line = '#'.repeat(Math.max(2, match[1].length + shift)) + match[2] - line = line - .replace(/\]\(#([^)]+)\)/g, (whole, anchor) => anchors.has(anchor) ? `](${anchors.get(anchor)})` : whole) - .replace(/\]\((LICENSE[^)]*)\)/g, `](${repo}$1)`) - } - out.push(line) - } - out.push('') - } - return out.join('\n').replace(/\n{3,}/g, '\n\n') -} - -const guide = join(docs, 'guide') -rmSync(guide, { recursive: true, force: true }) -mkdirSync(guide, { recursive: true }) -for (const page of pages) writeFileSync(join(guide, `${page.file}.md`), render(page)) - -// The artwork lives in assets; the site uses the light and dark versions of it. -const artwork = { 'icon.svg': 'mark-light.svg', 'icon-dark.svg': 'mark-dark.svg', 'icon-512.png': 'icon-512.png' } -mkdirSync(join(docs, 'public'), { recursive: true }) -for (const [from, to] of Object.entries(artwork)) copyFileSync(join(root, 'assets', from), join(docs, 'public', to)) -console.log(`Generated ${pages.length} guide pages from README.md`) diff --git a/src/PgSqlParser/PgSqlParser.csproj b/src/PgSqlParser/PgSqlParser.csproj index a8766bc..fc654f8 100644 --- a/src/PgSqlParser/PgSqlParser.csproj +++ b/src/PgSqlParser/PgSqlParser.csproj @@ -15,7 +15,7 @@ https://github.com/mysticmind/pgsqlparser-dotnet git postgresql;postgres;pg;pgsql;sql;parser;sql-parser;ast;libpg_query;pg_query;deparse;formatter;sql-formatter;fingerprint;normalize;tokenizer;plpgsql;query-analysis;sql-rewrite;native-aot - See https://github.com/mysticmind/pgsqlparser-dotnet#whats-new-in-21 + See https://mysticmind.github.io/pgsqlparser-dotnet/guide/whats-new pgsqlparser PgSqlParser From 7f994a234074de430ad4d5b656775f1ee3373be4 Mon Sep 17 00:00:00 2001 From: mysticmind Date: Fri, 9 Oct 2026 22:17:25 +0530 Subject: [PATCH 2/2] Use a simpler favicon for the docs site --- assets/favicon.svg | 2 ++ docs/.vitepress/config.mts | 2 +- docs/scripts/copy-artwork.mjs | 2 +- 3 files changed, 4 insertions(+), 2 deletions(-) create mode 100644 assets/favicon.svg diff --git a/assets/favicon.svg b/assets/favicon.svg new file mode 100644 index 0000000..dd96822 --- /dev/null +++ b/assets/favicon.svg @@ -0,0 +1,2 @@ + + diff --git a/docs/.vitepress/config.mts b/docs/.vitepress/config.mts index 8a7c8f0..63ccc8c 100644 --- a/docs/.vitepress/config.mts +++ b/docs/.vitepress/config.mts @@ -22,7 +22,7 @@ export default defineConfig({ cleanUrls: true, lastUpdated: false, head: [ - ['link', { rel: 'icon', type: 'image/svg+xml', href: `${base}mark-light.svg` }], + ['link', { rel: 'icon', type: 'image/svg+xml', href: `${base}favicon.svg` }], ['meta', { property: 'og:title', content: 'PgSqlParser' }], ['meta', { property: 'og:description', content: 'PostgreSQL SQL parser for .NET, built on libpg_query, the real PostgreSQL parser.' }], ['meta', { property: 'og:image', content: `https://mysticmind.github.io${base}icon-512.png` }], diff --git a/docs/scripts/copy-artwork.mjs b/docs/scripts/copy-artwork.mjs index 6382021..967f92e 100644 --- a/docs/scripts/copy-artwork.mjs +++ b/docs/scripts/copy-artwork.mjs @@ -7,6 +7,6 @@ import { fileURLToPath } from 'node:url' const docs = join(dirname(fileURLToPath(import.meta.url)), '..') const root = join(docs, '..') -const artwork = { 'icon.svg': 'mark-light.svg', 'icon-dark.svg': 'mark-dark.svg', 'icon-512.png': 'icon-512.png' } +const artwork = { 'icon.svg': 'mark-light.svg', 'icon-dark.svg': 'mark-dark.svg', 'icon-512.png': 'icon-512.png', 'favicon.svg': 'favicon.svg' } mkdirSync(join(docs, 'public'), { recursive: true }) for (const [from, to] of Object.entries(artwork)) copyFileSync(join(root, 'assets', from), join(docs, 'public', to))