From 640033b44098475805528205f8dc6432a4bcc5e2 Mon Sep 17 00:00:00 2001 From: mysticmind Date: Fri, 9 Oct 2026 20:48:53 +0530 Subject: [PATCH] Add lock analysis, query shape, tokens, operation summary and PL/pgSQL query parsing - GetLocks: the table locks a statement takes and what they block, following PostgreSQL's documented lock rules and, for ALTER TABLE, AlterTableGetLockLevel and the storage option table of PostgreSQL 18 - Column references carry their clause; GetOutputColumns names a SELECT's result columns as PostgreSQL does (a port of FigureColname) - Parser.Tokenize classifies tokens for display - Parser.OperationSummary gives the low-cardinality form OpenTelemetry's db.query.summary asks for - PlpgsqlNode.ParseQuery and PlpgsqlFunction.ParseQueries parse the SQL inside a function body in the mode PL/pgSQL uses for it - GetReferences no longer reports the targets of FOR UPDATE OF as tables --- README.md | 75 +++++ src/PackageSmokeTest/Program.cs | 11 + src/PgSqlParser.Tests/AnalysisTests.cs | 8 + src/PgSqlParser.Tests/InsightTests.cs | 315 ++++++++++++++++++++ src/PgSqlParser/Parser.cs | 142 +++++++++ src/PgSqlParser/PlpgsqlTree.cs | 42 +++ src/PgSqlParser/QueryReferences.cs | 68 ++++- src/PgSqlParser/QueryShape.cs | 133 +++++++++ src/PgSqlParser/SqlTokens.cs | 54 ++++ src/PgSqlParser/StatementLocks.cs | 379 +++++++++++++++++++++++++ 10 files changed, 1226 insertions(+), 1 deletion(-) create mode 100644 src/PgSqlParser.Tests/InsightTests.cs create mode 100644 src/PgSqlParser/QueryShape.cs create mode 100644 src/PgSqlParser/SqlTokens.cs create mode 100644 src/PgSqlParser/StatementLocks.cs diff --git a/README.md b/README.md index f661eb1..8f6a16a 100644 --- a/README.md +++ b/README.md @@ -382,6 +382,34 @@ foreach (var column in references.Columns) A column is matched to its table when the query alone settles it: its qualifier names a table or alias in scope, or it has no qualifier and there is only one table in scope. Otherwise `Table` is null, because it would take the database catalog to tell. `Summary` returns similar information as plain names and is faster; use `GetReferences` when you need the nodes, aliases or column matching. +Each column also has a `Clause` (select list, WHERE, join condition, GROUP BY, and so on), so the columns that filter rows are `references.Columns.Where(c => c.Clause is QueryClause.Where or QueryClause.JoinCondition)`. For a SELECT, `GetOutputColumns()` lists the result columns with the names PostgreSQL gives them: + +```csharp +var select = Parser.Parse("SELECT id, lower(name), total AS amount, 1 FROM t").GetValueOrThrow().Stmts[0].Stmt.SelectStmt; +Console.WriteLine(string.Join(", ", select.GetOutputColumns().Select(column => column.Name))); +// Output: id, lower, amount, ?column? +``` + +### Locks + +Find out which table locks a statement takes, and what they block. This follows PostgreSQL's documented lock rules, and for ALTER TABLE the same per-sub-command rules PostgreSQL itself uses. + +```csharp +using PgSqlParser; + +var script = "CREATE INDEX i ON orders (customer_id); ALTER TABLE orders ADD COLUMN note text; ALTER TABLE orders VALIDATE CONSTRAINT fk"; +foreach (var tableLock in Parser.Parse(script).GetValueOrThrow().GetLocks()) +{ + Console.WriteLine($"{tableLock} blocks reads: {tableLock.Mode.BlocksReads()}, blocks writes: {tableLock.Mode.BlocksWrites()}"); +} +// Output: +// orders: Share blocks reads: False, blocks writes: True +// orders: AccessExclusive blocks reads: True, blocks writes: True +// orders: ShareUpdateExclusive blocks reads: False, blocks writes: False +``` + +It reports locks on the tables a statement names. It cannot see locks on tables reached through a view, an index name, a partition or a trigger, since those need the database catalog, and it does not cover row-level locks. See the documentation of `GetLocks` for the list of statements covered. + ### Format Format a query across indented lines, keeping its comments. Each statement of a script is formatted on its own. @@ -404,6 +432,38 @@ Console.WriteLine(formatted); `FormatOptions` sets the indent size, the line length, comma placement, a trailing newline, and whether comments are kept. +### Tokenize + +Split a query into tokens classified for display, for syntax highlighting and similar uses. It works on SQL that does not parse, as long as it can be tokenized. + +```csharp +using PgSqlParser; + +var query = "SELECT name FROM users WHERE id = $1 -- by id"; +foreach (var token in Parser.Tokenize(query).GetValueOrThrow()) +{ + Console.WriteLine($"{query[token.Start..token.End]}: {token.Kind}"); +} +// Output: SELECT: Keyword, name: Keyword, FROM: Keyword, users: Identifier, WHERE: Keyword, id: Identifier, +// =: Operator, $1: Parameter, -- by id: Comment +``` + +`name` is a keyword to PostgreSQL, though an unreserved one; `token.KeywordKind` tells reserved keywords from those usable as names. + +### OperationSummary + +Describe a query in a few low-cardinality words: the operation and the tables it touches. Literals, parameters and columns are left out, so it is safe as a span name or a metric label. This is the form OpenTelemetry's `db.query.summary` attribute asks for. + +```csharp +using PgSqlParser; + +Console.WriteLine(Parser.OperationSummary("SELECT c.name FROM customers c JOIN orders o ON o.cid = c.id WHERE c.email = 'a@b.c'").GetValueOrThrow()); +// Output: SELECT customers orders + +Console.WriteLine(Parser.OperationSummary("INSERT INTO audit SELECT * FROM users").GetValueOrThrow()); +// Output: INSERT audit users +``` + ### ParameterRefs Find the parameter references (`$1`, `$2`, ...) in a query, with where each one is in the query string. References inside string literals and comments are not parameters and are left out. @@ -484,6 +544,16 @@ foreach (var query in function.Queries()) // Output: SELECT * FROM foo WHERE fooid > 0 ``` +`function.ParseQueries()` goes one step further and parses each of those pieces of SQL, in the mode PL/pgSQL itself uses for it, so a function body can be analysed with the same tools as any query: + +```csharp +var tables = function.ParseQueries() + .Where(query => query.Tree is not null) + .SelectMany(query => query.Tree!.GetReferences().Tables) + .Select(table => table.Name); +// foo +``` + libpg_query has no schema for its PL/pgSQL output, so a node (`PlpgsqlNode`) exposes its `Kind`, its `Children` and its raw `Json` (a `JsonElement`) and not typed properties per statement kind. ### Fingerprint @@ -716,6 +786,9 @@ All additions; nothing from 2.0 changes. - **Classify statements**: `Parser.Classify` gives each statement's kind, whether it is read-only, and facts such as a data-modifying CTE, `SELECT INTO`, a locking clause or `EXPLAIN ANALYZE`. - **Resolved references**: `GetReferences` returns the tables, functions and columns of a query as nodes, with table roles (read, write, DDL), CTE detection and columns matched to their table. - **Parameters**: `ParameterRefs` finds each `$n`, where it is, and the type it is cast to. +- **Query shape**: each column reference knows its clause (WHERE, join condition, GROUP BY, ...), and `GetOutputColumns` names a SELECT's result columns as PostgreSQL does. +- **Locks**: `GetLocks` reports the table locks a statement takes and what they block. +- **Inside PL/pgSQL**: `ParseQueries` parses the SQL in a function body. **Working with the tree** @@ -734,6 +807,8 @@ All additions; nothing from 2.0 changes. - **Format in one call**: `Parser.Format` pretty prints a query or a script and keeps its comments. - **Readable errors**: `Error.Format` and `Error.GetLineAndColumn`. +- **Tokens for display**: `Parser.Tokenize` classifies each token as keyword, identifier, literal, comment and so on. +- **Telemetry summary**: `Parser.OperationSummary` gives the short form OpenTelemetry's `db.query.summary` asks for. - **Parse fragments**: `ParseExpression` and `ParseTypeName`. - **Identifier quoting**: `PgIdentifier.Quote`, following PostgreSQL's `quote_ident`. diff --git a/src/PackageSmokeTest/Program.cs b/src/PackageSmokeTest/Program.cs index fde4ea5..4168dd5 100644 --- a/src/PackageSmokeTest/Program.cs +++ b/src/PackageSmokeTest/Program.cs @@ -81,6 +81,17 @@ void Check(string name, bool ok) var visitedTables = 0; rewritten.Walk(new NodeVisitor().On((_, _) => visitedTables++), new NodeVisitor().On((_, _) => WalkAction.Stop)); Check("NodeVisitor", visitedTables == 1); +Check("GetLocks", Parser.Parse("ALTER TABLE t VALIDATE CONSTRAINT c; CREATE INDEX i ON t (a)").Value!.GetLocks() + .Select(l => l.Mode).SequenceEqual([LockMode.ShareUpdateExclusive, LockMode.Share])); +var shape = Parser.Parse("SELECT id, lower(name) AS n FROM t WHERE a = 1").Value!; +Check("Clauses and output columns", shape.GetReferences().Columns.Last().Clause == QueryClause.Where + && shape.Stmts[0].Stmt.SelectStmt.GetOutputColumns().Select(c => c.Name).SequenceEqual(["id", "n"])); +Check("Tokenize", Parser.Tokenize("SELECT 1 -- x").Value?.Select(t => t.Kind) + .SequenceEqual([SqlTokenKind.Keyword, SqlTokenKind.NumericLiteral, SqlTokenKind.Comment]) == true); +Check("OperationSummary", Parser.OperationSummary("SELECT * FROM a JOIN b ON true WHERE x = 'secret'").Value == "SELECT a b"); +Check("PL/pgSQL ParseQueries", Parser.ParsePlpgsqlFunctions( + "CREATE FUNCTION f() RETURNS void AS $$ BEGIN PERFORM x FROM items; END $$ LANGUAGE plpgsql").Value![0] + .ParseQueries().Single().Tree!.GetReferences().Tables.Single().Name == "items"); Check("ParameterRefs", Parser.ParameterRefs("SELECT $1, '$2', $3").Value?.Select(p => p.Number).SequenceEqual([1, 3]) == true); Check("ParsePlpgsqlFunctions", Parser.ParsePlpgsqlFunctions( "CREATE FUNCTION f() RETURNS int AS $$ BEGIN RETURN 1; END; $$ LANGUAGE plpgsql").Value?[0].Queries().SequenceEqual(["1"]) == true); diff --git a/src/PgSqlParser.Tests/AnalysisTests.cs b/src/PgSqlParser.Tests/AnalysisTests.cs index 5bf30d3..0e88f65 100644 --- a/src/PgSqlParser.Tests/AnalysisTests.cs +++ b/src/PgSqlParser.Tests/AnalysisTests.cs @@ -236,6 +236,14 @@ public void ReferencesFromANodeAndWithNonAsciiNames() references.Columns.ShouldAllBe(column => column.Table != null && column.Table.Name == "données"); } + [Fact] + public void LockingClauseTargetsAreNotTables() + { + var references = Parser.Parse("SELECT * FROM accounts a JOIN users u ON true FOR UPDATE OF a").GetValueOrThrow().GetReferences(); + + references.Tables.Select(table => table.Name).ShouldBe(["accounts", "users"]); + } + [Fact] public void ReferencesFromASingleStatementNode() { diff --git a/src/PgSqlParser.Tests/InsightTests.cs b/src/PgSqlParser.Tests/InsightTests.cs new file mode 100644 index 0000000..a9fb13b --- /dev/null +++ b/src/PgSqlParser.Tests/InsightTests.cs @@ -0,0 +1,315 @@ +using Shouldly; +using Xunit; + +namespace PgSqlParser.Tests; + +public class InsightTests +{ + private static ParseResult Parse(string query) => Parser.Parse(query).GetValueOrThrow(); + + private static string Locks(string query) => + string.Join(", ", Parse(query).GetLocks().Select(tableLock => tableLock.ToString())); + + [Theory] + [InlineData("SELECT * FROM a JOIN b ON true", "a: AccessShare, b: AccessShare")] + [InlineData("SELECT * FROM a, b FOR UPDATE OF a", "a: RowShare, b: AccessShare")] + [InlineData("SELECT * FROM a FOR SHARE", "a: RowShare")] + [InlineData("SELECT * FROM a x FOR UPDATE OF x", "a: RowShare")] + [InlineData("WITH c AS (SELECT * FROM x) INSERT INTO t SELECT * FROM c JOIN y ON true", "t: RowExclusive, y: AccessShare, x: AccessShare")] + [InlineData("UPDATE t SET a = 1 FROM u WHERE u.id = t.id", "t: RowExclusive, u: AccessShare")] + [InlineData("DELETE FROM s.t", "s.t: RowExclusive")] + [InlineData("MERGE INTO t USING s ON true WHEN MATCHED THEN DELETE", "t: RowExclusive, s: AccessShare")] + [InlineData("COPY t FROM STDIN", "t: RowExclusive")] + [InlineData("COPY t TO STDOUT", "t: AccessShare")] + [InlineData("EXPLAIN SELECT * FROM a", "a: AccessShare")] + [InlineData("TRUNCATE a, s.b", "a: AccessExclusive, s.b: AccessExclusive")] + [InlineData("LOCK TABLE a", "a: AccessExclusive")] + [InlineData("LOCK TABLE a IN SHARE MODE", "a: Share")] + [InlineData("LOCK TABLE a IN ROW EXCLUSIVE MODE", "a: RowExclusive")] + [InlineData("CREATE INDEX i ON t (a)", "t: Share")] + [InlineData("CREATE INDEX CONCURRENTLY i ON t (a)", "t: ShareUpdateExclusive")] + [InlineData("CREATE TRIGGER g BEFORE INSERT ON t FOR EACH ROW EXECUTE FUNCTION f()", "t: ShareRowExclusive")] + [InlineData("CREATE STATISTICS st ON a, b FROM t", "t: ShareUpdateExclusive")] + [InlineData("CREATE TABLE n (id int REFERENCES parent, x int, FOREIGN KEY (x) REFERENCES s.other)", "parent: ShareRowExclusive, s.other: ShareRowExclusive")] + [InlineData("DROP TABLE a, s.b", "a: AccessExclusive, s.b: AccessExclusive")] + [InlineData("VACUUM t", "t: ShareUpdateExclusive")] + [InlineData("VACUUM FULL t", "t: AccessExclusive")] + [InlineData("VACUUM (FULL false, ANALYZE) t", "t: ShareUpdateExclusive")] + [InlineData("ANALYZE t", "t: ShareUpdateExclusive")] + [InlineData("CLUSTER t USING i", "t: AccessExclusive")] + [InlineData("REINDEX TABLE t", "t: Share")] + [InlineData("REINDEX TABLE CONCURRENTLY t", "t: ShareUpdateExclusive")] + [InlineData("REFRESH MATERIALIZED VIEW m", "m: AccessExclusive")] + [InlineData("REFRESH MATERIALIZED VIEW CONCURRENTLY m", "m: Exclusive")] + [InlineData("COMMENT ON TABLE s.t IS 'x'", "s.t: ShareUpdateExclusive")] + [InlineData("COMMENT ON COLUMN s.t.c IS 'x'", "s.t: ShareUpdateExclusive")] + // Not covered: the statement does not name the table, or it creates what it names. + [InlineData("DROP INDEX i", "")] + [InlineData("REINDEX INDEX i", "")] + [InlineData("ALTER INDEX i RENAME TO j", "")] + [InlineData("SELECT * INTO n FROM a", "a: AccessShare")] + [InlineData("SELECT 1", "")] + public void LocksPerStatement(string query, string expected) + { + Locks(query).ShouldBe(expected); + } + + // These follow AlterTableGetLockLevel in PostgreSQL 18's tablecmds.c. + [Theory] + [InlineData("ALTER TABLE t ADD COLUMN c int", "t: AccessExclusive")] + [InlineData("ALTER TABLE t DROP COLUMN c", "t: AccessExclusive")] + [InlineData("ALTER TABLE t ALTER COLUMN a TYPE bigint", "t: AccessExclusive")] + [InlineData("ALTER TABLE t ALTER COLUMN a SET DEFAULT 1", "t: AccessExclusive")] + [InlineData("ALTER TABLE t ALTER COLUMN a SET NOT NULL", "t: AccessExclusive")] + [InlineData("ALTER TABLE t ADD PRIMARY KEY (id)", "t: AccessExclusive")] + [InlineData("ALTER TABLE t ADD CONSTRAINT c CHECK (a > 0) NOT VALID", "t: AccessExclusive")] + [InlineData("ALTER TABLE t ADD CONSTRAINT fk FOREIGN KEY (a) REFERENCES p (id)", "p: ShareRowExclusive, t: ShareRowExclusive")] + [InlineData("ALTER TABLE t VALIDATE CONSTRAINT c", "t: ShareUpdateExclusive")] + [InlineData("ALTER TABLE t ALTER COLUMN a SET STATISTICS 100", "t: ShareUpdateExclusive")] + [InlineData("ALTER TABLE t SET (fillfactor = 70)", "t: ShareUpdateExclusive")] + [InlineData("ALTER TABLE t SET (autovacuum_enabled = false)", "t: ShareUpdateExclusive")] + [InlineData("ALTER TABLE t SET (fillfactor = 70, user_catalog_table = true)", "t: AccessExclusive")] + [InlineData("ALTER TABLE t RESET (fillfactor)", "t: ShareUpdateExclusive")] + [InlineData("ALTER TABLE t DISABLE TRIGGER ALL", "t: ShareRowExclusive")] + [InlineData("ALTER TABLE t ENABLE TRIGGER trg", "t: ShareRowExclusive")] + [InlineData("ALTER TABLE t CLUSTER ON i", "t: ShareUpdateExclusive")] + [InlineData("ALTER TABLE t ATTACH PARTITION p FOR VALUES IN (1)", "t: ShareUpdateExclusive")] + [InlineData("ALTER TABLE t DETACH PARTITION p", "t: AccessExclusive")] + [InlineData("ALTER TABLE t DETACH PARTITION p CONCURRENTLY", "t: ShareUpdateExclusive")] + [InlineData("ALTER TABLE t ENABLE ROW LEVEL SECURITY", "t: AccessExclusive")] + [InlineData("ALTER TABLE t SET LOGGED", "t: AccessExclusive")] + [InlineData("ALTER TABLE t RENAME COLUMN a TO b", "t: AccessExclusive")] + [InlineData("ALTER TABLE t RENAME TO u", "t: AccessExclusive")] + // The strongest sub-command decides. + [InlineData("ALTER TABLE t VALIDATE CONSTRAINT c, ALTER COLUMN a SET STATISTICS 10", "t: ShareUpdateExclusive")] + [InlineData("ALTER TABLE t VALIDATE CONSTRAINT c, ADD COLUMN x int", "t: AccessExclusive")] + public void AlterTableLocks(string query, string expected) + { + Locks(query).ShouldBe(expected); + } + + [Fact] + public void LockModesAndConflicts() + { + var locks = Parse("SELECT 1 FROM a; CREATE INDEX i ON a (x); ALTER TABLE a ADD COLUMN y int").GetLocks(); + + locks.Select(tableLock => (tableLock.Table, tableLock.Mode, tableLock.StatementIndex)).ShouldBe( + [("a", LockMode.AccessShare, 0), ("a", LockMode.Share, 1), ("a", LockMode.AccessExclusive, 2)]); + + // CREATE INDEX blocks writes but not reads; ALTER TABLE ADD COLUMN blocks both. + LockMode.Share.BlocksWrites().ShouldBeTrue(); + LockMode.Share.BlocksReads().ShouldBeFalse(); + LockMode.ShareUpdateExclusive.BlocksWrites().ShouldBeFalse(); + LockMode.AccessExclusive.BlocksReads().ShouldBeTrue(); + + // The conflict table is symmetric and AccessShare only conflicts with AccessExclusive. + foreach (var a in Enum.GetValues()) + { + foreach (var b in Enum.GetValues()) + a.ConflictsWith(b).ShouldBe(b.ConflictsWith(a)); + + LockMode.AccessShare.ConflictsWith(a).ShouldBe(a == LockMode.AccessExclusive); + LockMode.AccessExclusive.ConflictsWith(a).ShouldBeTrue(); + } + + LockMode.ShareUpdateExclusive.ConflictsWith(LockMode.ShareUpdateExclusive).ShouldBeTrue(); + LockMode.Share.ConflictsWith(LockMode.Share).ShouldBeFalse(); + LockMode.RowExclusive.ConflictsWith(LockMode.RowExclusive).ShouldBeFalse(); + Parse("VACUUM t").Stmts[0].GetLocks().Single().StatementIndex.ShouldBeNull(); + } + + [Fact] + public void ColumnsKnowTheirClause() + { + const string query = + "SELECT c.name, count(*) FROM customers c JOIN orders o ON o.customer_id = c.id " + + "WHERE c.region = 'eu' AND o.total > 10 GROUP BY c.name HAVING count(o.id) > 1 ORDER BY c.name; " + + "UPDATE t SET a = b + 1 WHERE id = 5 RETURNING a; DELETE FROM t USING u WHERE u.x = t.x; " + + "MERGE INTO t USING s ON s.id = t.id WHEN MATCHED THEN DELETE"; + + var columns = Parse(query).GetReferences().Columns; + + columns.Select(column => (column.ToString(), column.Clause)).ShouldBe( + [ + ("c.name", QueryClause.SelectList), + ("o.customer_id", QueryClause.JoinCondition), ("c.id", QueryClause.JoinCondition), + ("c.region", QueryClause.Where), ("o.total", QueryClause.Where), + ("c.name", QueryClause.GroupBy), ("o.id", QueryClause.Having), ("c.name", QueryClause.OrderBy), + ("b", QueryClause.Set), ("id", QueryClause.Where), ("a", QueryClause.Returning), + ("u.x", QueryClause.Where), ("t.x", QueryClause.Where), + ("s.id", QueryClause.JoinCondition), ("t.id", QueryClause.JoinCondition) + ]); + + // The columns that filter rows, per table: the input for index advice. + var filters = columns + .Where(column => column is { Clause: QueryClause.Where or QueryClause.JoinCondition, Table: not null, StatementIndex: 0 }) + .Select(column => $"{column.Table!.Name}.{column.Name}"); + filters.ShouldBe(["orders.customer_id", "customers.id", "customers.region", "orders.total"]); + } + + [Fact] + public void ColumnClauseBelongsToTheNearestStatement() + { + var columns = Parse("SELECT a FROM t WHERE b IN (SELECT c FROM u WHERE d = 1 ORDER BY e)").GetReferences().Columns; + + columns.Select(column => (column.Name, column.Clause)).ShouldBe( + [ + ("a", QueryClause.SelectList), ("b", QueryClause.Where), + ("c", QueryClause.SelectList), ("d", QueryClause.Where), ("e", QueryClause.OrderBy) + ]); + } + + [Fact] + public void OutputColumnsAreNamedAsPostgresNamesThem() + { + const string query = + "SELECT a, t.b, lower(c), d AS x, 1, 'x'::text, e::int, count(*), CASE WHEN a THEN b END, " + + "CASE WHEN a THEN 1 ELSE f END, coalesce(a, b), greatest(1, 2), (SELECT g FROM u), EXISTS (SELECT 1), " + + "ARRAY[1], ROW(1, 2), current_date, a.b.c, nullif(a, b), a + 1, (r).field, a COLLATE \"C\" FROM t"; + + var columns = Parse(query).Stmts[0].Stmt.SelectStmt.GetOutputColumns(); + + columns.Select(column => column.Name).ShouldBe( + [ + "a", "b", "lower", "x", "?column?", "text", "e", "count", "case", "f", "coalesce", "greatest", "g", "exists", + "array", "row", "current_date", "c", "nullif", "?column?", "field", "a" + ]); + columns.ShouldAllBe(column => !column.IsStar); + columns[3].Expression.ShouldBeOfType(); + QueryShape.UnnamedColumn.ShouldBe("?column?"); + } + + [Fact] + public void OutputColumnsOfStarsSetOperationsAndValues() + { + var star = Parse("SELECT *, t.*, a FROM t").Stmts[0].Stmt.SelectStmt.GetOutputColumns(); + star.Select(column => (column.Name, column.IsStar)).ShouldBe([(null, true), (null, true), ("a", false)]); + + Parse("SELECT a, b FROM t UNION SELECT c, d FROM u UNION ALL SELECT e, f FROM v").Stmts[0].Stmt.SelectStmt + .GetOutputColumns().Select(column => column.Name).ShouldBe(["a", "b"]); + + Parse("VALUES (1, 'x'), (2, 'y')").Stmts[0].Stmt.SelectStmt + .GetOutputColumns().Select(column => column.Name).ShouldBe(["column1", "column2"]); + } + + [Fact] + public void TokenizeClassifiesEveryToken() + { + const string query = "SELECT a, 'x' AS \"B\", 1.5 + $1 FROM s.t -- note\n WHERE a::int >= 2 /* c */ AND b <> 'é😀';"; + + var tokens = Parser.Tokenize(query).GetValueOrThrow(); + + tokens.Select(token => (query[token.Start..token.End], token.Kind)).ShouldBe( + [ + ("SELECT", SqlTokenKind.Keyword), ("a", SqlTokenKind.Identifier), (",", SqlTokenKind.Punctuation), + ("'x'", SqlTokenKind.StringLiteral), ("AS", SqlTokenKind.Keyword), ("\"B\"", SqlTokenKind.Identifier), + (",", SqlTokenKind.Punctuation), ("1.5", SqlTokenKind.NumericLiteral), ("+", SqlTokenKind.Operator), + ("$1", SqlTokenKind.Parameter), ("FROM", SqlTokenKind.Keyword), ("s", SqlTokenKind.Identifier), + (".", SqlTokenKind.Punctuation), ("t", SqlTokenKind.Identifier), ("-- note", SqlTokenKind.Comment), + ("WHERE", SqlTokenKind.Keyword), ("a", SqlTokenKind.Identifier), ("::", SqlTokenKind.Operator), + ("int", SqlTokenKind.Keyword), (">=", SqlTokenKind.Operator), ("2", SqlTokenKind.NumericLiteral), + ("/* c */", SqlTokenKind.Comment), ("AND", SqlTokenKind.Keyword), ("b", SqlTokenKind.Identifier), + ("<>", SqlTokenKind.Operator), ("'é😀'", SqlTokenKind.StringLiteral), (";", SqlTokenKind.Punctuation) + ]); + tokens[0].KeywordKind.ShouldBe(KeywordKind.ReservedKeyword); + tokens[0].Token.ShouldBe(Token.Select); + } + + [Fact] + public async Task TokenizeEdgeCases() + { + Parser.Tokenize("").GetValueOrThrow().ShouldBeEmpty(); + Parser.Tokenize("SELECT 'unterminated").IsSuccess.ShouldBeFalse(); + // Tokens are available for SQL that does not parse. + Parser.Tokenize("SELECT FROM WHERE").GetValueOrThrow().Select(token => token.Kind) + .ShouldBe([SqlTokenKind.Keyword, SqlTokenKind.Keyword, SqlTokenKind.Keyword]); + (await Parser.TokenizeAsync("a = 1")).GetValueOrThrow().Select(token => token.Kind) + .ShouldBe([SqlTokenKind.Identifier, SqlTokenKind.Operator, SqlTokenKind.NumericLiteral]); + } + + [Theory] + [InlineData("SELECT c.name FROM customers c JOIN public.orders o ON true WHERE c.id = 5", "SELECT customers public.orders")] + [InlineData("SELECT * FROM t WHERE secret = 'hunter2' AND id = 42", "SELECT t")] + [InlineData("SELECT * FROM t t1 JOIN t t2 ON true", "SELECT t")] + [InlineData("INSERT INTO audit SELECT * FROM users", "INSERT audit users")] + [InlineData("WITH r AS (SELECT 1) UPDATE t SET a = 1 FROM r", "UPDATE t")] + [InlineData("DELETE FROM t", "DELETE t")] + [InlineData("VALUES (1)", "VALUES")] + [InlineData("SELECT 1", "SELECT")] + [InlineData("CREATE TABLE s.t (a int)", "CREATE TABLE s.t")] + [InlineData("ALTER TABLE t ADD COLUMN b int", "ALTER TABLE t")] + [InlineData("DROP TABLE IF EXISTS a, s.b", "DROP TABLE a s.b")] + [InlineData("CREATE UNIQUE INDEX CONCURRENTLY i ON t (a)", "CREATE INDEX t")] + [InlineData("CREATE OR REPLACE VIEW v AS SELECT * FROM t", "CREATE VIEW v t")] + [InlineData("VACUUM t", "VACUUM t")] + [InlineData("TRUNCATE t", "TRUNCATE t")] + [InlineData("BEGIN", "BEGIN")] + [InlineData("SET search_path = x", "SET")] + [InlineData("/* hi */ GRANT SELECT ON t TO x", "GRANT t")] + [InlineData("EXPLAIN ANALYZE SELECT * FROM t", "EXPLAIN t")] + [InlineData("CALL p()", "CALL")] + [InlineData("SELECT 1; DELETE FROM t", "SELECT; DELETE t")] + public void OperationSummary(string query, string expected) + { + Parser.OperationSummary(query).GetValueOrThrow().ShouldBe(expected); + } + + [Fact] + public void OperationSummaryIsCutAtAWordBoundary() + { + const string query = "SELECT * FROM customers, orders, order_items"; + + Parser.OperationSummary(query).GetValueOrThrow().ShouldBe("SELECT customers orders order_items"); + Parser.OperationSummary(query, 20).GetValueOrThrow().ShouldBe("SELECT customers"); + Parser.OperationSummary(query, 23).GetValueOrThrow().ShouldBe("SELECT customers orders"); + Parser.OperationSummary("SELECT FROM WHERE").IsSuccess.ShouldBeFalse(); + Parser.OperationSummary("").GetValueOrThrow().ShouldBe(""); + } + + [Fact] + public void PlpgsqlQueriesAreParsedInTheirOwnMode() + { + const string sql = """ + CREATE FUNCTION f(n int) RETURNS int AS $$ + DECLARE total int := 0; r record; + BEGIN + FOR r IN SELECT * FROM items WHERE qty > n LOOP total := total + r.qty; END LOOP; + IF total > 100 THEN RETURN total; END IF; + PERFORM log_it(total); + RETURN 0; + END $$ LANGUAGE plpgsql + """; + var function = Parser.ParsePlpgsqlFunctions(sql).GetValueOrThrow().ShouldHaveSingleItem(); + + var queries = function.ParseQueries(); + + queries.ShouldAllBe(query => query.Tree != null && query.Error == null); + queries.Select(query => (query.Sql, query.Tree!.Stmts[0].Stmt.Unwrap()!.Descriptor.Name)).ShouldBe( + [ + ("0", "SelectStmt"), + // An assignment is not a query; PL/pgSQL parses it in its own mode. + ("total := total + r.qty", "PLAssignStmt"), + ("SELECT * FROM items WHERE qty > n", "SelectStmt"), + ("total > 100", "SelectStmt"), + ("SELECT log_it(total)", "SelectStmt"), + ("0", "SelectStmt") + ]); + + // The tables and functions a function body uses, through the same APIs as any query. + var trees = queries.Select(query => query.Tree!).ToList(); + trees.SelectMany(tree => tree.GetReferences().Tables).Select(table => table.Name).ShouldBe(["items"]); + trees.SelectMany(tree => tree.GetReferences().Functions).Select(call => call.Name).ShouldBe(["log_it"]); + } + + [Fact] + public void PlpgsqlParseQueryOnOtherNodesAndBrokenSql() + { + var function = Parser.ParsePlpgsqlFunctions( + "CREATE FUNCTION f() RETURNS void AS $$ BEGIN PERFORM 1; END $$ LANGUAGE plpgsql").GetValueOrThrow()[0]; + + var body = function.Body.ShouldNotBeNull(); + body.ParseQuery().Error.ShouldNotBeNull().Message.ShouldBe("a PLpgSQL_stmt_block node holds no SQL to parse"); + function.Root.Descendants().Single(node => node.Kind == "PLpgSQL_expr").ParseQuery().GetValueOrThrow() + .Deparse().GetValueOrThrow().ShouldBe("SELECT 1"); + } +} diff --git a/src/PgSqlParser/Parser.cs b/src/PgSqlParser/Parser.cs index aa5ff92..c527e13 100644 --- a/src/PgSqlParser/Parser.cs +++ b/src/PgSqlParser/Parser.cs @@ -1051,6 +1051,148 @@ public static Task> FormatAsync(string query, FormatOptions? opti return RunAsync(() => Format(query, options), cancellationToken); } + /// + /// Split a query into tokens classified for display: keywords, identifiers, literals, parameters, + /// operators, punctuation and comments. Whitespace is not a token. See for the + /// raw scanner output. + /// + /// + /// + public static Result> Tokenize(string query) + { + if (!Scan(query).TryGetValue(out var scan, out var error)) + return Result>.Failure(error); + + var tokens = new List(scan.Tokens.Count); + foreach (var token in scan.Tokens) + { + tokens.Add(new SqlToken(token.Start, token.End, SqlTokenClassifier.KindOf(token.Token, token.KeywordKind), + token.Token, token.KeywordKind)); + } + + return Result>.Success(tokens); + } + + /// + /// Async split a query into classified tokens + /// + /// + /// + /// + public static Task>> TokenizeAsync(string query, CancellationToken cancellationToken = default) + { + return RunAsync(() => Tokenize(query), cancellationToken); + } + + /// + /// Describe a query in a few low-cardinality words: the operation and the tables it touches, such as + /// SELECT customers orders or INSERT audit. Literals, parameters and columns are left + /// out, so it is safe to use as a span name or metric label. This is the form OpenTelemetry's + /// db.query.summary attribute asks for. + /// + /// + /// The summary is cut at a word boundary to fit. 255 is OpenTelemetry's limit. + /// + public static Result OperationSummary(string query, int maxLength = 255) + { + if (!Parse(query).TryGetValue(out var tree, out var error)) + return Result.Failure(error); + + var parts = new List(); + foreach (var statement in tree.Stmts) + { + var words = new List { OperationName(statement, query) }; + var names = statement.GetReferences().Tables.Where(table => !table.IsCte).Select(table => table.ToString()) + .Concat(DroppedRelations(statement)); + foreach (var name in names) + { + if (!words.Contains(name)) + words.Add(name); + } + + parts.Add(string.Join(' ', words)); + } + + var summary = string.Join("; ", parts); + if (summary.Length <= maxLength) + return Result.Success(summary); + + var cut = summary.LastIndexOf(' ', Math.Max(0, maxLength)); + return Result.Success(summary[..(cut > 0 ? cut : Math.Max(0, maxLength))].TrimEnd(';', ' ')); + } + + // The operation of a statement: SELECT, CREATE TABLE, DROP INDEX, VACUUM, ... + private static string OperationName(RawStmt statement, string query) + { + var node = statement.Stmt?.Unwrap(); + var known = node switch + { + SelectStmt { ValuesLists.Count: > 0 } => "VALUES", + SelectStmt => "SELECT", + InsertStmt => "INSERT", + UpdateStmt => "UPDATE", + DeleteStmt => "DELETE", + MergeStmt => "MERGE", + CreateStmt => "CREATE TABLE", + CreateTableAsStmt { Objtype: ObjectType.ObjectMatview } => "CREATE MATERIALIZED VIEW", + CreateTableAsStmt => "CREATE TABLE", + IndexStmt => "CREATE INDEX", + ViewStmt => "CREATE VIEW", + CreateSeqStmt => "CREATE SEQUENCE", + CreateSchemaStmt => "CREATE SCHEMA", + CreateFunctionStmt { IsProcedure: true } => "CREATE PROCEDURE", + CreateFunctionStmt => "CREATE FUNCTION", + CreateTrigStmt => "CREATE TRIGGER", + AlterTableStmt { Objtype: ObjectType.ObjectIndex } => "ALTER INDEX", + AlterTableStmt => "ALTER TABLE", + DropStmt drop => drop.RemoveType switch + { + ObjectType.ObjectTable => "DROP TABLE", + ObjectType.ObjectIndex => "DROP INDEX", + ObjectType.ObjectView => "DROP VIEW", + ObjectType.ObjectMatview => "DROP MATERIALIZED VIEW", + ObjectType.ObjectSequence => "DROP SEQUENCE", + ObjectType.ObjectSchema => "DROP SCHEMA", + ObjectType.ObjectFunction => "DROP FUNCTION", + ObjectType.ObjectType => "DROP TYPE", + _ => "DROP" + }, + _ => null + }; + if (known is not null) + return known; + + // For the rest, the keyword the statement starts with: GRANT, VACUUM, SET, BEGIN, EXPLAIN, ... + var text = statement.GetText(query); + if (Scan(text).TryGetValue(out var scan)) + { + foreach (var token in scan.Tokens) + { + if (token.Token is not (Token.SqlComment or Token.CComment)) + return text[token.Start..token.End].ToUpperInvariant(); + } + } + + return node?.Descriptor.Name ?? "UNKNOWN"; + } + + // The tables a statement names outside of table references: the targets of DROP TABLE and similar. + private static IEnumerable DroppedRelations(RawStmt statement) + { + if (statement.Stmt?.Unwrap() is not DropStmt { RemoveType: ObjectType.ObjectTable or ObjectType.ObjectView + or ObjectType.ObjectMatview or ObjectType.ObjectForeignTable } drop) + { + yield break; + } + + foreach (var target in drop.Objects) + { + var parts = target.List?.Items.Select(part => part.String?.Sval).OfType().ToList(); + if (parts is { Count: > 0 }) + yield return string.Join('.', parts); + } + } + /// /// Classify each statement of a query: its kind, whether it is read-only, and facts such as a /// data-modifying CTE or a locking clause. See . diff --git a/src/PgSqlParser/PlpgsqlTree.cs b/src/PgSqlParser/PlpgsqlTree.cs index 1f891b6..86b3775 100644 --- a/src/PgSqlParser/PlpgsqlTree.cs +++ b/src/PgSqlParser/PlpgsqlTree.cs @@ -51,6 +51,20 @@ public IEnumerable Descendants() return Json.TryGetProperty(name, out var value) && value.ValueKind == JsonValueKind.String ? value.GetString() : null; } + /// + /// Parses the SQL of a PLpgSQL_expr node, in the mode PL/pgSQL itself uses for it: a query, an + /// expression (parsed as SELECT expression) or an assignment. Fails for any other kind of node. + /// + public Result ParseQuery() + { + if (Kind != "PLpgSQL_expr" || GetString("query") is not { } query) + return Result.Failure(new Error($"a {Kind} node holds no SQL to parse", null, null, 0, 0, null)); + + // parseMode is PostgreSQL's RawParseMode, which ParserOptions uses the same numbers for. + var mode = Json.TryGetProperty("parseMode", out var value) && value.TryGetInt32(out var number) ? number : 0; + return Parser.Parse(query, (ParserOptions)mode); + } + public override string ToString() => Kind; // A node is serialized as an object with one property: its kind, holding the node's fields. @@ -153,5 +167,33 @@ public IEnumerable Queries() .OfType(); } + /// + /// Parses every expression and query in the function, in document order. Each item has the node, + /// its SQL text, and either the parse tree or the error. + /// + public IReadOnlyList ParseQueries() + { + var queries = new List(); + foreach (var node in Root.Descendants()) + { + if (node.Kind != "PLpgSQL_expr" || node.GetString("query") is not { } sql) + continue; + + var parsed = node.ParseQuery(); + queries.Add(new PlpgsqlQuery(node, sql, parsed.Value, parsed.Error)); + } + + return queries; + } + private static PlpgsqlNode? ToNode(JsonElement element) => PlpgsqlNode.TryCreate(element, out var node) ? node : null; } + +/// +/// A piece of SQL inside a PL/pgSQL function, parsed by . +/// +/// The PLpgSQL_expr node it comes from. +/// The SQL text as written in the function. +/// The parse tree, or null if it did not parse. +/// Why it did not parse, or null. +public sealed record PlpgsqlQuery(PlpgsqlNode Node, string Sql, ParseResult? Tree, Error? Error); diff --git a/src/PgSqlParser/QueryReferences.cs b/src/PgSqlParser/QueryReferences.cs index 0ccf70b..38948a4 100644 --- a/src/PgSqlParser/QueryReferences.cs +++ b/src/PgSqlParser/QueryReferences.cs @@ -17,6 +17,28 @@ public enum TableRole Ddl } +/// +/// The part of a statement a column reference is in. +/// +public enum QueryClause +{ + /// Somewhere not covered by the other values. + Other, + SelectList, + From, + + /// The ON condition of a join, or of MERGE. + JoinCondition, + Where, + GroupBy, + Having, + OrderBy, + Returning, + + /// The SET list of an UPDATE. + Set +} + /// /// A table named in a query. /// @@ -60,6 +82,12 @@ public sealed record FunctionReference(FuncCall Node, string? Schema, string Nam /// The top-level statement it is in, when known. public sealed record ColumnReference(ColumnRef Node, string? Qualifier, string Name, TableReference? Table, int? StatementIndex) { + /// + /// The clause of the nearest enclosing statement the column is in. Columns in + /// and are the ones that filter rows. + /// + public QueryClause Clause { get; init; } + public override string ToString() => Qualifier is null ? Name : $"{Qualifier}.{Name}"; } @@ -101,11 +129,16 @@ public static QueryReferences GetReferences(this IMessage root) var functions = new List(); var columnVisits = new List(); var tableByNode = new Dictionary(ReferenceEqualityComparer.Instance); + var visitByNode = new Dictionary(ReferenceEqualityComparer.Instance); foreach (var visit in root.Walk()) { + visitByNode[visit.Node] = visit; switch (visit.Node) { + // FOR UPDATE OF x names an item of the FROM clause, not another table. + case RangeVar when visit.Parent is LockingClause: + break; case RangeVar table: var reference = new TableReference(table, RoleOf(visit, top), IsCte(table, visit, top), visit.StatementIndex); tables.Add(reference); @@ -122,7 +155,9 @@ public static QueryReferences GetReferences(this IMessage root) } // Columns are resolved last, since a column can come before the table it belongs to. - var columns = columnVisits.Select(visit => ResolveColumn(visit, top, tableByNode)).ToList(); + var columns = columnVisits + .Select(visit => ResolveColumn(visit, top, tableByNode) with { Clause = ClauseOf(visit, top, visitByNode) }) + .ToList(); return new QueryReferences(tables, functions, columns); } @@ -185,6 +220,37 @@ private static bool IsCte(RangeVar table, NodeVisit visit, IMessage top) return false; } + // Climbs from the column to the nearest statement and maps the property it hangs from. + private static QueryClause ClauseOf(NodeVisit visit, IMessage top, Dictionary visitByNode) + { + var current = visit; + while (true) + { + var parent = current.Parent ?? top; + if (parent is JoinExpr && current.FieldName == nameof(JoinExpr.Quals)) + return QueryClause.JoinCondition; + + if (parent is SelectStmt or UpdateStmt or DeleteStmt or InsertStmt or MergeStmt) + { + return current.FieldName switch + { + nameof(SelectStmt.TargetList) => parent is UpdateStmt ? QueryClause.Set : QueryClause.SelectList, + nameof(SelectStmt.FromClause) or nameof(DeleteStmt.UsingClause) => QueryClause.From, + nameof(SelectStmt.WhereClause) => QueryClause.Where, + nameof(SelectStmt.GroupClause) => QueryClause.GroupBy, + nameof(SelectStmt.HavingClause) => QueryClause.Having, + nameof(SelectStmt.SortClause) => QueryClause.OrderBy, + nameof(MergeStmt.JoinCondition) => QueryClause.JoinCondition, + { } name when name.StartsWith("Returning", StringComparison.Ordinal) => QueryClause.Returning, + _ => QueryClause.Other + }; + } + + if (current.Parent is null || !visitByNode.TryGetValue(current.Parent, out current)) + return QueryClause.Other; + } + } + private static ColumnReference ResolveColumn(NodeVisit visit, IMessage top, Dictionary tableByNode) { var column = (ColumnRef)visit.Node; diff --git a/src/PgSqlParser/QueryShape.cs b/src/PgSqlParser/QueryShape.cs new file mode 100644 index 0000000..a354535 --- /dev/null +++ b/src/PgSqlParser/QueryShape.cs @@ -0,0 +1,133 @@ +using Google.Protobuf; + +namespace PgSqlParser; + +/// +/// A column a SELECT returns. +/// +/// +/// The name PostgreSQL gives the column: its alias, or the name it derives from the expression, such as +/// the column or function name. Null for *, whose columns only the catalog can list. +/// +/// The expression that produces it. +/// True for * and table.*, which stand for several columns. +public sealed record OutputColumn(string? Name, IMessage Expression, bool IsStar); + +/// +/// Reads the shape of a query's result from its parse tree. +/// +public static class QueryShape +{ + /// The name PostgreSQL uses for a column it cannot name from its expression. + public const string UnnamedColumn = "?column?"; + + /// + /// Returns the columns a SELECT returns, named as PostgreSQL names them. For a UNION, INTERSECT or + /// EXCEPT they come from the first branch, and for a VALUES list they are column1, column2, ... + /// + public static IReadOnlyList GetOutputColumns(this SelectStmt select) + { + ArgumentNullException.ThrowIfNull(select); + + // The leftmost branch of a set operation decides the names. + while (select.Op != SetOperation.SetopNone && select.Larg is not null) + select = select.Larg; + + if (select.ValuesLists.Count > 0) + { + var row = select.ValuesLists[0].List?.Items ?? []; + return row.Select((value, index) => new OutputColumn($"column{index + 1}", value.Unwrap() ?? value, false)).ToList(); + } + + var columns = new List(); + foreach (var item in select.TargetList) + { + if (item.ResTarget is not { } target) + continue; + + var expression = target.Val?.Unwrap(); + if (expression is ColumnRef { Fields.Count: > 0 } column && column.Fields[^1].AStar is not null) + columns.Add(new OutputColumn(null, column, true)); + else + columns.Add(new OutputColumn(target.Name.Length > 0 ? target.Name : FigureName(expression).Name ?? UnnamedColumn, + expression ?? target, false)); + } + + return columns; + } + + // A port of FigureColnameInternal in PostgreSQL's parse_target.c. Strength 2 is a good name, + // 1 a weak one (a type name, "case") that a better name from further in overrides. + private static (string? Name, int Strength) FigureName(IMessage? node) + { + switch (node) + { + case ColumnRef column: + return column.Fields.LastOrDefault(field => field.String is not null)?.String.Sval is { } name ? (name, 2) : (null, 0); + case A_Indirection indirection: + return indirection.Indirection.LastOrDefault(item => item.String is not null)?.String.Sval is { } field + ? (field, 2) + : FigureName(indirection.Arg?.Unwrap()); + case FuncCall call: + return (call.Funcname.LastOrDefault()?.String?.Sval, 2); + case A_Expr { Kind: A_Expr_Kind.AexprNullif }: + return ("nullif", 2); + case TypeCast cast: + var inner = FigureName(cast.Arg?.Unwrap()); + return inner.Strength <= 1 && cast.TypeName?.Names.LastOrDefault()?.String?.Sval is { } type ? (type, 1) : inner; + case CollateClause collate: + return FigureName(collate.Arg?.Unwrap()); + case GroupingFunc: + return ("grouping", 2); + case MergeSupportFunc: + return ("merge_action", 2); + case SubLink { SubLinkType: SubLinkType.ExistsSublink }: + return ("exists", 2); + case SubLink { SubLinkType: SubLinkType.ArraySublink }: + return ("array", 2); + case SubLink { SubLinkType: SubLinkType.ExprSublink } link when link.Subselect?.Unwrap() is SelectStmt subselect: + return subselect.GetOutputColumns().FirstOrDefault() is { IsStar: false, Name: { } first } && first != UnnamedColumn + ? (first, 2) + : (null, 0); + case CaseExpr @case: + var result = FigureName(@case.Defresult?.Unwrap()); + return result.Strength <= 1 ? ("case", 1) : result; + case A_ArrayExpr: + return ("array", 2); + case RowExpr: + return ("row", 2); + case CoalesceExpr: + return ("coalesce", 2); + case MinMaxExpr minMax: + return (minMax.Op == MinMaxOp.IsGreatest ? "greatest" : "least", 2); + case SQLValueFunction function: + return (function.Op switch + { + SQLValueFunctionOp.SvfopCurrentDate => "current_date", + SQLValueFunctionOp.SvfopCurrentTime or SQLValueFunctionOp.SvfopCurrentTimeN => "current_time", + SQLValueFunctionOp.SvfopCurrentTimestamp or SQLValueFunctionOp.SvfopCurrentTimestampN => "current_timestamp", + SQLValueFunctionOp.SvfopLocaltime or SQLValueFunctionOp.SvfopLocaltimeN => "localtime", + SQLValueFunctionOp.SvfopLocaltimestamp or SQLValueFunctionOp.SvfopLocaltimestampN => "localtimestamp", + SQLValueFunctionOp.SvfopCurrentRole => "current_role", + SQLValueFunctionOp.SvfopCurrentUser => "current_user", + SQLValueFunctionOp.SvfopUser => "user", + SQLValueFunctionOp.SvfopSessionUser => "session_user", + SQLValueFunctionOp.SvfopCurrentCatalog => "current_catalog", + SQLValueFunctionOp.SvfopCurrentSchema => "current_schema", + _ => null + }, 2); + case XmlSerialize: + return ("xmlserialize", 2); + case JsonObjectConstructor: + return ("json_object", 2); + case JsonArrayConstructor or JsonArrayQueryConstructor: + return ("json_array", 2); + case JsonObjectAgg: + return ("json_objectagg", 2); + case JsonArrayAgg: + return ("json_arrayagg", 2); + default: + return (null, 0); + } + } +} diff --git a/src/PgSqlParser/SqlTokens.cs b/src/PgSqlParser/SqlTokens.cs new file mode 100644 index 0000000..d7df110 --- /dev/null +++ b/src/PgSqlParser/SqlTokens.cs @@ -0,0 +1,54 @@ +namespace PgSqlParser; + +/// +/// What a token of a query is, for syntax highlighting and similar uses. +/// +public enum SqlTokenKind +{ + Other, + Keyword, + Identifier, + StringLiteral, + NumericLiteral, + Parameter, + Operator, + Punctuation, + Comment +} + +/// +/// A token of a query. +/// +/// Where it starts in the query string, as a UTF-16 offset. +/// Where it ends (exclusive), so query[Start..End] is its text. +/// What it is. +/// The scanner's own token, for finer distinctions. +/// For a keyword, how reserved it is. +public readonly record struct SqlToken(int Start, int End, SqlTokenKind Kind, Token Token, KeywordKind KeywordKind); + +internal static class SqlTokenClassifier +{ + public static SqlTokenKind KindOf(Token token, KeywordKind keywordKind) + { + if (keywordKind != KeywordKind.NoKeyword) + return SqlTokenKind.Keyword; + + return token switch + { + Token.Ident or Token.Uident => SqlTokenKind.Identifier, + Token.Sconst or Token.Usconst or Token.Bconst or Token.Xconst => SqlTokenKind.StringLiteral, + Token.Iconst or Token.Fconst => SqlTokenKind.NumericLiteral, + Token.Param => SqlTokenKind.Parameter, + Token.SqlComment or Token.CComment => SqlTokenKind.Comment, + Token.Op or Token.Typecast or Token.DotDot or Token.ColonEquals or Token.EqualsGreater or Token.LessEquals + or Token.GreaterEquals or Token.NotEquals => SqlTokenKind.Operator, + // + - * / % ^ < > = and the lone $ are operators. + Token.Ascii37 or Token.Ascii42 or Token.Ascii43 or Token.Ascii45 or Token.Ascii47 or Token.Ascii60 + or Token.Ascii61 or Token.Ascii62 or Token.Ascii94 => SqlTokenKind.Operator, + // ( ) , . : ; [ ] and the rest. + Token.Ascii36 or Token.Ascii40 or Token.Ascii41 or Token.Ascii44 or Token.Ascii46 or Token.Ascii58 + or Token.Ascii59 or Token.Ascii63 or Token.Ascii91 or Token.Ascii92 or Token.Ascii93 => SqlTokenKind.Punctuation, + _ => SqlTokenKind.Other + }; + } +} diff --git a/src/PgSqlParser/StatementLocks.cs b/src/PgSqlParser/StatementLocks.cs new file mode 100644 index 0000000..3ab742e --- /dev/null +++ b/src/PgSqlParser/StatementLocks.cs @@ -0,0 +1,379 @@ +using Google.Protobuf; + +namespace PgSqlParser; + +/// +/// PostgreSQL's table-level lock modes, weakest first. The values are PostgreSQL's own lock mode numbers. +/// +public enum LockMode +{ + AccessShare = 1, + RowShare = 2, + RowExclusive = 3, + ShareUpdateExclusive = 4, + Share = 5, + ShareRowExclusive = 6, + Exclusive = 7, + AccessExclusive = 8 +} + +/// +/// A table lock a statement takes. +/// +/// The schema as written in the statement, or null. +/// The table name. +/// The lock mode. +/// The top-level statement that takes it, when known. +public sealed record TableLock(string? Schema, string Table, LockMode Mode, int? StatementIndex) +{ + public override string ToString() => $"{(Schema is null ? Table : $"{Schema}.{Table}")}: {Mode}"; +} + +/// +/// Works out which table locks a statement takes, from its parse tree and PostgreSQL's documented +/// rules. See for what is covered. +/// +public static class StatementLocks +{ + // The conflict table of PostgreSQL's "Table-Level Lock Modes" documentation, as bit masks by mode number. + private static readonly int[] Conflicts = + [ + 0, + Mask(LockMode.AccessExclusive), + Mask(LockMode.Exclusive, LockMode.AccessExclusive), + Mask(LockMode.Share, LockMode.ShareRowExclusive, LockMode.Exclusive, LockMode.AccessExclusive), + Mask(LockMode.ShareUpdateExclusive, LockMode.Share, LockMode.ShareRowExclusive, LockMode.Exclusive, LockMode.AccessExclusive), + Mask(LockMode.RowExclusive, LockMode.ShareUpdateExclusive, LockMode.ShareRowExclusive, LockMode.Exclusive, LockMode.AccessExclusive), + Mask(LockMode.RowExclusive, LockMode.ShareUpdateExclusive, LockMode.Share, LockMode.ShareRowExclusive, LockMode.Exclusive, LockMode.AccessExclusive), + Mask(LockMode.RowShare, LockMode.RowExclusive, LockMode.ShareUpdateExclusive, LockMode.Share, LockMode.ShareRowExclusive, LockMode.Exclusive, LockMode.AccessExclusive), + Mask(LockMode.AccessShare, LockMode.RowShare, LockMode.RowExclusive, LockMode.ShareUpdateExclusive, LockMode.Share, LockMode.ShareRowExclusive, LockMode.Exclusive, LockMode.AccessExclusive) + ]; + + // Storage parameters whose change needs ACCESS EXCLUSIVE; the other known ones need SHARE UPDATE + // EXCLUSIVE. From the relOpts tables in PostgreSQL 18's reloptions.c. + private static readonly HashSet AccessExclusiveOptions = + [ + "autosummarize", "buffering", "check_option", "fastupdate", "gin_pending_list_limit", "pages_per_range", + "security_barrier", "security_invoker", "user_catalog_table" + ]; + + private static readonly HashSet ShareUpdateExclusiveOptions = + [ + "autovacuum_analyze_scale_factor", "autovacuum_analyze_threshold", "autovacuum_enabled", + "autovacuum_freeze_max_age", "autovacuum_freeze_min_age", "autovacuum_freeze_table_age", + "autovacuum_multixact_freeze_max_age", "autovacuum_multixact_freeze_min_age", + "autovacuum_multixact_freeze_table_age", "autovacuum_vacuum_cost_delay", "autovacuum_vacuum_cost_limit", + "autovacuum_vacuum_insert_scale_factor", "autovacuum_vacuum_insert_threshold", + "autovacuum_vacuum_max_threshold", "autovacuum_vacuum_scale_factor", "autovacuum_vacuum_threshold", + "deduplicate_items", "effective_io_concurrency", "fillfactor", "log_autovacuum_min_duration", + "maintenance_io_concurrency", "n_distinct", "n_distinct_inherited", "parallel_workers", "random_page_cost", + "seq_page_cost", "toast_tuple_target", "vacuum_cleanup_index_scale_factor", "vacuum_index_cleanup", + "vacuum_max_eager_freeze_failure_rate", "vacuum_truncate" + ]; + + /// True if a session holding on a table blocks one asking for , and the reverse. + public static bool ConflictsWith(this LockMode mode, LockMode other) => (Conflicts[(int)mode] & Mask(other)) != 0; + + /// True if the lock blocks plain SELECTs on the table. Only ACCESS EXCLUSIVE does. + public static bool BlocksReads(this LockMode mode) => mode.ConflictsWith(LockMode.AccessShare); + + /// True if the lock blocks INSERT, UPDATE and DELETE on the table. + public static bool BlocksWrites(this LockMode mode) => mode.ConflictsWith(LockMode.RowExclusive); + + /// + /// Returns the table locks each statement takes on the tables it names, one entry per table and + /// statement with the strongest mode. + /// + /// Covered: SELECT (with and without FOR UPDATE and similar), INSERT, UPDATE, DELETE, MERGE, COPY, + /// TRUNCATE, LOCK, CREATE INDEX, CREATE TRIGGER, CREATE STATISTICS, ALTER TABLE (per sub-command, as + /// PostgreSQL's AlterTableGetLockLevel decides it), DROP TABLE and similar, VACUUM, ANALYZE, CLUSTER, + /// REINDEX TABLE, REFRESH MATERIALIZED VIEW, COMMENT ON TABLE or COLUMN, and the table referenced by + /// a new foreign key. + /// + /// + /// Not covered, because the statement does not name the table or it takes the catalog to tell: locks + /// on a table reached through a view, an index (DROP INDEX, REINDEX INDEX, ALTER INDEX), a partition + /// or an inheritance child, and locks taken by triggers and functions. Other statement types give + /// no entries. Row-level locks are not included. + /// + /// + public static IReadOnlyList GetLocks(this ParseResult tree) + { + ArgumentNullException.ThrowIfNull(tree); + + var locks = new List(); + for (var i = 0; i < tree.Stmts.Count; i++) + locks.AddRange(LocksOf(tree.Stmts[i].Stmt?.Unwrap(), i)); + + return locks; + } + + /// Returns the table locks one statement takes. See . + public static IReadOnlyList GetLocks(this RawStmt statement) + { + ArgumentNullException.ThrowIfNull(statement); + return LocksOf(statement.Stmt?.Unwrap(), null); + } + + private static List LocksOf(IMessage? statement, int? index) + { + var found = new List<(string? Schema, string Table, LockMode Mode)>(); + if (statement is not null) + Collect(statement, found); + + // One entry per table, with the strongest mode, in the order first seen. + return found + .GroupBy(item => (item.Schema, item.Table)) + .Select(group => new TableLock(group.Key.Schema, group.Key.Table, group.Max(item => item.Mode), index)) + .ToList(); + } + + private static void Collect(IMessage statement, List<(string?, string, LockMode)> found) + { + void Add(RangeVar? table, LockMode mode) + { + if (table is not null) + found.Add((table.Schemaname.Length > 0 ? table.Schemaname : null, table.Relname, mode)); + } + + switch (statement) + { + case SelectStmt or InsertStmt or UpdateStmt or DeleteStmt or MergeStmt: + AddQueryLocks(statement, found); + break; + + case CopyStmt copy: + Add(copy.Relation, copy.IsFrom ? LockMode.RowExclusive : LockMode.AccessShare); + if (copy.Query?.Unwrap() is { } query) + AddQueryLocks(query, found); + break; + + case ExplainStmt explain when explain.Query?.Unwrap() is { } explained: + // EXPLAIN plans the statement, which takes the same table locks as running it. + Collect(explained, found); + break; + + case TruncateStmt truncate: + foreach (var relation in truncate.Relations) + Add(relation.RangeVar, LockMode.AccessExclusive); + break; + + case LockStmt explicitLock: + foreach (var relation in explicitLock.Relations) + Add(relation.RangeVar, (LockMode)explicitLock.Mode); + break; + + case IndexStmt index: + Add(index.Relation, index.Concurrent ? LockMode.ShareUpdateExclusive : LockMode.Share); + break; + + case CreateTrigStmt trigger: + Add(trigger.Relation, LockMode.ShareRowExclusive); + Add(trigger.Constrrel, LockMode.ShareRowExclusive); + break; + + case CreateStatsStmt statistics: + foreach (var relation in statistics.Relations) + Add(relation.RangeVar, LockMode.ShareUpdateExclusive); + break; + + case CreateStmt create: + // A new table is not locked against anyone, but a foreign key locks the table it points to. + foreach (var constraint in create.Descendants()) + { + if (constraint.Contype == ConstrType.ConstrForeign) + Add(constraint.Pktable, LockMode.ShareRowExclusive); + } + + break; + + case AlterTableStmt alter when alter.Objtype is ObjectType.ObjectTable or ObjectType.ObjectForeignTable + or ObjectType.ObjectMatview or ObjectType.ObjectView: + AddAlterTableLocks(alter, found); + break; + + case RenameStmt rename when rename.Relation is not null && rename.RelationType != ObjectType.ObjectIndex + && rename.RenameType != ObjectType.ObjectIndex: + Add(rename.Relation, LockMode.AccessExclusive); + break; + + case DropStmt drop when drop.RemoveType is ObjectType.ObjectTable or ObjectType.ObjectView + or ObjectType.ObjectMatview or ObjectType.ObjectForeignTable: + foreach (var target in drop.Objects) + AddByName(target.List?.Items, LockMode.AccessExclusive, found); + break; + + case VacuumStmt vacuum: + // VACUUM FULL rewrites the table; plain VACUUM and ANALYZE do not. + var full = vacuum.IsVacuumcmd && vacuum.Options.Any(option => IsOn(option, "full")); + foreach (var relation in vacuum.Rels) + Add(relation.VacuumRelation?.Relation, full ? LockMode.AccessExclusive : LockMode.ShareUpdateExclusive); + break; + + case ClusterStmt cluster: + Add(cluster.Relation, LockMode.AccessExclusive); + break; + + case ReindexStmt { Kind: ReindexObjectType.ReindexObjectTable } reindex: + // REINDEX TABLE blocks writes but not reads of the table; CONCURRENTLY blocks neither. + Add(reindex.Relation, reindex.Params.Any(option => IsOn(option, "concurrently")) + ? LockMode.ShareUpdateExclusive + : LockMode.Share); + break; + + case RefreshMatViewStmt refresh: + Add(refresh.Relation, refresh.Concurrent ? LockMode.Exclusive : LockMode.AccessExclusive); + break; + + case CommentStmt { Objtype: ObjectType.ObjectTable } comment: + AddByName(comment.Object?.List?.Items, LockMode.ShareUpdateExclusive, found); + break; + + case CommentStmt { Objtype: ObjectType.ObjectColumn } comment + when comment.Object?.List?.Items is { Count: > 1 } parts: + // The last part is the column. + AddByName(parts.Take(parts.Count - 1).ToList(), LockMode.ShareUpdateExclusive, found); + break; + } + } + + // SELECT and data-changing statements: the target is locked ROW EXCLUSIVE, everything read is locked + // ACCESS SHARE, or ROW SHARE under FOR UPDATE and similar. + private static void AddQueryLocks(IMessage statement, List<(string?, string, LockMode)> found) + { + var references = statement.GetReferences(); + var visits = new Dictionary(ReferenceEqualityComparer.Instance); + foreach (var visit in statement.Walk()) + { + if (visit.Node is RangeVar node) + visits[node] = visit; + } + + foreach (var table in references.Tables) + { + if (table.IsCte) + continue; + + var mode = table.Role switch + { + TableRole.Write => LockMode.RowExclusive, + // SELECT ... INTO creates its target, which nobody else can see yet. + TableRole.Ddl => (LockMode?)null, + _ => IsUnderLockingClause(visits[table.Node], statement) ? LockMode.RowShare : LockMode.AccessShare + }; + + if (mode is { } known) + found.Add((table.Schema, table.Name, known)); + } + } + + // FOR UPDATE and similar apply to the tables in the FROM clause of the SELECT that carries them. + private static bool IsUnderLockingClause(NodeVisit visit, IMessage top) + { + var select = visit.FindAncestor() ?? top as SelectStmt; + if (select is null || select.LockingClause.Count == 0) + return false; + + var table = (RangeVar)visit.Node; + var name = table.Alias is { Aliasname.Length: > 0 } alias ? alias.Aliasname : table.Relname; + foreach (var clause in select.LockingClause) + { + var locked = clause.LockingClause.LockedRels; + // Without OF, every table of the FROM clause is locked. + if (locked.Count == 0 || locked.Any(item => item.RangeVar?.Relname == name)) + return true; + } + + return false; + } + + // The lock of an ALTER TABLE is the strongest any of its sub-commands needs, following + // AlterTableGetLockLevel in PostgreSQL 18's tablecmds.c. + private static void AddAlterTableLocks(AlterTableStmt alter, List<(string?, string, LockMode)> found) + { + var mode = LockMode.AccessShare; + var any = false; + foreach (var item in alter.Cmds) + { + if (item.AlterTableCmd is not { } command) + continue; + + any = true; + var needed = command.Subtype switch + { + AlterTableType.AtEnableTrig or AlterTableType.AtEnableAlwaysTrig or AlterTableType.AtEnableReplicaTrig + or AlterTableType.AtEnableTrigAll or AlterTableType.AtEnableTrigUser or AlterTableType.AtDisableTrig + or AlterTableType.AtDisableTrigAll or AlterTableType.AtDisableTrigUser => LockMode.ShareRowExclusive, + + AlterTableType.AtAddConstraint or AlterTableType.AtReAddConstraint or AlterTableType.AtReAddDomainConstraint + when command.Def?.Constraint is { Contype: ConstrType.ConstrForeign } => LockMode.ShareRowExclusive, + + AlterTableType.AtSetStatistics or AlterTableType.AtClusterOn or AlterTableType.AtDropCluster + or AlterTableType.AtSetOptions or AlterTableType.AtResetOptions or AlterTableType.AtValidateConstraint + or AlterTableType.AtAttachPartition or AlterTableType.AtDetachPartitionFinalize => LockMode.ShareUpdateExclusive, + + AlterTableType.AtDetachPartition when command.Def?.PartitionCmd is { Concurrent: true } => LockMode.ShareUpdateExclusive, + + AlterTableType.AtSetRelOptions or AlterTableType.AtResetRelOptions => OptionsLock(command.Def?.List?.Items), + + _ => LockMode.AccessExclusive + }; + + if (needed > mode) + mode = needed; + + // A new foreign key also locks the table it points to. + if (command.Def?.Constraint is { Contype: ConstrType.ConstrForeign, Pktable: { } referenced }) + found.Add((referenced.Schemaname.Length > 0 ? referenced.Schemaname : null, referenced.Relname, LockMode.ShareRowExclusive)); + } + + if (any && alter.Relation is { } relation) + found.Add((relation.Schemaname.Length > 0 ? relation.Schemaname : null, relation.Relname, mode)); + } + + // AlterTableGetRelOptionsLockLevel: the strongest lock any of the named options needs. + private static LockMode OptionsLock(IEnumerable? options) + { + LockMode? mode = null; + foreach (var option in options ?? []) + { + var name = option.DefElem?.Defname; + if (name is null) + continue; + + if (AccessExclusiveOptions.Contains(name)) + return LockMode.AccessExclusive; + + if (ShareUpdateExclusiveOptions.Contains(name)) + mode = LockMode.ShareUpdateExclusive; + } + + // Options this table does not know are treated as needing the strongest lock. + return mode ?? LockMode.AccessExclusive; + } + + private static void AddByName(IEnumerable? parts, LockMode mode, List<(string?, string, LockMode)> found) + { + var names = parts?.Select(part => part.String?.Sval).OfType().ToList(); + if (names is { Count: > 0 }) + found.Add((names.Count > 1 ? names[^2] : null, names[^1], mode)); + } + + // An option given as a bare word, or with a true value. + private static bool IsOn(Node option, string name) + { + if (option.DefElem is not { } element || element.Defname != name) + return false; + + return element.Arg?.Unwrap() switch + { + null => true, + Boolean boolean => boolean.Boolval, + Integer integer => integer.Ival != 0, + String text => text.Sval is "true" or "on" or "1", + _ => true + }; + } + + private static int Mask(params LockMode[] modes) => modes.Aggregate(0, (mask, mode) => mask | 1 << (int)mode); +}